diff --git a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala index ad31d4c5..a303f481 100644 --- a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala +++ b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala @@ -134,6 +134,65 @@ object SjsonnetMainBase { importer: Option[Importer], std: Val.Obj, jsonnetPathEnv: Option[String], + rawOutputStream: OutputStream): Int = + main0Impl( + args, + parseCache, + stdout, + stderr, + wd, + allowedInputs, + importer, + std, + null, + lazyStdProvider = false, + jsonnetPathEnv, + rawOutputStream + ) + + /** + * Internal CLI entry point for platforms that provide augmented stdlib modules. `stdProvider` is + * called at most once, and only if the Jsonnet program references `std`. + */ + private[sjsonnet] def main0WithStdProvider( + args: Array[String], + parseCache: ParseCache, + @unused stdin: InputStream, + stdout: PrintStream, + stderr: PrintStream, + wd: os.Path, + allowedInputs: Option[Set[os.Path]], + importer: Option[Importer], + stdProvider: () => Val.Obj, + jsonnetPathEnv: Option[String], + rawOutputStream: OutputStream): Int = + main0Impl( + args, + parseCache, + stdout, + stderr, + wd, + allowedInputs, + importer, + null, + stdProvider, + lazyStdProvider = true, + jsonnetPathEnv, + rawOutputStream + ) + + private def main0Impl( + args: Array[String], + parseCache: ParseCache, + stdout: PrintStream, + stderr: PrintStream, + wd: os.Path, + allowedInputs: Option[Set[os.Path]], + importer: Option[Importer], + std: Val.Obj, + stdProvider: () => Val.Obj, + lazyStdProvider: Boolean, + jsonnetPathEnv: Option[String], rawOutputStream: OutputStream): Int = { var hasWarnings = false @@ -180,32 +239,52 @@ object SjsonnetMainBase { debugStats = if (config.debugStats.value) { val s = new DebugStats; statsToReport = s; s } else null - outputStr <- mainConfigured( - file, - config, - new Settings( - preserveOrder = config.preserveOrder.value, - strict = config.strict.value, - throwErrorForInvalidSets = config.throwErrorForInvalidSets.value, - maxParserRecursionDepth = config.maxParserRecursionDepth, - brokenAssertionLogic = config.brokenAssertionLogic.value, - maxStack = config.maxStack - ), - parseCache, - wd, - importer.getOrElse { - new SimpleImporter( - config.getOrderedJpaths(jsonnetPathEnv).map(p => OsPath(os.Path(p, wd))), - allowedInputs, - debugImporter = config.debugImporter.value - ) - }, - warn, - std, - debugStats = debugStats, - profileOpt = config.profile, - stdoutStream = if (rawOutputStream != null) rawOutputStream else stdout + settings = new Settings( + preserveOrder = config.preserveOrder.value, + strict = config.strict.value, + throwErrorForInvalidSets = config.throwErrorForInvalidSets.value, + maxParserRecursionDepth = config.maxParserRecursionDepth, + brokenAssertionLogic = config.brokenAssertionLogic.value, + maxStack = config.maxStack ) + resolvedImporter = importer.getOrElse { + new SimpleImporter( + config.getOrderedJpaths(jsonnetPathEnv).map(p => OsPath(os.Path(p, wd))), + allowedInputs, + debugImporter = config.debugImporter.value + ) + } + stdoutTarget = if (rawOutputStream != null) rawOutputStream else stdout + outputStr <- { + if (lazyStdProvider) + mainConfiguredWithStdProvider( + file, + config, + settings, + parseCache, + wd, + resolvedImporter, + warn, + stdProvider, + debugStats = debugStats, + profileOpt = config.profile, + stdoutStream = stdoutTarget + ) + else + mainConfigured( + file, + config, + settings, + parseCache, + wd, + resolvedImporter, + warn, + std, + debugStats = debugStats, + profileOpt = config.profile, + stdoutStream = stdoutTarget + ) + } res <- { if (hasWarnings && config.fatalWarnings.value) Left("") else Right(outputStr) @@ -393,7 +472,73 @@ object SjsonnetMainBase { evaluatorOverride: Option[Evaluator] = None, debugStats: DebugStats = null, profileOpt: Option[String] = None, - stdoutStream: OutputStream = null): Either[String, String] = { + stdoutStream: OutputStream = null): Either[String, String] = + mainConfiguredImpl( + file, + config, + settings, + parseCache, + wd, + importer, + warnLogger, + std, + null, + lazyStdProvider = false, + evaluatorOverride, + debugStats, + profileOpt, + stdoutStream + ) + + /** + * Internal configured entry point for platforms that provide augmented stdlib modules. + * `stdProvider` is called at most once, and only if the Jsonnet program references `std`. + */ + private[sjsonnet] def mainConfiguredWithStdProvider( + file: String, + config: Config, + settings: Settings, + parseCache: ParseCache, + wd: os.Path, + importer: Importer, + warnLogger: Evaluator.Logger, + stdProvider: () => Val.Obj, + evaluatorOverride: Option[Evaluator] = None, + debugStats: DebugStats = null, + profileOpt: Option[String] = None, + stdoutStream: OutputStream = null): Either[String, String] = + mainConfiguredImpl( + file, + config, + settings, + parseCache, + wd, + importer, + warnLogger, + null, + stdProvider, + lazyStdProvider = true, + evaluatorOverride, + debugStats, + profileOpt, + stdoutStream + ) + + private def mainConfiguredImpl( + file: String, + config: Config, + settings: Settings, + parseCache: ParseCache, + wd: os.Path, + importer: Importer, + warnLogger: Evaluator.Logger, + std: Val.Obj, + stdProvider: () => Val.Obj, + lazyStdProvider: Boolean, + evaluatorOverride: Option[Evaluator], + debugStats: DebugStats, + profileOpt: Option[String], + stdoutStream: OutputStream): Either[String, String] = { val (jsonnetCode, path) = if (config.exec.value) (file, wd / Util.wrapInLessThanGreaterThan("exec")) @@ -435,35 +580,69 @@ object SjsonnetMainBase { var currentPos: Position = null var profilerInstance: Profiler = null - val interp = new Interpreter( - queryExtVar = (key: String) => extBinding.get(key).map(ExternalVariable.code), - queryTlaVar = (key: String) => tlaBinding.get(key).map(ExternalVariable.code), - OsPath(wd), - importer = importer, - parseCache, - settings = settings, - storePos = (position: Position) => if (config.yamlDebug.value) currentPos = position else (), - logger = warnLogger, - std = std, - variableResolver = _ => None, - debugStats = debugStats, - formatCache = FormatCache.SharedDefault - ) { - override def createEvaluator( - resolver: CachedResolver, - extVars: String => Option[Expr], - wd: Path, - settings: Settings): Evaluator = { - val ev = evaluatorOverride.getOrElse( - super.createEvaluator(resolver, extVars, wd, settings) - ) - profileFormat.foreach { fmt => - profilerInstance = new Profiler(fmt, wd) - ev.profiler = profilerInstance + val interp = + if (lazyStdProvider) + new Interpreter( + queryExtVar = (key: String) => extBinding.get(key).map(ExternalVariable.code), + queryTlaVar = (key: String) => tlaBinding.get(key).map(ExternalVariable.code), + OsPath(wd), + importer = importer, + parseCache, + settings = settings, + storePos = (position: Position) => + if (config.yamlDebug.value) currentPos = position else (), + logger = warnLogger, + stdProvider = stdProvider, + variableResolver = _ => None, + debugStats = debugStats, + formatCache = FormatCache.SharedDefault + ) { + override def createEvaluator( + resolver: CachedResolver, + extVars: String => Option[Expr], + wd: Path, + settings: Settings): Evaluator = { + val ev = evaluatorOverride.getOrElse( + super.createEvaluator(resolver, extVars, wd, settings) + ) + profileFormat.foreach { fmt => + profilerInstance = new Profiler(fmt, wd) + ev.profiler = profilerInstance + } + ev + } + } + else + new Interpreter( + queryExtVar = (key: String) => extBinding.get(key).map(ExternalVariable.code), + queryTlaVar = (key: String) => tlaBinding.get(key).map(ExternalVariable.code), + OsPath(wd), + importer = importer, + parseCache, + settings = settings, + storePos = (position: Position) => + if (config.yamlDebug.value) currentPos = position else (), + logger = warnLogger, + std = std, + variableResolver = _ => None, + debugStats = debugStats, + formatCache = FormatCache.SharedDefault + ) { + override def createEvaluator( + resolver: CachedResolver, + extVars: String => Option[Expr], + wd: Path, + settings: Settings): Evaluator = { + val ev = evaluatorOverride.getOrElse( + super.createEvaluator(resolver, extVars, wd, settings) + ) + profileFormat.foreach { fmt => + profilerInstance = new Profiler(fmt, wd) + ev.profiler = profilerInstance + } + ev + } } - ev - } - } val result = (config.multi, config.yamlStream.value) match { case (Some(multiPath), _) => diff --git a/sjsonnet/src-jvm/sjsonnet/SjsonnetMain.scala b/sjsonnet/src-jvm/sjsonnet/SjsonnetMain.scala index 6976019e..6cfc6315 100644 --- a/sjsonnet/src-jvm/sjsonnet/SjsonnetMain.scala +++ b/sjsonnet/src-jvm/sjsonnet/SjsonnetMain.scala @@ -4,7 +4,7 @@ import sjsonnet.stdlib.{NativeGzip, NativeRegex, NativeXz} object SjsonnetMain { def main(args: Array[String]): Unit = { - val exitCode = SjsonnetMainBase.main0( + val exitCode = SjsonnetMainBase.main0WithStdProvider( args, new DefaultParseCache, System.in, @@ -12,9 +12,13 @@ object SjsonnetMain { System.err, os.pwd, None, - std = new sjsonnet.stdlib.StdLibModule(nativeFunctions = - Map() ++ NativeXz.functions ++ NativeGzip.functions ++ NativeRegex.functions - ).module + None, + () => + new sjsonnet.stdlib.StdLibModule(nativeFunctions = + Map() ++ NativeXz.functions ++ NativeGzip.functions ++ NativeRegex.functions + ).module, + None, + null ) System.exit(exitCode) } diff --git a/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala b/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala index cb2deb55..fcc9db34 100644 --- a/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala +++ b/sjsonnet/src-native/sjsonnet/SjsonnetMain.scala @@ -5,7 +5,7 @@ import scala.scalanative.libc.stdio object SjsonnetMain { def main(args: Array[String]): Unit = { - val exitCode = SjsonnetMainBase.main0( + val exitCode = SjsonnetMainBase.main0WithStdProvider( args, new DefaultParseCache, System.in, @@ -14,9 +14,10 @@ object SjsonnetMain { os.pwd, None, None, - new sjsonnet.stdlib.StdLibModule(nativeFunctions = - Map.from(NativeGzip.functions ++ NativeRegex.functions) - ).module, + () => + new sjsonnet.stdlib.StdLibModule(nativeFunctions = + Map.from(NativeGzip.functions ++ NativeRegex.functions) + ).module, None, new NativeOutputStream(stdio.stdout) ) diff --git a/sjsonnet/src/sjsonnet/BaseByteRenderer.scala b/sjsonnet/src/sjsonnet/BaseByteRenderer.scala index ce5f4907..2dbc6428 100644 --- a/sjsonnet/src/sjsonnet/BaseByteRenderer.scala +++ b/sjsonnet/src/sjsonnet/BaseByteRenderer.scala @@ -319,9 +319,8 @@ class BaseByteRenderer[T <: java.io.OutputStream]( arr(pos + 1 + bLen) = '"'.toByte elemBuilder.length = pos + bLen + 2 } else { - val escapedLen = escapedStringLength(bytes, bLen, firstEscape) - elemBuilder.ensureLength(escapedLen) - val arr = elemBuilder.arr + elemBuilder.ensureLength(bLen + 2 + (bLen >>> 5)) + var arr = elemBuilder.arr var outPos = elemBuilder.length arr(outPos) = '"'.toByte outPos += 1 @@ -330,41 +329,39 @@ class BaseByteRenderer[T <: java.io.OutputStream]( while (escPos >= 0) { if (escPos > from) { val chunkLen = escPos - from + elemBuilder.length = outPos + elemBuilder.ensureLength(chunkLen + 6) + arr = elemBuilder.arr + outPos = elemBuilder.length System.arraycopy(bytes, from, arr, outPos, chunkLen) outPos += chunkLen } + elemBuilder.length = outPos + elemBuilder.ensureLength(6) + arr = elemBuilder.arr + outPos = elemBuilder.length outPos = escapeByteInline(bytes(escPos) & 0xff, arr, outPos) from = escPos + 1 escPos = if (from < bLen) CharSWAR.findFirstEscapeChar(bytes, from, bLen) else -1 } if (from < bLen) { val tailLen = bLen - from + elemBuilder.length = outPos + elemBuilder.ensureLength(tailLen + 1) + arr = elemBuilder.arr + outPos = elemBuilder.length System.arraycopy(bytes, from, arr, outPos, tailLen) outPos += tailLen } + elemBuilder.length = outPos + elemBuilder.ensureLength(1) + arr = elemBuilder.arr + outPos = elemBuilder.length arr(outPos) = '"'.toByte elemBuilder.length = outPos + 1 } } - private def escapedStringLength(bytes: Array[Byte], bLen: Int, firstEscape: Int): Int = { - var len = bLen + 2 - var from = firstEscape - var escPos = firstEscape - while (escPos >= 0) { - len += escapeExtraLength(bytes(escPos) & 0xff) - from = escPos + 1 - escPos = if (from < bLen) CharSWAR.findFirstEscapeChar(bytes, from, bLen) else -1 - } - len - } - - @inline private def escapeExtraLength(b: Int): Int = - (b: @scala.annotation.switch) match { - case '"' | '\\' | '\b' | '\f' | '\n' | '\r' | '\t' => 1 - case _ => 5 - } - /** Inline JSON escape for one byte that is known to require escaping. */ @inline private def escapeByteInline(b: Int, arr: Array[Byte], outPos0: Int): Int = { val outPos = outPos0 diff --git a/sjsonnet/src/sjsonnet/Interpreter.scala b/sjsonnet/src/sjsonnet/Interpreter.scala index d40adf2e..a94de624 100644 --- a/sjsonnet/src/sjsonnet/Interpreter.scala +++ b/sjsonnet/src/sjsonnet/Interpreter.scala @@ -75,6 +75,37 @@ class Interpreter( FormatCache.SharedDefault ) + private[sjsonnet] def this( + queryExtVar: String => Option[ExternalVariable[?]], + queryTlaVar: String => Option[ExternalVariable[?]], + wd: Path, + importer: Importer, + parseCache: ParseCache, + settings: Settings, + storePos: Position => Unit, + logger: Evaluator.Logger, + stdProvider: () => Val.Obj, + variableResolver: String => Option[Expr], + debugStats: DebugStats, + formatCache: FormatCache) = { + this( + queryExtVar, + queryTlaVar, + wd, + importer, + parseCache, + settings, + storePos, + logger, + null.asInstanceOf[Val.Obj], + variableResolver, + debugStats, + formatCache + ) + this.stdProvider = stdProvider + this.lazyStdProviderMode = true + } + private val noOffsetPos = new Position(new FileScope(wd), -1) protected val internedStrings = new mutable.HashMap[String, String] @@ -84,6 +115,14 @@ class Interpreter( java.util.LinkedHashMap[String, java.lang.Boolean] ] + private var stdProvider: () => Val.Obj = () => std + private var lazyStdProviderMode: Boolean = false + + private lazy val std0: Val.Obj = { + val explicitStd = stdProvider() + if (explicitStd == null) sjsonnet.stdlib.StdLibModule.Default.module else explicitStd + } + val resolver: CachedResolver = createResolver( if (debugStats != null) new CountingParseCache(parseCache, debugStats) else parseCache ) @@ -109,7 +148,11 @@ class Interpreter( override def process(expr: Expr, fs: FileScope): Either[Error, (Expr, FileScope)] = { handleException( ( - createOptimizer(evaluator, std, internedStrings, internedStaticFieldSets).optimize(expr), + ( + if (lazyStdProviderMode) + createOptimizer(evaluator, () => std0, internedStrings, internedStaticFieldSets) + else createOptimizer(evaluator, std0, internedStrings, internedStaticFieldSets) + ).optimize(expr), fs ) ) @@ -127,6 +170,17 @@ class Interpreter( new StaticOptimizer(ev, variableResolver, std, internedStrings, internedStaticFieldSets) } + protected def createOptimizer( + ev: EvalScope, + stdProvider: () => Val.Obj, + internedStrings: mutable.HashMap[String, String], + internedStaticFieldSets: mutable.HashMap[ + Val.StaticObjectFieldSet, + java.util.LinkedHashMap[String, java.lang.Boolean] + ]): StaticOptimizer = { + new StaticOptimizer(ev, variableResolver, stdProvider, internedStrings, internedStaticFieldSets) + } + /** * A cache for parsing variables. */ diff --git a/sjsonnet/src/sjsonnet/StaticOptimizer.scala b/sjsonnet/src/sjsonnet/StaticOptimizer.scala index 1ed07ac3..ac53b7f8 100644 --- a/sjsonnet/src/sjsonnet/StaticOptimizer.scala +++ b/sjsonnet/src/sjsonnet/StaticOptimizer.scala @@ -31,6 +31,24 @@ class StaticOptimizer( java.util.LinkedHashMap[String, java.lang.Boolean] ]) extends ScopedExprTransform { + + private var stdProvider: () => Val.Obj = () => std + + private[sjsonnet] def this( + ev: EvalScope, + variableResolver: String => Option[Expr], + stdProvider: () => Val.Obj, + internedStrings: mutable.HashMap[String, String], + internedStaticFieldSets: mutable.HashMap[ + Val.StaticObjectFieldSet, + java.util.LinkedHashMap[String, java.lang.Boolean] + ]) = { + this(ev, variableResolver, null.asInstanceOf[Val.Obj], internedStrings, internedStaticFieldSets) + this.stdProvider = stdProvider + } + + private def std0: Val.Obj = stdProvider() + def optimize(e: Expr): Expr = transform(e) override def transform(_e: Expr): Expr = { @@ -76,8 +94,8 @@ class StaticOptimizer( scope.get(name) match { case ScopedVal(v: Val, _, _) => v case ScopedVal(_, _, idx) => ValidId(pos, name, idx) - case null if name == f"$$std" => std - case null if name == "std" => std + case null if name == f"$$std" => std0 + case null if name == "std" => std0 case null => variableResolver(name) match { case Some(v) => v // additional variable resolution diff --git a/sjsonnet/test/src-jvm/sjsonnet/MainTests.scala b/sjsonnet/test/src-jvm/sjsonnet/MainTests.scala index c66eb0e2..e65c00d2 100644 --- a/sjsonnet/test/src-jvm/sjsonnet/MainTests.scala +++ b/sjsonnet/test/src-jvm/sjsonnet/MainTests.scala @@ -54,6 +54,40 @@ object MainTests extends TestSuite { assert(err1.nonEmpty, err3.nonEmpty) } + test("stdlibProviderIsLazy") { + var calls = 0 + def run(source: String): (Int, String, Int) = { + val out = new ByteArrayOutputStream() + val pout = new PrintStream(out, true, "UTF-8") + val err = new ByteArrayOutputStream() + val perr = new PrintStream(err, true, "UTF-8") + val res = SjsonnetMainBase.main0WithStdProvider( + Array(source, "--exec"), + new DefaultParseCache, + System.in, + pout, + perr, + workspaceRoot, + None, + None, + () => { + calls += 1 + sjsonnet.stdlib.StdLibModule.Default.module + }, + None, + null + ) + assert(err.toString("UTF-8").isEmpty) + (res, out.toString("UTF-8"), calls) + } + + val noStd = run("1 + 1") + assert(noStd == ((0, s"2$eol", 0))) + + val withStd = run("std.length([1, 2, 3])") + assert(withStd == ((0, s"3$eol", 1))) + } + val streamedOut = """--- 1 |--- 2 diff --git a/sjsonnet/test/src/sjsonnet/EvaluatorTests.scala b/sjsonnet/test/src/sjsonnet/EvaluatorTests.scala index 57d61390..f2ef9181 100644 --- a/sjsonnet/test/src/sjsonnet/EvaluatorTests.scala +++ b/sjsonnet/test/src/sjsonnet/EvaluatorTests.scala @@ -3,6 +3,8 @@ package sjsonnet import utest._ import TestUtils.{eval, evalErr} +import scala.collection.mutable + object EvaluatorTests extends TestSuite { def tests: Tests = Tests { test("arithmetic") { @@ -11,6 +13,38 @@ object EvaluatorTests extends TestSuite { eval("-1 + 2 * 3") ==> ujson.Num(5) eval("6 - 3 + 2") ==> ujson.Num(5) } + test("strictStdPreservesLegacyCreateOptimizerOverride") { + var optimizerCalls = 0 + var stdCalls = 0 + val interpreter = new Interpreter( + Map.empty[String, String], + Map.empty[String, String], + DummyPath(), + Importer.empty, + new DefaultParseCache, + std = { + stdCalls += 1 + sjsonnet.stdlib.StdLibModule.Default.module + }, + variableResolver = _ => None + ) { + override protected def createOptimizer( + ev: EvalScope, + std: Val.Obj, + internedStrings: mutable.HashMap[String, String], + internedStaticFieldSets: mutable.HashMap[ + Val.StaticObjectFieldSet, + java.util.LinkedHashMap[String, java.lang.Boolean] + ]): StaticOptimizer = { + optimizerCalls += 1 + super.createOptimizer(ev, std, internedStrings, internedStaticFieldSets) + } + } + + interpreter.interpret("std.length([1])", DummyPath("(memory)")) ==> Right(ujson.Num(1)) + assert(optimizerCalls == 1) + assert(stdCalls == 1) + } test("objects") { eval("{x: 1}.x") ==> ujson.Num(1) eval("std.objectKeysValues({a: error 'unused'})[0].key") ==> ujson.Str("a") diff --git a/sjsonnet/test/src/sjsonnet/RendererTests.scala b/sjsonnet/test/src/sjsonnet/RendererTests.scala index 65577f25..f21372b7 100644 --- a/sjsonnet/test/src/sjsonnet/RendererTests.scala +++ b/sjsonnet/test/src/sjsonnet/RendererTests.scala @@ -1,5 +1,6 @@ package sjsonnet +import java.io.ByteArrayOutputStream import utest._ object RendererTests extends TestSuite { @@ -65,6 +66,13 @@ object RendererTests extends TestSuite { ujson.transform(ujson.Num(1e15), new Renderer()).toString ==> "1000000000000000" } + test("byteRendererLongEscapedString") { + val s = ("abc\n\"\\\t\u0001" * 40) + "tail" + val out = new ByteArrayOutputStream + ujson.transform(ujson.Str(s), new ByteRenderer(out)) + out.toString("UTF-8") ==> ujson.transform(ujson.Str(s), new Renderer()).toString + } + test("indentZero") { // indent=0 should produce newlines but no spaces ujson.transform(ujson.Arr(1, 2), new Renderer(indent = 0)).toString ==>