From f9fb6eca8fb9471cfd4280a054c8952051ee0ba0 Mon Sep 17 00:00:00 2001 From: He-Pin Date: Mon, 16 Mar 2026 14:01:07 +0800 Subject: [PATCH] chore: sync tests from upstream. --- sjsonnet/src-js/sjsonnet/Platform.scala | 92 +++++++++---------- sjsonnet/src-native/sjsonnet/Platform.scala | 90 +++++++++--------- .../resources/go_test_suite/parseYaml.jsonnet | 20 +++- .../go_test_suite/parseYaml.jsonnet.golden | 17 +++- .../go_test_suite/stdlib_smoke_test.jsonnet | 13 ++- .../test/src/sjsonnet/ParseYamlTests.scala | 29 ++++++ 6 files changed, 161 insertions(+), 100 deletions(-) diff --git a/sjsonnet/src-js/sjsonnet/Platform.scala b/sjsonnet/src-js/sjsonnet/Platform.scala index e18d617a..22cfb9f0 100644 --- a/sjsonnet/src-js/sjsonnet/Platform.scala +++ b/sjsonnet/src-js/sjsonnet/Platform.scala @@ -10,7 +10,7 @@ object Platform { private def nodeToJson(node: Node): ujson.Value = node match { case _: Node.ScalarNode => YamlDecoder.forAny.construct(node).getOrElse("") match { - case null => ujson.Null + case null | None => ujson.Null case v: String => ujson.read(s"\"${v.replace("\"", "\\\"").replace("\n", "\\n")}\"", false) case v: Boolean => ujson.Bool(v) case v: Int => ujson.Num(v.toDouble) @@ -43,66 +43,66 @@ object Platform { } def yamlToJson(s: String): ujson.Value = { - // Split YAML multi-document stream manually, similar to SnakeYAML's loadAll - // since parseManyYamls doesn't handle all cases correctly - val documents = splitYamlDocuments(s) - - documents.size match { - case 0 => ujson.Null - case 1 => parseSingleDocument(documents.head) - case _ => - val buf = new mutable.ArrayBuffer[ujson.Value](documents.size) - for (doc <- documents) { - buf += parseSingleDocument(doc) + if (s.trim.isEmpty) return ujson.Null + + // Preprocess to add explicit nulls for empty documents, + // since scala-yaml's parseManyYamls can't handle empty documents + // (DocumentStart immediately followed by DocumentEnd). + val preprocessed = addExplicitNullsForEmptyDocs(s) + + parseManyYamls(preprocessed) match { + case Right(documents) => + documents.size match { + case 0 => ujson.Null + case 1 => nodeToJson(documents.head) + case _ => + val buf = new mutable.ArrayBuffer[ujson.Value](documents.size) + for (doc <- documents) { + buf += nodeToJson(doc) + } + ujson.Arr(buf) } - ujson.Arr(buf) + case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage) } } - private def splitYamlDocuments(s: String): List[String] = { - if (s.trim.isEmpty) return Nil - - // Split on document separator "---" at line start - // But only if it's followed by whitespace or end of line + /** + * Inserts explicit "null" content for empty YAML documents. An empty document is one where a + * "---" marker has no content before the next "---" marker or end of input. + */ + private def addExplicitNullsForEmptyDocs(s: String): String = { val lines = s.split("\n", -1).toList - val documents = mutable.ArrayBuffer[String]() - val currentDoc = mutable.ArrayBuffer[String]() - var isFirstDoc = true + val result = new mutable.ArrayBuffer[String](lines.size + 4) + var pendingEmptySep = false for (line <- lines) { val trimmed = line.trim - // Check if this line starts with "---" and is followed by whitespace or end - if (trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)) { - // Save previous document if not empty - if (currentDoc.nonEmpty || !isFirstDoc) { - documents += currentDoc.mkString("\n") + val isSep = + trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace) + + if (isSep) { + if (pendingEmptySep) { + // Previous "---" had no content after it; insert explicit null + result += "null" } - currentDoc.clear() - isFirstDoc = false + result += line + // Check if this separator has inline content (e.g. "--- 3", "--- >") + val afterMarker = trimmed.substring(3).trim + pendingEmptySep = afterMarker.isEmpty } else { - currentDoc += line + if (pendingEmptySep && trimmed.nonEmpty) { + pendingEmptySep = false + } + result += line } } - // Add last document - if (currentDoc.nonEmpty || documents.nonEmpty) { - documents += currentDoc.mkString("\n") + // Handle trailing "---" with no content + if (pendingEmptySep) { + result += "null" } - documents.toList - } - - private def parseSingleDocument(doc: String): ujson.Value = { - val trimmed = doc.trim - if (trimmed.isEmpty) { - ujson.Null - } else { - // Use parseYaml for single document - parseYaml(trimmed) match { - case Right(node) => nodeToJson(node) - case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage) - } - } + result.mkString("\n") } def md5(s: String): String = { diff --git a/sjsonnet/src-native/sjsonnet/Platform.scala b/sjsonnet/src-native/sjsonnet/Platform.scala index 527e8f09..8ea3be50 100644 --- a/sjsonnet/src-native/sjsonnet/Platform.scala +++ b/sjsonnet/src-native/sjsonnet/Platform.scala @@ -78,66 +78,66 @@ object Platform { } def yamlToJson(s: String): ujson.Value = { - // Split YAML multi-document stream manually, similar to SnakeYAML's loadAll - // since parseManyYamls doesn't handle all cases correctly - val documents = splitYamlDocuments(s) - - documents.size match { - case 0 => ujson.Null - case 1 => parseSingleDocument(documents.head) - case _ => - val buf = new mutable.ArrayBuffer[ujson.Value](documents.size) - for (doc <- documents) { - buf += parseSingleDocument(doc) + if (s.trim.isEmpty) return ujson.Null + + // Preprocess to add explicit nulls for empty documents, + // since scala-yaml's parseManyYamls can't handle empty documents + // (DocumentStart immediately followed by DocumentEnd). + val preprocessed = addExplicitNullsForEmptyDocs(s) + + parseManyYamls(preprocessed) match { + case Right(documents) => + documents.size match { + case 0 => ujson.Null + case 1 => nodeToJson(documents.head) + case _ => + val buf = new mutable.ArrayBuffer[ujson.Value](documents.size) + for (doc <- documents) { + buf += nodeToJson(doc) + } + ujson.Arr(buf) } - ujson.Arr(buf) + case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage) } } - private def splitYamlDocuments(s: String): List[String] = { - if (s.trim.isEmpty) return Nil - - // Split on document separator "---" at line start - // But only if it's followed by whitespace or end of line + /** + * Inserts explicit "null" content for empty YAML documents. An empty document is one where a + * "---" marker has no content before the next "---" marker or end of input. + */ + private def addExplicitNullsForEmptyDocs(s: String): String = { val lines = s.split("\n", -1).toList - val documents = mutable.ArrayBuffer[String]() - val currentDoc = mutable.ArrayBuffer[String]() - var isFirstDoc = true + val result = new mutable.ArrayBuffer[String](lines.size + 4) + var pendingEmptySep = false for (line <- lines) { val trimmed = line.trim - // Check if this line starts with "---" and is followed by whitespace or end - if (trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)) { - // Save previous document if not empty - if (currentDoc.nonEmpty || !isFirstDoc) { - documents += currentDoc.mkString("\n") + val isSep = + trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace) + + if (isSep) { + if (pendingEmptySep) { + // Previous "---" had no content after it; insert explicit null + result += "null" } - currentDoc.clear() - isFirstDoc = false + result += line + // Check if this separator has inline content (e.g. "--- 3", "--- >") + val afterMarker = trimmed.substring(3).trim + pendingEmptySep = afterMarker.isEmpty } else { - currentDoc += line + if (pendingEmptySep && trimmed.nonEmpty) { + pendingEmptySep = false + } + result += line } } - // Add last document - if (currentDoc.nonEmpty || documents.nonEmpty) { - documents += currentDoc.mkString("\n") + // Handle trailing "---" with no content + if (pendingEmptySep) { + result += "null" } - documents.toList - } - - private def parseSingleDocument(doc: String): ujson.Value = { - val trimmed = doc.trim - if (trimmed.isEmpty) { - ujson.Null - } else { - // Use parseYaml for single document - parseYaml(trimmed) match { - case Right(node) => nodeToJson(node) - case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage) - } - } + result.mkString("\n") } private def computeHash(algorithm: String, s: String) = { diff --git a/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet b/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet index bc82dc18..9910f8f2 100644 --- a/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet +++ b/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet @@ -32,7 +32,7 @@ ||| --- a: 1 - --- + --- a: 2 |||, @@ -42,5 +42,23 @@ ---a: 2 a---: 3 |||, + + // Scalar documents can start on the same line as the document-start marker + ||| + a: 1 + --- > + hello + world + --- 3 + |||, + + // Documents can be empty; this is interpreted as null + ||| + a: 1 + --- + --- 2 + |||, + + "---", ] ] diff --git a/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet.golden b/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet.golden index 8c1a439b..6082f337 100644 --- a/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet.golden +++ b/sjsonnet/test/resources/go_test_suite/parseYaml.jsonnet.golden @@ -37,5 +37,20 @@ "---a": 2, "a": 1, "a---": 3 - } + }, + [ + { + "a": 1 + }, + "hello world\n", + 3 + ], + [ + { + "a": 1 + }, + null, + 2 + ], + null ] diff --git a/sjsonnet/test/resources/go_test_suite/stdlib_smoke_test.jsonnet b/sjsonnet/test/resources/go_test_suite/stdlib_smoke_test.jsonnet index 1d2b2b80..339625bb 100644 --- a/sjsonnet/test/resources/go_test_suite/stdlib_smoke_test.jsonnet +++ b/sjsonnet/test/resources/go_test_suite/stdlib_smoke_test.jsonnet @@ -4,7 +4,6 @@ // Functions without optional arguments need only one line // Functions with optional arguments need two lines - one with none of the optional arguments // and the other with all of them. - local assertClose(a, b) = // Using 1e-12 as tolerance. Jsonnet uses double-precision floats with machine epsilon of 2**-53 ≈ 1.11e-16. // This tolerance is ~9000x the machine epsilon, which is quite lenient and should work across @@ -79,13 +78,13 @@ local assertClose(a, b) = mantissa: std.mantissa(x=5), floor: std.floor(x=5), ceil: std.ceil(x=5), - sqrt: std.assertEqual(std.sqrt(x=5), 2.23606797749979), + sqrt: assertClose(std.sqrt(x=5), 2.2360679774997898), sin: assertClose(std.sin(x=5), -0.9589242746631385), - cos: assertClose(std.cos(x=5), 0.28366218546322625), - tan: assertClose(std.tan(x=5), -3.380515006246586), - asin: assertClose(std.asin(x=0.5), 0.5235987755982989), - acos: assertClose(std.acos(x=0.5), 1.0471975511965979), - atan: assertClose(std.atan(x=5), 1.373400766945016), + cos: assertClose(std.cos(x=5), 0.2836621854632263), + tan: assertClose(std.tan(x=5), -3.3805150062465854), + asin: assertClose(std.asin(x=0.5), 0.52359877559829893), + acos: assertClose(std.acos(x=0.5), 1.0471975511965976), + atan: assertClose(std.atan(x=5), 1.3734007669450157), // Assertions and debugging assertEqual: std.assertEqual(a="a", b="a"), diff --git a/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala b/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala index fccd82d8..880bc285 100644 --- a/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala +++ b/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala @@ -44,5 +44,34 @@ object ParseYamlTests extends TestSuite { // Test that trailing empty document with whitespace is handled eval("std.parseYaml('1\\n---\\n')") ==> ujson.Value("""[1,null]""") } + test { + // Scalar documents can start on the same line as the document-start marker + // "--- 3" as standalone + eval("std.parseYaml('--- 3\\n')") ==> ujson.Value("""3""") + } + test { + // Folded scalar as document + eval("std.parseYaml('--- >\\n hello\\n world\\n')") ==> ujson.Value(""""hello world\n"""") + } + test { + // Combined: scalar docs on same line as marker + eval("std.parseYaml('a: 1\\n--- >\\n hello\\n world\\n--- 3\\n')") ==> ujson.Value( + """[{"a": 1}, "hello world\n", 3]""" + ) + } + test { + // empty doc then inline scalar + eval("std.parseYaml('a: 1\\n---\\n--- 2\\n')") ==> ujson.Value( + """[{"a": 1}, null, 2]""" + ) + } + test { + // Bare document separator + eval("""std.parseYaml("---")""") ==> ujson.Value("""null""") + } + test { + // Folded scalar without document marker (directly) + eval("std.parseYaml('>\\n hello\\n world\\n')") ==> ujson.Value(""""hello world\n"""") + } } }