From 75a1b1228496dcf895e688eef65c06075f5bd75c Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Sat, 12 Nov 2011 16:36:07 -0500 Subject: [PATCH] Allow omitting commas as long as there's a newline --- .../java/com/typesafe/config/impl/Parser.java | 82 ++++++++++++++----- src/test/resources/equiv01/no-commas.conf | 55 +++++++++++++ .../typesafe/config/impl/ConfParserTest.scala | 82 +++++++++++++++++++ .../config/impl/EquivalentsTest.scala | 2 +- .../com/typesafe/config/impl/TestUtils.scala | 3 + 5 files changed, 202 insertions(+), 22 deletions(-) create mode 100644 src/test/resources/equiv01/no-commas.conf diff --git a/src/main/java/com/typesafe/config/impl/Parser.java b/src/main/java/com/typesafe/config/impl/Parser.java index 0ea4bd12..9f11c78e 100644 --- a/src/main/java/com/typesafe/config/impl/Parser.java +++ b/src/main/java/com/typesafe/config/impl/Parser.java @@ -156,6 +156,42 @@ final class Parser { return t; } + // In arrays and objects, comma can be omitted + // as long as there's at least one newline instead. + // this skips any newlines in front of a comma, + // skips the comma, and returns true if it found + // either a newline or a comma. The iterator + // is left just after the comma or the newline. + private boolean checkElementSeparator() { + if (flavor == SyntaxFlavor.JSON) { + Token t = nextTokenIgnoringNewline(); + if (t == Tokens.COMMA) { + return true; + } else { + putBack(t); + return false; + } + } else { + boolean sawSeparatorOrNewline = false; + Token t = nextToken(); + while (true) { + if (Tokens.isNewline(t)) { + lineNumber = Tokens.getLineNumber(t); + sawSeparatorOrNewline = true; + // we want to continue to also eat + // a comma if there is one. + } else if (t == Tokens.COMMA) { + return true; + } else { + // non-newline-or-comma + putBack(t); + return sawSeparatorOrNewline; + } + t = nextToken(); + } + } + } + // merge a bunch of adjacent values into one // value; change unquoted text into a string // value. @@ -461,25 +497,27 @@ final class Parser { afterComma = false; } - t = nextTokenIgnoringNewline(); - if (t == Tokens.CLOSE_CURLY) { - if (!hadOpenCurly) { - throw parseError("unbalanced close brace '}' with no open brace"); - } - break; - } else if (t == Tokens.COMMA) { + if (checkElementSeparator()) { // continue looping afterComma = true; - } else if (hadOpenCurly) { - throw parseError("Expecting close brace } or a comma, got " - + t); } else { - if (t == Tokens.END) { - putBack(t); + t = nextTokenIgnoringNewline(); + if (t == Tokens.CLOSE_CURLY) { + if (!hadOpenCurly) { + throw parseError("unbalanced close brace '}' with no open brace"); + } break; - } else { - throw parseError("Expecting end of input or a comma, got " + } else if (hadOpenCurly) { + throw parseError("Expecting close brace } or a comma, got " + t); + } else { + if (t == Tokens.END) { + putBack(t); + break; + } else { + throw parseError("Expecting end of input or a comma, got " + + t); + } } } } @@ -514,14 +552,16 @@ final class Parser { // now remaining elements while (true) { // just after a value - t = nextTokenIgnoringNewline(); - if (t == Tokens.CLOSE_SQUARE) { - return new SimpleConfigList(arrayOrigin, values); - } else if (t == Tokens.COMMA) { - // OK + if (checkElementSeparator()) { + // comma (or newline equivalent) consumed } else { - throw parseError("List should have ended with ] or had a comma, instead had token: " - + t); + t = nextTokenIgnoringNewline(); + if (t == Tokens.CLOSE_SQUARE) { + return new SimpleConfigList(arrayOrigin, values); + } else { + throw parseError("List should have ended with ] or had a comma, instead had token: " + + t); + } } // now just after a comma diff --git a/src/test/resources/equiv01/no-commas.conf b/src/test/resources/equiv01/no-commas.conf new file mode 100644 index 00000000..2dc6872e --- /dev/null +++ b/src/test/resources/equiv01/no-commas.conf @@ -0,0 +1,55 @@ +{ + "ints" : { + "fortyTwo" : 42 + "fortyTwoAgain" : 42 + } + + "floats" : { + "fortyTwoPointOne" : 42.1 + "fortyTwoPointOneAgain" : 42.1 + } + + "strings" : { + "abcd" : "abcd" + "abcdAgain" : "abcd" + "a" : "a" + "b" : "b" + "c" : "c" + "d" : "d" + "concatenated" : "null bar 42 baz true 3.14 hi" + } + + "arrays" : { + "empty" : [] + "1" : [ 1 ] + "12" : [1 + 2] + "123" : [1 + 2 + 3 ] + "ofString" : [ + +"a" + "b" + "c" + +] + } + + "booleans" : { + + "true" : true + + "trueAgain" : true + + "false" : false + + "falseAgain" : false + + } + + "nulls" : { + "null" : null + "nullAgain" : null + } +} diff --git a/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala b/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala index 6aa5c957..53283842 100644 --- a/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala +++ b/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala @@ -160,4 +160,86 @@ class ConfParserTest extends TestUtils { assertEquals(2, obj.getInt("a.b.c.y")) assertEquals(100, obj.getInt("a.b.c.z")) } + + @Test + def impliedCommaHandling() { + val valids = Seq( + """ +// one line +{ + a : y, b : z, c : [ 1, 2, 3 ] +}""", """ +// multiline but with all commas +{ + a : y, + b : z, + c : [ + 1, + 2, + 3, + ], +} +""", """ +// multiline with no commas +{ + a : y + b : z + c : [ + 1 + 2 + 3 + ] +} +""") + + def dropCurlies(s: String) = { + // drop the outside curly braces + val first = s.indexOf('{') + val last = s.lastIndexOf('}') + s.substring(0, first) + s.substring(first + 1, last) + s.substring(last + 1, s.length()) + } + + val changes = Seq( + { s: String => s }, + { s: String => s.replace("\n", "\n\n") }, + { s: String => s.replace("\n", "\n\n\n") }, + { s: String => s.replace(",\n", "\n,\n") }, + { s: String => s.replace(",\n", "\n\n,\n\n") }, + { s: String => s.replace("\n", " \n ") }, + { s: String => s.replace(",\n", " \n \n , \n \n ") }, + { s: String => dropCurlies(s) }) + + var tested = 0; + for (v <- valids; change <- changes) { + tested += 1; + val obj = parseObject(change(v)) + assertEquals(3, obj.size()) + assertEquals("y", obj.getString("a")) + assertEquals("z", obj.getString("b")) + assertEquals(Seq(1, 2, 3), obj.getIntList("c").asScala) + } + + assertEquals(valids.length * changes.length, tested) + + // with no newline or comma, we do value concatenation + val noNewlineInArray = parseObject(" { c : [ 1 2 3 ] } ") + assertEquals(Seq("1 2 3"), noNewlineInArray.getStringList("c").asScala) + + val noNewlineInArrayWithQuoted = parseObject(""" { c : [ "4" "5" "6" ] } """) + assertEquals(Seq("4 5 6"), noNewlineInArrayWithQuoted.getStringList("c").asScala) + + val noNewlineInObject = parseObject(" { a : b c } ") + assertEquals("b c", noNewlineInObject.getString("a")) + + val noNewlineAtEnd = parseObject("a : b") + assertEquals("b", noNewlineAtEnd.getString("a")) + + intercept[ConfigException] { + parseObject("{ a : y b : z }") + } + + intercept[ConfigException] { + parseObject("""{ "a" : "y" "b" : "z" }""") + } + } } diff --git a/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala b/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala index 5587d55c..33f017df 100644 --- a/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala +++ b/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala @@ -87,6 +87,6 @@ class EquivalentsTest extends TestUtils { // it breaks every time you add a file, so you have to update it. assertEquals(2, dirCount) // this is the number of files not named original.* - assertEquals(10, fileCount) + assertEquals(11, fileCount) } } diff --git a/src/test/scala/com/typesafe/config/impl/TestUtils.scala b/src/test/scala/com/typesafe/config/impl/TestUtils.scala index 15c5117a..1d63221a 100644 --- a/src/test/scala/com/typesafe/config/impl/TestUtils.scala +++ b/src/test/scala/com/typesafe/config/impl/TestUtils.scala @@ -212,6 +212,9 @@ abstract trait TestUtils { "{ a : b, }", // single trailing comma in object (unquoted strings) "{ a : b \n , \n }", // single trailing comma in object with newlines "a : b, c : d,", // single trailing comma in object with no root braces + "{ a : b\nc : d }", // skip comma if there's a newline + "a : b\nc : d", // skip comma if there's a newline and no root braces + "a : b\nc : d,", // skip one comma but still have one at the end "[ foo ]", // not a known token in JSON "[ t ]", // start of "true" but ends wrong in JSON "[ tx ]",