From fd8d159919bb03333aa8bad6fe4e11b998e74dda Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 6 Aug 2015 14:14:52 -0700 Subject: [PATCH] subs in quotes --- .../com/typesafe/config/impl/Tokenizer.java | 76 ++++++++++++++----- .../typesafe/config/impl/TokenizerTest.scala | 25 ++++++ .../com/typesafe/config/impl/UtilTest.scala | 7 +- 3 files changed, 89 insertions(+), 19 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/Tokenizer.java b/config/src/main/java/com/typesafe/config/impl/Tokenizer.java index e61c9943..f00eeb1e 100644 --- a/config/src/main/java/com/typesafe/config/impl/Tokenizer.java +++ b/config/src/main/java/com/typesafe/config/impl/Tokenizer.java @@ -38,7 +38,7 @@ final class Tokenizer { return "tab"; else if (codepoint == -1) return "end of file"; - else if (ConfigImplUtil.isC0Control(codepoint)) + else if (Character.isISOControl(codepoint)) return String.format("control character 0x%x", codepoint); else return String.format("%c", codepoint); @@ -172,6 +172,12 @@ final class Tokenizer { buffer.push(c); } + private int peekNextCharRaw() { + int c = nextCharRaw(); + putBack(c); + return c; + } + static boolean isWhitespace(int c) { return ConfigImplUtil.isWhitespace(c); } @@ -418,6 +424,9 @@ final class Tokenizer { case 't': sb.append('\t'); break; + case '$': + sb.append("\\$"); + break; case 'u': { // kind of absurdly slow, but screw it for now char[] a = new char[4]; @@ -477,7 +486,9 @@ final class Tokenizer { } } - private Token pullQuotedString() throws ProblemException { + private List pullQuotedString() throws ProblemException { + List tokens = new ArrayList(); + // the open quote has already been consumed StringBuilder sb = new StringBuilder(); @@ -488,6 +499,23 @@ final class Tokenizer { StringBuilder sbOrig = new StringBuilder(); sbOrig.appendCodePoint('"'); + // First, check for triple quotes + if (peekNextCharRaw() == '"') { // Double quotes + int second = nextCharRaw(); + if (peekNextCharRaw() == '"') { // Triple quotes! Append and return token + int third = nextCharRaw(); + sbOrig.appendCodePoint(second); + sbOrig.appendCodePoint(third); + appendTripleQuotedString(sb, sbOrig); + + tokens.add(Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString())); + return tokens; + } else { // Empty string, handled by normal string termination case below + putBack(second); + } + } + + // Single quoted string with possible substitutions while (true) { int c = nextCharRaw(); if (c == -1) @@ -497,8 +525,19 @@ final class Tokenizer { pullEscapeSequence(sb, sbOrig); } else if (c == '"') { sbOrig.appendCodePoint(c); + tokens.add(Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString())); break; - } else if (ConfigImplUtil.isC0Control(c)) { + } else if (c == '$' && peekNextCharRaw() == '{') { // Substition + // Tokenize what we have so far + tokens.add(Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString())); + + // Add substition + tokens.add(pullSubstitution()); + + // Reset and continue + sb = new StringBuilder(); + sbOrig = new StringBuilder(); + } else if (Character.isISOControl(c)) { throw problem(asString(c), "JSON does not allow unescaped " + asString(c) + " in quoted strings, use a backslash escape"); } else { @@ -507,18 +546,7 @@ final class Tokenizer { } } - // maybe switch to triple-quoted string, sort of hacky... - if (sb.length() == 0) { - int third = nextCharRaw(); - if (third == '"') { - sbOrig.appendCodePoint(third); - appendTripleQuotedString(sb, sbOrig); - } else { - putBack(third); - } - - } - return Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString()); + return tokens; } private Token pullPlusEquals() throws ProblemException { @@ -575,7 +603,17 @@ final class Tokenizer { return Tokens.newSubstitution(origin, optional, expression); } + // Occasionally pullNextToken will encounter a situation where it needs to + // parse multiple tokens. When that happens it will populate this queue and pop + // from it until empty before attempting to parse a new token. + // Substitutions within quoted strings are an example of this. + private static Queue nextTokensQueue = new LinkedList(); + private Token pullNextToken(WhitespaceSaver saver) throws ProblemException { + if (!nextTokensQueue.isEmpty()) { + return nextTokensQueue.remove(); + } + int c = nextCharAfterWhitespace(saver); if (c == -1) { return Tokens.END; @@ -592,7 +630,11 @@ final class Tokenizer { } else { switch (c) { case '"': - t = pullQuotedString(); + List all = pullQuotedString(); + t = all.remove(0); + for (Token n: all) { + nextTokensQueue.add(n); + } break; case '$': t = pullSubstitution(); @@ -692,4 +734,4 @@ final class Tokenizer { "Does not make sense to remove items from token stream"); } } -} \ No newline at end of file +} diff --git a/config/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala b/config/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala index 7ce2b3b5..54e72513 100644 --- a/config/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala @@ -152,6 +152,31 @@ class TokenizerTest extends TestUtils { tokenizerTest(expected, source) } + @Test + def tokenizeSubstitutionsInQuoted() { + val source = "\"foo${bar}baz\"\n" + val expected = List(tokenString("foo"), tokenSubstitution(tokenUnquoted("bar")), + tokenString("baz"), + tokenLine(1)) + tokenizerTest(expected, source) + } + + @Test + def tokenizeSubstitutionsInQuotedAtBeg() { + val source = "\"${bar}baz\"\n" + val expected = List(tokenString(""), tokenSubstitution(tokenUnquoted("bar")), + tokenString("baz"), + tokenLine(1)) + tokenizerTest(expected, source) + } + + @Test + def tokenizeSubstitutionsInQuotedAtEnd() { + val source = "\"foo${bar}\"" + val expected = List(tokenString("foo"), tokenSubstitution(tokenUnquoted("bar")), tokenString("")) + tokenizerTest(expected, source) + } + @Test def tokenizerUnescapeStrings(): Unit = { case class UnescapeTest(escaped: String, result: ConfigString) diff --git a/config/src/test/scala/com/typesafe/config/impl/UtilTest.scala b/config/src/test/scala/com/typesafe/config/impl/UtilTest.scala index cf31a716..3067865f 100644 --- a/config/src/test/scala/com/typesafe/config/impl/UtilTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/UtilTest.scala @@ -57,8 +57,6 @@ class UtilTest extends TestUtils { assertTrue(ConfigImplUtil.equalsHandlingNull("", "")) } - val lotsOfStrings = (invalidJson ++ validConf).map(_.test) - private def roundtripJson(s: String) { val rendered = ConfigImplUtil.renderJsonString(s) val parsed = parseConfig("{ foo: " + rendered + "}").getString("foo") @@ -77,6 +75,11 @@ class UtilTest extends TestUtils { s == parsed) } + // These strings are used in many different ways, but for testing how things + // render we don't want to have any substitutions because this render code + // does not resolve the configs. + val lotsOfStrings = (invalidJson ++ validConf).map(_.test).filter(_.indexOf("${") == -1) + @Test def renderJsonString() { for (s <- lotsOfStrings) {