Merge pull request #3 from twitter-forks/substitutions-in-quotes

Substitutions in quotes

config/src/main/java/com/typesafe/config/impl/Tokenizer.java

@@ -172,6 +172,12 @@ final class Tokenizer {
             buffer.push(c);
         }
 
+        private int peekNextCharRaw() {
+            int c = nextCharRaw();
+            putBack(c);
+            return c;
+        }
+
         static boolean isWhitespace(int c) {
             return ConfigImplUtil.isWhitespace(c);
         }
@@ -477,7 +483,9 @@ final class Tokenizer {
             }
         }
 
-        private Token pullQuotedString() throws ProblemException {
+        private List<Token> pullQuotedString() throws ProblemException {
+            List<Token> tokens = new ArrayList<Token>();
+
             // the open quote has already been consumed
             StringBuilder sb = new StringBuilder();
 
@@ -488,6 +496,23 @@ final class Tokenizer {
             StringBuilder sbOrig = new StringBuilder();
             sbOrig.appendCodePoint('"');
 
+            // First, check for triple quotes
+            if (peekNextCharRaw() == '"') { // Double quotes
+                int second = nextCharRaw();
+                if (peekNextCharRaw() == '"') { // Triple quotes! Append and return token
+                    int third = nextCharRaw();
+                    sbOrig.appendCodePoint(second);
+                    sbOrig.appendCodePoint(third);
+                    appendTripleQuotedString(sb, sbOrig);
+
+                    tokens.add(Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString()));
+                    return tokens;
+                } else { // Empty string, handled by normal string termination case below
+                    putBack(second);
+                }
+            }
+
+            // Single quoted string with possible substitutions
             while (true) {
                 int c = nextCharRaw();
                 if (c == -1)
@@ -497,7 +522,18 @@ final class Tokenizer {
                     pullEscapeSequence(sb, sbOrig);
                 } else if (c == '"') {
                     sbOrig.appendCodePoint(c);
+                    tokens.add(Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString()));
                     break;
+                } else if (c == '$' && peekNextCharRaw() == '{') { // Substition
+                    // Tokenize what we have so far
+                    tokens.add(Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString()));
+
+                    // Add substition
+                    tokens.add(pullSubstitution());
+
+                    // Reset and continue
+                    sb = new StringBuilder();
+                    sbOrig = new StringBuilder();
                 } else if (ConfigImplUtil.isC0Control(c)) {
                     throw problem(asString(c), "JSON does not allow unescaped " + asString(c)
                             + " in quoted strings, use a backslash escape");
@@ -507,18 +543,7 @@ final class Tokenizer {
                 }
             }
 
-            // maybe switch to triple-quoted string, sort of hacky...
-            if (sb.length() == 0) {
-                int third = nextCharRaw();
-                if (third == '"') {
-                    sbOrig.appendCodePoint(third);
-                    appendTripleQuotedString(sb, sbOrig);
-                } else {
-                    putBack(third);
-                }
-
-            }
-            return Tokens.newString(lineOrigin, sb.toString(), sbOrig.toString());
+            return tokens;
         }
 
         private Token pullPlusEquals() throws ProblemException {
@@ -575,7 +600,17 @@ final class Tokenizer {
             return Tokens.newSubstitution(origin, optional, expression);
         }
 
+        // Occasionally pullNextToken will encounter a situation where it needs to
+        // parse multiple tokens. When that happens it will populate this queue and pop
+        // from it until empty before attempting to parse a new token.
+        // Substitutions within quoted strings are an example of this.
+        private static Queue<Token> nextTokensQueue = new LinkedList<Token>();
+
         private Token pullNextToken(WhitespaceSaver saver) throws ProblemException {
+            if (!nextTokensQueue.isEmpty()) {
+                return nextTokensQueue.remove();
+            }
+
             int c = nextCharAfterWhitespace(saver);
             if (c == -1) {
                 return Tokens.END;
@@ -592,7 +627,11 @@ final class Tokenizer {
                 } else {
                     switch (c) {
                     case '"':
-                        t = pullQuotedString();
+                        List<Token> all = pullQuotedString();
+                        t = all.remove(0);
+                        for (Token n: all) {
+                            nextTokensQueue.add(n);
+                        }
                         break;
                     case '$':
                         t = pullSubstitution();
@@ -692,4 +731,4 @@ final class Tokenizer {
                     "Does not make sense to remove items from token stream");
         }
     }
-}
+}

config/src/test/scala/com/typesafe/config/impl/ConfigSubstitutionTest.scala

@@ -40,6 +40,7 @@ class ConfigSubstitutionTest extends TestUtils {
        "bool" : true,
        "null" : null,
        "string" : "hello",
+       "stringwsub": "hello ${foo} bar",
        "double" : 3.14
     }
 }
@@ -88,6 +89,13 @@ class ConfigSubstitutionTest extends TestUtils {
         assertEquals(stringValue("hello"), v)
     }
 
+    @Test
+    def resolveStringWSub() {
+        val s = subst("bar.stringwsub")
+        val v = resolveWithoutFallbacks(s, simpleObject)
+        assertEquals(stringValue("hello 42 bar"), v)
+    }
+
     @Test
     def resolveDouble() {
         val s = subst("bar.double")

config/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala

@@ -152,6 +152,31 @@ class TokenizerTest extends TestUtils {
         tokenizerTest(expected, source)
     }
 
+    @Test
+    def tokenizeSubstitutionsInQuoted() {
+        val source = "\"foo${bar}baz\"\n"
+        val expected = List(tokenString("foo"), tokenSubstitution(tokenUnquoted("bar")),
+            tokenString("baz"),
+            tokenLine(1))
+        tokenizerTest(expected, source)
+    }
+
+    @Test
+    def tokenizeSubstitutionsInQuotedAtBeg() {
+        val source = "\"${bar}baz\"\n"
+        val expected = List(tokenString(""), tokenSubstitution(tokenUnquoted("bar")),
+            tokenString("baz"),
+            tokenLine(1))
+        tokenizerTest(expected, source)
+    }
+
+    @Test
+    def tokenizeSubstitutionsInQuotedAtEnd() {
+        val source = "\"foo${bar}\""
+        val expected = List(tokenString("foo"), tokenSubstitution(tokenUnquoted("bar")), tokenString(""))
+        tokenizerTest(expected, source)
+    }
+
     @Test
     def tokenizerUnescapeStrings(): Unit = {
         case class UnescapeTest(escaped: String, result: ConfigString)

config/src/test/scala/com/typesafe/config/impl/UtilTest.scala

@@ -57,8 +57,6 @@ class UtilTest extends TestUtils {
         assertTrue(ConfigImplUtil.equalsHandlingNull("", ""))
     }
 
-    val lotsOfStrings = (invalidJson ++ validConf).map(_.test)
-
     private def roundtripJson(s: String) {
         val rendered = ConfigImplUtil.renderJsonString(s)
         val parsed = parseConfig("{ foo: " + rendered + "}").getString("foo")
@@ -77,6 +75,11 @@ class UtilTest extends TestUtils {
             s == parsed)
     }
 
+    // These strings are used in many different ways, but for testing how things
+    // render we don't want to have any substitutions because this render code
+    // does not resolve the configs.
+    val lotsOfStrings = (invalidJson ++ validConf).map(_.test).filter(_.indexOf("${") == -1)
+
     @Test
     def renderJsonString() {
         for (s <- lotsOfStrings) {