Implement ${} substitutions in the tokenizer

SPEC.md

@@ -191,6 +191,12 @@ simplified HOCON value when merged:
 Substitutions are a way of referring to other parts of the configuration
 tree.
 
+The syntax is `${stringvalue}` where the `stringvalue` may be an unquoted or a
+quoted string, following the usual rules. `stringvalue` may not be a value
+concatenation, only a single string, so for example whitespace requires quoting.
+`stringvalue` may not be a non-string value such as `true`, you would have to quote
+it as `"true"`.
+
 Substitution processing is performed as the last parsing step, so a
 substitution can look forward in the configuration file and even retrieve a
 value from a Java System property. This also means that a substitution will
@@ -210,6 +216,8 @@ to be concatenated) then it is an error if the substituted value is an
 object or array. Otherwise the value is converted to a string value as follows:
 
  - `null` is converted to an empty string, not the string `null`
+   (note that this differs from a literal null value in a value
+   concatenation, which becomes the string "null")
  - strings are already strings
  - numbers are converted to a string that would parse as a valid number in
    HOCON

src/main/java/com/typesafe/config/impl/TokenType.java

@@ -1,5 +1,5 @@
 package com.typesafe.config.impl;
 
 enum TokenType {
-    START, END, COMMA, COLON, OPEN_CURLY, CLOSE_CURLY, OPEN_SQUARE, CLOSE_SQUARE, VALUE, NEWLINE, UNQUOTED_TEXT;
+    START, END, COMMA, COLON, OPEN_CURLY, CLOSE_CURLY, OPEN_SQUARE, CLOSE_SQUARE, VALUE, NEWLINE, UNQUOTED_TEXT, SUBSTITUTION;
 }

src/main/java/com/typesafe/config/impl/Tokenizer.java

@@ -262,6 +262,48 @@ final class Tokenizer {
             return Tokens.newString(lineOrigin(), sb.toString());
         }
 
+        private Token pullSubstitution() {
+            // the initial '$' has already been consumed
+            ConfigOrigin origin = lineOrigin();
+            int c = nextChar();
+            if (c != '{') {
+                throw parseError("'$' not followed by {");
+            }
+
+            String reference = null;
+            boolean wasQuoted = false;
+
+            do {
+                c = nextChar();
+                if (c == -1)
+                    throw parseError("End of input but substitution was still open");
+
+                if (c == '"') {
+                    if (reference != null)
+                        throw parseError("Substitution contains multiple string values");
+                    Token t = pullQuotedString();
+                    AbstractConfigValue v = Tokens.getValue(t);
+                    reference = ((ConfigString) v).unwrapped();
+                    wasQuoted = true;
+                } else if (c == '}') {
+                    // end the loop, done!
+                } else {
+                    if (reference != null || notInUnquotedText.indexOf(c) >= 0
+                            || isWhitespace(c))
+                        throw parseError("Substitution contains multiple string values or invalid char: '"
+                                + ((char) c) + "'");
+                    putBack(c);
+                    Token t = pullUnquotedText();
+                    if (!Tokens.isUnquotedText(t)) {
+                        throw parseError("Substitution contains non-string token, try quoting it: "
+                                + t);
+                    }
+                    reference = Tokens.getUnquotedText(t);
+                }
+            } while (c != '}');
+            return Tokens.newSubstitution(origin, reference, wasQuoted);
+        }
+
         // called if the next token is not a simple value;
         // discards any whitespace we were saving between
         // simple values.
@@ -307,6 +349,10 @@ final class Tokenizer {
                     t = pullQuotedString();
                     tIsSimpleValue = true;
                     break;
+                case '$':
+                    t = pullSubstitution();
+                    tIsSimpleValue = true;
+                    break;
                 case ':':
                     t = Tokens.COLON;
                     break;

src/main/java/com/typesafe/config/impl/Tokens.java

@@ -121,6 +121,59 @@ final class Tokens {
         }
     }
 
+    // This is not a Value, because it requires special processing
+    static private class Substitution extends Token {
+        private ConfigOrigin origin;
+        private String value;
+        private boolean isPath;
+
+        Substitution(ConfigOrigin origin, String s, boolean wasQuoted) {
+            super(TokenType.SUBSTITUTION);
+            this.origin = origin;
+            this.value = s;
+            // if the string is not quoted and contains '.' then
+            // it's a path rather than just a key name.
+
+            this.isPath = (!wasQuoted) && s.indexOf('.') >= 0;
+        }
+
+        ConfigOrigin origin() {
+            return origin;
+        }
+
+        String value() {
+            return value;
+        }
+
+        boolean isPath() {
+            return isPath;
+        }
+
+        @Override
+        public String toString() {
+            return tokenType().name() + "(" + value + ",isPath=" + isPath
+                    + ")";
+        }
+
+        @Override
+        protected boolean canEqual(Object other) {
+            return other instanceof Substitution;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            return super.equals(other)
+                    && ((Substitution) other).value.equals(value)
+                    && ((Substitution) other).isPath() == this.isPath();
+        }
+
+        @Override
+        public int hashCode() {
+            return 41 * (41 * (41 + super.hashCode()) + value.hashCode())
+                    + new Boolean(isPath()).hashCode();
+        }
+    }
+
     static boolean isValue(Token token) {
         return token instanceof Value;
     }
@@ -173,16 +226,36 @@ final class Tokens {
         }
     }
 
-    /*
+    static boolean isSubstitution(Token token) {
-     * static ConfigString newStringValueFromTokens(Token... tokens) {
+        return token instanceof Substitution;
-     * StringBuilder sb = new StringBuilder(); for (Token t : tokens) { if
+    }
-     * (isValue(t)) { ConfigValue v = getValue(t); if (v instanceof
+
-     * ConfigString) { sb.append(((ConfigString) v).unwrapped()); } else { //
+    static String getSubstitution(Token token) {
-     * FIXME convert non-strings to string throw new
+        if (token instanceof Substitution) {
-     * ConfigException.BugOrBroken( "not handling non-strings here"); } } else
+            return ((Substitution) token).value();
-     * if (isUnquotedText(t)) { String s = getUnquotedText(t); sb.append(s); }
+        } else {
-     * else { throw new ConfigException. } } }
+            throw new ConfigException.BugOrBroken(
-     */
+                    "tried to get substitution from " + token);
+        }
+    }
+
+    static ConfigOrigin getSubstitutionOrigin(Token token) {
+        if (token instanceof Substitution) {
+            return ((Substitution) token).origin();
+        } else {
+            throw new ConfigException.BugOrBroken(
+                    "tried to get substitution origin from " + token);
+        }
+    }
+
+    static boolean getSubstitutionIsPath(Token token) {
+        if (token instanceof Substitution) {
+            return ((Substitution) token).isPath();
+        } else {
+            throw new ConfigException.BugOrBroken(
+                    "tried to get substitution is path from " + token);
+        }
+    }
 
     static Token START = new Token(TokenType.START);
     static Token END = new Token(TokenType.END);
@@ -201,6 +274,11 @@ final class Tokens {
         return new UnquotedText(origin, s);
     }
 
+    static Token newSubstitution(ConfigOrigin origin, String s,
+            boolean wasQuoted) {
+        return new Substitution(origin, s, wasQuoted);
+    }
+
     static Token newValue(AbstractConfigValue value) {
         return new Value(value);
     }

src/test/scala/com/typesafe/config/impl/TokenizerTest.scala

@@ -18,6 +18,8 @@ class TokenizerTest extends TestUtils {
     def tokenFalse = Tokens.newBoolean(fakeOrigin(), false)
     def tokenNull = Tokens.newNull(fakeOrigin())
     def tokenUnquoted(s: String) = Tokens.newUnquotedText(fakeOrigin(), s)
+    def tokenKeySubstitution(s: String) = Tokens.newSubstitution(fakeOrigin(), s, true /* wasQuoted */ )
+    def tokenPathSubstitution(s: String) = Tokens.newSubstitution(fakeOrigin(), s, false /* wasQuoted */ )
     def tokenString(s: String) = Tokens.newString(fakeOrigin(), s)
     def tokenDouble(d: Double) = Tokens.newDouble(fakeOrigin(), d)
     def tokenInt(i: Int) = Tokens.newInt(fakeOrigin(), i)
@@ -57,8 +59,9 @@ class TokenizerTest extends TestUtils {
         val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
             Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"),
             tokenTrue, tokenDouble(3.14), tokenFalse,
-            tokenLong(42), tokenNull, Tokens.newLine(0), Tokens.END)
+            tokenLong(42), tokenNull, tokenPathSubstitution("a.b"),
-        assertEquals(expected, tokenizeAsList(""",:}{]["foo"true3.14false42null""" + "\n"))
+            tokenKeySubstitution("c.d"), Tokens.newLine(0), Tokens.END)
+        assertEquals(expected, tokenizeAsList(""",:}{]["foo"true3.14false42null${a.b}${"c.d"}""" + "\n"))
     }
 
     @Test
@@ -67,8 +70,9 @@ class TokenizerTest extends TestUtils {
             Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"),
             tokenUnquoted(" "), tokenLong(42), tokenUnquoted(" "), tokenTrue, tokenUnquoted(" "),
             tokenDouble(3.14), tokenUnquoted(" "), tokenFalse, tokenUnquoted(" "), tokenNull,
+            tokenUnquoted(" "), tokenPathSubstitution("a.b"), tokenUnquoted(" "), tokenKeySubstitution("c.d"),
             Tokens.newLine(0), Tokens.END)
-        assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n "))
+        assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null ${a.b} ${"c.d"} """ + "\n "))
     }
 
     @Test
@@ -77,8 +81,10 @@ class TokenizerTest extends TestUtils {
             Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"),
             tokenUnquoted("   "), tokenLong(42), tokenUnquoted("   "), tokenTrue, tokenUnquoted("   "),
             tokenDouble(3.14), tokenUnquoted("   "), tokenFalse, tokenUnquoted("   "), tokenNull,
+            tokenUnquoted("   "), tokenPathSubstitution("a.b"), tokenUnquoted("   "),
+            tokenKeySubstitution("c.d"),
             Tokens.newLine(0), Tokens.END)
-        assertEquals(expected, tokenizeAsList("""   ,   :   }   {   ]   [   "foo"   42   true   3.14   false   null   """ + "\n   "))
+        assertEquals(expected, tokenizeAsList("""   ,   :   }   {   ]   [   "foo"   42   true   3.14   false   null   ${a.b}   ${"c.d"}  """ + "\n   "))
     }
 
     @Test