improve error messages for reserved chars

Requires passing the char through the tokenizer to the parser
since the parser will have more contextual information to
include in the error.

config/src/main/java/com/typesafe/config/impl/TokenType.java

@@ -4,5 +4,18 @@
 package com.typesafe.config.impl;
 
 enum TokenType {
-    START, END, COMMA, EQUALS, COLON, OPEN_CURLY, CLOSE_CURLY, OPEN_SQUARE, CLOSE_SQUARE, VALUE, NEWLINE, UNQUOTED_TEXT, SUBSTITUTION;
+    START,
+    END,
+    COMMA,
+    EQUALS,
+    COLON,
+    OPEN_CURLY,
+    CLOSE_CURLY,
+    OPEN_SQUARE,
+    CLOSE_SQUARE,
+    VALUE,
+    NEWLINE,
+    UNQUOTED_TEXT,
+    SUBSTITUTION,
+    RESERVED_CHAR;
 }

config/src/main/java/com/typesafe/config/impl/Tokenizer.java

@@ -482,9 +482,7 @@ final class Tokenizer {
                     if (firstNumberChars.indexOf(c) >= 0) {
                         t = pullNumber(c);
                     } else if (notInUnquotedText.indexOf(c) >= 0) {
-                        throw parseError(String
-                                .format("Character '%c' is not the start of any valid token",
-                                        c));
+                        t = Tokens.newReservedChar(lineOrigin(), c);
                     } else {
                         putBack(c);
                         t = pullUnquotedText();

config/src/main/java/com/typesafe/config/impl/Tokens.java

@@ -119,6 +119,45 @@ final class Tokens {
         }
     }
 
+    static private class ReservedChar extends Token {
+        final private ConfigOrigin origin;
+        final private int value;
+
+        ReservedChar(ConfigOrigin origin, int c) {
+            super(TokenType.RESERVED_CHAR);
+            this.origin = origin;
+            this.value = c;
+        }
+
+        ConfigOrigin origin() {
+            return origin;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            sb.append('\'');
+            sb.appendCodePoint(value);
+            sb.append('\'');
+            return sb.toString();
+        }
+
+        @Override
+        protected boolean canEqual(Object other) {
+            return other instanceof ReservedChar;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            return super.equals(other) && ((ReservedChar) other).value == value;
+        }
+
+        @Override
+        public int hashCode() {
+            return 41 * (41 + super.hashCode()) + value;
+        }
+    }
+
     // This is not a Value, because it requires special processing
     static private class Substitution extends Token {
         final private ConfigOrigin origin;
@@ -200,6 +239,18 @@ final class Tokens {
         }
     }
 
+    static boolean isReservedChar(Token token) {
+        return token instanceof ReservedChar;
+    }
+
+    static ConfigOrigin getReservedCharOrigin(Token token) {
+        if (token instanceof ReservedChar) {
+            return ((ReservedChar) token).origin();
+        } else {
+            throw new ConfigException.BugOrBroken("tried to get reserved char origin from " + token);
+        }
+    }
+
     static boolean isUnquotedText(Token token) {
         return token instanceof UnquotedText;
     }
@@ -267,6 +318,10 @@ final class Tokens {
         return new Line(lineNumberJustEnded);
     }
 
+    static Token newReservedChar(ConfigOrigin origin, int codepoint) {
+        return new ReservedChar(origin, codepoint);
+    }
+
     static Token newUnquotedText(ConfigOrigin origin, String s) {
         return new UnquotedText(origin, s);
     }

config/src/test/scala/com/typesafe/config/impl/TestUtils.scala

@@ -383,6 +383,7 @@ abstract trait TestUtils {
     def tokenTrue = Tokens.newBoolean(fakeOrigin(), true)
     def tokenFalse = Tokens.newBoolean(fakeOrigin(), false)
     def tokenNull = Tokens.newNull(fakeOrigin())
+    def tokenReserved(c: Int) = Tokens.newReservedChar(fakeOrigin(), c)
     def tokenUnquoted(s: String) = Tokens.newUnquotedText(fakeOrigin(), s)
     def tokenString(s: String) = Tokens.newString(fakeOrigin(), s)
     def tokenDouble(d: Double) = Tokens.newDouble(fakeOrigin(), d, null)

config/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala

@@ -214,4 +214,13 @@ class TokenizerTest extends TestUtils {
         tokenizerTest(List(tokenDouble(3.14)), "3.14//comment")
         tokenizerTest(List(tokenDouble(3.14)), "3.14#comment")
     }
+
+    @Test
+    def tokenizeReservedChars() {
+        val tokenized = tokenizeAsList("+`^?!@*&\\")
+        assertEquals(Seq(Tokens.START, tokenReserved('+'), tokenReserved('`'),
+            tokenReserved('^'), tokenReserved('?'), tokenReserved('!'), tokenReserved('@'),
+            tokenReserved('*'), tokenReserved('&'), tokenReserved('\\'),
+            Tokens.END), tokenized)
+    }
 }