From 01cc2c755cb3b227b18957e838a85ddf5b10fc13 Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Mon, 7 Nov 2011 23:02:38 -0500 Subject: [PATCH] Add support for unquoted string values --- .../com/typesafe/config/ConfigException.java | 12 +- .../config/impl/AbstractConfigObject.java | 4 +- .../config/impl/AbstractConfigValue.java | 9 + .../typesafe/config/impl/ConfigBoolean.java | 5 + .../typesafe/config/impl/ConfigDouble.java | 5 + .../com/typesafe/config/impl/ConfigInt.java | 5 + .../com/typesafe/config/impl/ConfigLong.java | 5 + .../com/typesafe/config/impl/ConfigNull.java | 5 + .../typesafe/config/impl/ConfigString.java | 5 + .../config/impl/DefaultTransformer.java | 8 +- .../java/com/typesafe/config/impl/Parser.java | 159 ++++++++++++++---- .../typesafe/config/impl/SyntaxFlavor.java | 2 +- .../com/typesafe/config/impl/TokenType.java | 2 +- .../com/typesafe/config/impl/Tokenizer.java | 100 +++++------ .../java/com/typesafe/config/impl/Tokens.java | 82 ++++++++- src/test/resources/equiv01/no-whitespace.json | 2 +- src/test/resources/equiv01/original.json | 6 +- src/test/resources/equiv01/unquoted.conf | 41 +++++ .../typesafe/config/impl/ConfParserTest.scala | 51 ++++++ .../config/impl/EquivalentsTest.scala | 11 +- .../com/typesafe/config/impl/JsonTest.scala | 96 ++--------- .../com/typesafe/config/impl/TestUtils.scala | 89 ++++++++++ .../typesafe/config/impl/TokenizerTest.scala | 64 ++++++- 23 files changed, 590 insertions(+), 178 deletions(-) create mode 100644 src/test/resources/equiv01/unquoted.conf create mode 100644 src/test/scala/com/typesafe/config/impl/ConfParserTest.scala diff --git a/src/main/java/com/typesafe/config/ConfigException.java b/src/main/java/com/typesafe/config/ConfigException.java index 24dcd58b..316d0ee4 100644 --- a/src/main/java/com/typesafe/config/ConfigException.java +++ b/src/main/java/com/typesafe/config/ConfigException.java @@ -70,10 +70,18 @@ public class ConfigException extends RuntimeException { public static class Null extends WrongType { private static final long serialVersionUID = 1L; + private static String makeMessage(String path, String expected) { + if (expected != null) { + return "Configuration key '" + path + + "' is set to null but expected " + expected; + } else { + return "Configuration key '" + path + "' is null"; + } + } + public Null(ConfigOrigin origin, String path, String expected, Throwable cause) { - super(origin, "Configuration key '" + path - + "' is set to null but expected " + expected, cause); + super(origin, makeMessage(path, expected), cause); } public Null(ConfigOrigin origin, String path, String expected) { diff --git a/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java b/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java index 64235b1f..8c57a1cb 100644 --- a/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java +++ b/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java @@ -66,7 +66,7 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements if (v.valueType() == ConfigValueType.NULL) throw new ConfigException.Null(v.origin(), originalPath, - expected.name()); + expected != null ? expected.name() : null); else if (expected != null && v.valueType() != expected) throw new ConfigException.WrongType(v.origin(), originalPath, expected.name(), v.valueType().name()); @@ -361,7 +361,7 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements for (String k : keySet()) { sb.append(k); sb.append("->"); - sb.append(get(k).toString()); + sb.append(peek(k).toString()); sb.append(","); } if (!keySet().isEmpty()) diff --git a/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java b/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java index 9bf00468..8b026567 100644 --- a/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java +++ b/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java @@ -55,4 +55,13 @@ abstract class AbstractConfigValue implements ConfigValue { public String toString() { return valueType().name() + "(" + unwrapped() + ")"; } + + // toString() is a debugging-oriented string but this is defined + // to create a string that would parse back to the value in JSON. + // It only works for primitive values (that would be a single token) + // which are auto-converted to strings when concatenating with + // other strings or by the DefaultTransformer. + String transformToString() { + return null; + } } diff --git a/src/main/java/com/typesafe/config/impl/ConfigBoolean.java b/src/main/java/com/typesafe/config/impl/ConfigBoolean.java index e12645a6..acbd301a 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigBoolean.java +++ b/src/main/java/com/typesafe/config/impl/ConfigBoolean.java @@ -21,4 +21,9 @@ final class ConfigBoolean extends AbstractConfigValue { public Boolean unwrapped() { return value; } + + @Override + String transformToString() { + return value ? "true" : "false"; + } } diff --git a/src/main/java/com/typesafe/config/impl/ConfigDouble.java b/src/main/java/com/typesafe/config/impl/ConfigDouble.java index 853c2c1d..c92455de 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigDouble.java +++ b/src/main/java/com/typesafe/config/impl/ConfigDouble.java @@ -21,4 +21,9 @@ final class ConfigDouble extends AbstractConfigValue { public Double unwrapped() { return value; } + + @Override + String transformToString() { + return Double.toString(value); + } } diff --git a/src/main/java/com/typesafe/config/impl/ConfigInt.java b/src/main/java/com/typesafe/config/impl/ConfigInt.java index ca45a018..1425ff49 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigInt.java +++ b/src/main/java/com/typesafe/config/impl/ConfigInt.java @@ -44,4 +44,9 @@ final class ConfigInt extends AbstractConfigValue { // note that "origin" is deliberately NOT part of equality return value; } + + @Override + String transformToString() { + return Integer.toString(value); + } } diff --git a/src/main/java/com/typesafe/config/impl/ConfigLong.java b/src/main/java/com/typesafe/config/impl/ConfigLong.java index 8bf9fe05..58cdab74 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigLong.java +++ b/src/main/java/com/typesafe/config/impl/ConfigLong.java @@ -49,4 +49,9 @@ final class ConfigLong extends AbstractConfigValue { else return unwrapped().hashCode(); // use Long.hashCode() } + + @Override + String transformToString() { + return Long.toString(value); + } } diff --git a/src/main/java/com/typesafe/config/impl/ConfigNull.java b/src/main/java/com/typesafe/config/impl/ConfigNull.java index ccbb9cf4..84a7a415 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigNull.java +++ b/src/main/java/com/typesafe/config/impl/ConfigNull.java @@ -26,4 +26,9 @@ final class ConfigNull extends AbstractConfigValue { public Object unwrapped() { return null; } + + @Override + String transformToString() { + return "null"; + } } diff --git a/src/main/java/com/typesafe/config/impl/ConfigString.java b/src/main/java/com/typesafe/config/impl/ConfigString.java index 33034c00..04deeda3 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigString.java +++ b/src/main/java/com/typesafe/config/impl/ConfigString.java @@ -21,4 +21,9 @@ final class ConfigString extends AbstractConfigValue { public String unwrapped() { return value; } + + @Override + String transformToString() { + return value; + } } diff --git a/src/main/java/com/typesafe/config/impl/DefaultTransformer.java b/src/main/java/com/typesafe/config/impl/DefaultTransformer.java index cf2070a9..af944939 100644 --- a/src/main/java/com/typesafe/config/impl/DefaultTransformer.java +++ b/src/main/java/com/typesafe/config/impl/DefaultTransformer.java @@ -38,15 +38,17 @@ class DefaultTransformer implements ConfigTransformer { break; } } else if (requested == ConfigValueType.STRING) { + // if we converted null to string here, then you wouldn't properly + // get a missing-value error if you tried to get a null value + // as a string. switch (value.valueType()) { case NUMBER: // FALL THROUGH case BOOLEAN: - return new ConfigString(value.origin(), value.unwrapped() - .toString()); + return new ConfigString(value.origin(), + ((AbstractConfigValue) value).transformToString()); } } return value; } - } diff --git a/src/main/java/com/typesafe/config/impl/Parser.java b/src/main/java/com/typesafe/config/impl/Parser.java index 2fcd9874..9fe15efb 100644 --- a/src/main/java/com/typesafe/config/impl/Parser.java +++ b/src/main/java/com/typesafe/config/impl/Parser.java @@ -15,6 +15,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Stack; import com.typesafe.config.ConfigException; import com.typesafe.config.ConfigOrigin; @@ -54,9 +55,15 @@ final class Parser { if (f.getName().endsWith(".json")) flavor = SyntaxFlavor.JSON; else if (f.getName().endsWith(".conf")) - flavor = SyntaxFlavor.HOCON; + flavor = SyntaxFlavor.CONF; else throw new ConfigException.IO(origin, "Unknown filename extension"); + return parse(flavor, f); + } + + static AbstractConfigValue parse(SyntaxFlavor flavor, File f) { + ConfigOrigin origin = new SimpleConfigOrigin(f.getPath()); + AbstractConfigValue result = null; try { InputStream stream = new BufferedInputStream(new FileInputStream(f)); @@ -70,24 +77,104 @@ final class Parser { static private final class ParseContext { private int lineNumber; + private Stack buffer; + private Iterator tokens; private SyntaxFlavor flavor; private ConfigOrigin baseOrigin; - ParseContext(SyntaxFlavor flavor, ConfigOrigin origin) { + ParseContext(SyntaxFlavor flavor, ConfigOrigin origin, + Iterator tokens) { lineNumber = 0; + buffer = new Stack(); + this.tokens = tokens; this.flavor = flavor; this.baseOrigin = origin; } - private Token nextTokenIgnoringNewline(Iterator tokens) { - Token t = tokens.next(); + private Token nextToken() { + Token t = null; + if (buffer.isEmpty()) { + t = tokens.next(); + } else { + t = buffer.pop(); + } + + if (Tokens.isUnquotedText(t) && flavor == SyntaxFlavor.JSON) { + throw parseError("Token not allowed in valid JSON: '" + + Tokens.getUnquotedText(t) + "'"); + } else { + return t; + } + } + + private void putBack(Token token) { + buffer.push(token); + } + + private Token nextTokenIgnoringNewline() { + Token t = nextToken(); while (Tokens.isNewline(t)) { lineNumber = Tokens.getLineNumber(t); - t = tokens.next(); + t = nextToken(); } return t; } + // merge a bunch of adjacent values into one + // value; change unquoted text into a string + // value. + private void consolidateValueTokens() { + // this trick is not done in JSON + if (flavor == SyntaxFlavor.JSON) + return; + + List values = null; // create only if we have value tokens + Token t = nextTokenIgnoringNewline(); // ignore a newline up front + while (Tokens.isValue(t) + || Tokens.isUnquotedText(t)) { + if (values == null) + values = new ArrayList(); + values.add(t); + t = nextToken(); // but don't consolidate across a newline + } + // the last one wasn't a value token + putBack(t); + + if (values == null) + return; + + if (values.size() == 1 && !Tokens.isUnquotedText(values.get(0))) { + // a single value token requires no consolidation + putBack(values.get(0)); + return; + } + + // we have multiple value tokens or one unquoted text token; + // collapse into a string token. + StringBuilder sb = new StringBuilder(); + ConfigOrigin firstOrigin = null; + for (Token valueToken : values) { + if (Tokens.isValue(valueToken)) { + AbstractConfigValue v = Tokens.getValue(valueToken); + sb.append(v.transformToString()); + if (firstOrigin == null) + firstOrigin = v.origin(); + } else if (Tokens.isUnquotedText(valueToken)) { + String text = Tokens.getUnquotedText(valueToken); + if (firstOrigin == null) + firstOrigin = Tokens.getUnquotedTextOrigin(valueToken); + sb.append(text); + } else { + throw new ConfigException.BugOrBroken( + "should not be trying to consolidate token: " + + valueToken); + } + } + + Token consolidated = Tokens.newString(firstOrigin, sb.toString()); + putBack(consolidated); + } + private ConfigOrigin lineOrigin() { return new SimpleConfigOrigin(baseOrigin.description() + ": line " + lineNumber); @@ -101,48 +188,49 @@ final class Parser { return new ConfigException.Parse(lineOrigin(), message, cause); } - private AbstractConfigValue parseValue(Token token, - Iterator tokens) { + private AbstractConfigValue parseValue(Token token) { if (Tokens.isValue(token)) { return Tokens.getValue(token); } else if (token == Tokens.OPEN_CURLY) { - return parseObject(tokens); + return parseObject(); } else if (token == Tokens.OPEN_SQUARE) { - return parseArray(tokens); + return parseArray(); } else { throw parseError("Expecting a value but got wrong token: " + token); } } - private AbstractConfigObject parseObject(Iterator tokens) { + private AbstractConfigObject parseObject() { // invoked just after the OPEN_CURLY Map values = new HashMap(); ConfigOrigin objectOrigin = lineOrigin(); while (true) { - Token t = nextTokenIgnoringNewline(tokens); + Token t = nextTokenIgnoringNewline(); if (Tokens.isValueWithType(t, ConfigValueType.STRING)) { String key = (String) Tokens.getValue(t).unwrapped(); - Token afterKey = nextTokenIgnoringNewline(tokens); + Token afterKey = nextTokenIgnoringNewline(); if (afterKey != Tokens.COLON) { throw parseError("Key not followed by a colon, followed by token " + afterKey); } - Token valueToken = nextTokenIgnoringNewline(tokens); + + consolidateValueTokens(); + Token valueToken = nextTokenIgnoringNewline(); // note how we handle duplicate keys: the last one just // wins. // FIXME in strict JSON, dups should be an error; while in // our custom config language, they should be merged if the // value is an object. - values.put(key, parseValue(valueToken, tokens)); + values.put(key, parseValue(valueToken)); } else if (t == Tokens.CLOSE_CURLY) { break; } else { throw parseError("Expecting close brace } or a field name, got " + t); } - t = nextTokenIgnoringNewline(tokens); + t = nextTokenIgnoringNewline(); if (t == Tokens.CLOSE_CURLY) { break; } else if (t == Tokens.COMMA) { @@ -155,22 +243,25 @@ final class Parser { return new SimpleConfigObject(objectOrigin, null, values); } - private ConfigList parseArray(Iterator tokens) { + private ConfigList parseArray() { // invoked just after the OPEN_SQUARE ConfigOrigin arrayOrigin = lineOrigin(); List values = new ArrayList(); - Token t = nextTokenIgnoringNewline(tokens); + + consolidateValueTokens(); + + Token t = nextTokenIgnoringNewline(); // special-case the first element if (t == Tokens.CLOSE_SQUARE) { return new ConfigList(arrayOrigin, Collections. emptyList()); } else if (Tokens.isValue(t)) { - values.add(parseValue(t, tokens)); + values.add(parseValue(t)); } else if (t == Tokens.OPEN_CURLY) { - values.add(parseObject(tokens)); + values.add(parseObject()); } else if (t == Tokens.OPEN_SQUARE) { - values.add(parseArray(tokens)); + values.add(parseArray()); } else { throw parseError("List should have ] or a first element after the open [, instead had token: " + t); @@ -179,7 +270,7 @@ final class Parser { // now remaining elements while (true) { // just after a value - t = nextTokenIgnoringNewline(tokens); + t = nextTokenIgnoringNewline(); if (t == Tokens.CLOSE_SQUARE) { return new ConfigList(arrayOrigin, values); } else if (t == Tokens.COMMA) { @@ -190,13 +281,15 @@ final class Parser { } // now just after a comma - t = nextTokenIgnoringNewline(tokens); + consolidateValueTokens(); + + t = nextTokenIgnoringNewline(); if (Tokens.isValue(t)) { - values.add(parseValue(t, tokens)); + values.add(parseValue(t)); } else if (t == Tokens.OPEN_CURLY) { - values.add(parseObject(tokens)); + values.add(parseObject()); } else if (t == Tokens.OPEN_SQUARE) { - values.add(parseArray(tokens)); + values.add(parseArray()); } else { throw parseError("List should have had new element after a comma, instead had token: " + t); @@ -204,8 +297,8 @@ final class Parser { } } - AbstractConfigValue parse(Iterator tokens) { - Token t = nextTokenIgnoringNewline(tokens); + AbstractConfigValue parse() { + Token t = nextTokenIgnoringNewline(); if (t == Tokens.START) { // OK } else { @@ -213,12 +306,12 @@ final class Parser { "token stream did not begin with START, had " + t); } - t = nextTokenIgnoringNewline(tokens); + t = nextTokenIgnoringNewline(); AbstractConfigValue result = null; if (t == Tokens.OPEN_CURLY) { - result = parseObject(tokens); + result = parseObject(); } else if (t == Tokens.OPEN_SQUARE) { - result = parseArray(tokens); + result = parseArray(); } else if (t == Tokens.END) { throw parseError("Empty document"); } else { @@ -226,7 +319,7 @@ final class Parser { + t); } - t = nextTokenIgnoringNewline(tokens); + t = nextTokenIgnoringNewline(); if (t == Tokens.END) { return result; } else { @@ -239,7 +332,7 @@ final class Parser { private static AbstractConfigValue parse(SyntaxFlavor flavor, ConfigOrigin origin, Iterator tokens) { - ParseContext context = new ParseContext(flavor, origin); - return context.parse(tokens); + ParseContext context = new ParseContext(flavor, origin, tokens); + return context.parse(); } } diff --git a/src/main/java/com/typesafe/config/impl/SyntaxFlavor.java b/src/main/java/com/typesafe/config/impl/SyntaxFlavor.java index 6572e680..868b413a 100644 --- a/src/main/java/com/typesafe/config/impl/SyntaxFlavor.java +++ b/src/main/java/com/typesafe/config/impl/SyntaxFlavor.java @@ -1,5 +1,5 @@ package com.typesafe.config.impl; enum SyntaxFlavor { - JSON, HOCON + JSON, CONF } diff --git a/src/main/java/com/typesafe/config/impl/TokenType.java b/src/main/java/com/typesafe/config/impl/TokenType.java index af545b1c..5694c275 100644 --- a/src/main/java/com/typesafe/config/impl/TokenType.java +++ b/src/main/java/com/typesafe/config/impl/TokenType.java @@ -1,5 +1,5 @@ package com.typesafe.config.impl; enum TokenType { - START, END, COMMA, COLON, OPEN_CURLY, CLOSE_CURLY, OPEN_SQUARE, CLOSE_SQUARE, VALUE, NEWLINE; + START, END, COMMA, COLON, OPEN_CURLY, CLOSE_CURLY, OPEN_SQUARE, CLOSE_SQUARE, VALUE, NEWLINE, UNQUOTED_TEXT; } diff --git a/src/main/java/com/typesafe/config/impl/Tokenizer.java b/src/main/java/com/typesafe/config/impl/Tokenizer.java index 1506f262..4a46011e 100644 --- a/src/main/java/com/typesafe/config/impl/Tokenizer.java +++ b/src/main/java/com/typesafe/config/impl/Tokenizer.java @@ -73,48 +73,58 @@ final class Tokenizer { return new ConfigException.Parse(lineOrigin(), message, cause); } - private void checkNextOrThrow(String expectedBefore, String expectedNow) { - int i = 0; - while (i < expectedNow.length()) { - int expected = expectedNow.charAt(i); - int actual = nextChar(); - - if (actual == -1) - throw parseError(String.format( - "Expecting '%s%s' but input data ended", - expectedBefore, expectedNow)); - - if (actual != expected) - throw parseError(String - .format("Expecting '%s%s' but got char '%c' rather than '%c'", - expectedBefore, expectedNow, actual, - expected)); - - ++i; - } - } - private ConfigOrigin lineOrigin() { return new SimpleConfigOrigin(origin.description() + ": line " + lineNumber); } - private Token pullTrue() { - // "t" has been already seen - checkNextOrThrow("t", "rue"); - return Tokens.newBoolean(lineOrigin(), true); - } + // chars JSON allows a number to start with + static final String firstNumberChars = "0123456789-"; + // chars JSON allows to be part of a number + static final String numberChars = "0123456789eE+-."; + // chars that stop an unquoted string + static final String notInUnquotedText = "$\"{}[]:=\n,"; - private Token pullFalse() { - // "f" has been already seen - checkNextOrThrow("f", "alse"); - return Tokens.newBoolean(lineOrigin(), false); - } + // The rules here are intended to maximize convenience while + // avoiding confusion with real valid JSON. Basically anything + // that parses as JSON is treated the JSON way and otherwise + // we assume it's a string and let the parser sort it out. + private Token pullUnquotedText() { + ConfigOrigin origin = lineOrigin(); + StringBuilder sb = new StringBuilder(); + int c = nextChar(); + while (true) { + if (c == -1) { + break; + } else if (notInUnquotedText.indexOf(c) >= 0) { + break; + } else { + sb.append((char) c); + } - private Token pullNull() { - // "n" has been already seen - checkNextOrThrow("n", "ull"); - return Tokens.newNull(lineOrigin()); + // we parse true/false/null tokens as such no matter + // what is after them. + if (sb.length() == 4) { + String s = sb.toString(); + if (s.equals("true")) + return Tokens.newBoolean(origin, true); + else if (s.equals("null")) + return Tokens.newNull(origin); + } else if (sb.length() == 5) { + String s = sb.toString(); + if (s.equals("false")) + return Tokens.newBoolean(origin, false); + } + + c = nextChar(); + } + + // put back the char that ended the unquoted text + putBack(c); + + // chop trailing whitespace; have to quote to have trailing spaces. + String s = sb.toString().trim(); + return Tokens.newUnquotedText(origin, s); } private Token pullNumber(int firstChar) { @@ -122,7 +132,7 @@ final class Tokenizer { sb.append((char) firstChar); boolean containedDecimalOrE = false; int c = nextChar(); - while (c != -1 && "0123456789eE+-.".indexOf(c) >= 0) { + while (c != -1 && numberChars.indexOf(c) >= 0) { if (c == '.' || c == 'e' || c == 'E') containedDecimalOrE = true; sb.append((char) c); @@ -255,25 +265,21 @@ final class Tokenizer { case ']': t = Tokens.CLOSE_SQUARE; break; - case 't': - t = pullTrue(); - break; - case 'f': - t = pullFalse(); - break; - case 'n': - t = pullNull(); - break; } + if (t == null) { - if ("-0123456789".indexOf(c) >= 0) { + if (firstNumberChars.indexOf(c) >= 0) { t = pullNumber(c); - } else { + } else if (notInUnquotedText.indexOf(c) >= 0) { throw parseError(String .format("Character '%c' is not the start of any valid token", c)); + } else { + putBack(c); + t = pullUnquotedText(); } } + if (t == null) throw new ConfigException.BugOrBroken( "bug: failed to generate next token"); diff --git a/src/main/java/com/typesafe/config/impl/Tokens.java b/src/main/java/com/typesafe/config/impl/Tokens.java index b3eda291..781918b6 100644 --- a/src/main/java/com/typesafe/config/impl/Tokens.java +++ b/src/main/java/com/typesafe/config/impl/Tokens.java @@ -80,6 +80,47 @@ final class Tokens { } } + // This is not a Value, because it requires special processing + static private class UnquotedText extends Token { + private ConfigOrigin origin; + private String value; + + UnquotedText(ConfigOrigin origin, String s) { + super(TokenType.UNQUOTED_TEXT); + this.origin = origin; + this.value = s; + } + + ConfigOrigin origin() { + return origin; + } + + String value() { + return value; + } + + @Override + public String toString() { + return tokenType().name() + "(" + value + ")"; + } + + @Override + protected boolean canEqual(Object other) { + return other instanceof UnquotedText; + } + + @Override + public boolean equals(Object other) { + return super.equals(other) + && ((UnquotedText) other).value.equals(value); + } + + @Override + public int hashCode() { + return 41 * (41 + super.hashCode()) + value.hashCode(); + } + } + static boolean isValue(Token token) { return token instanceof Value; } @@ -89,7 +130,7 @@ final class Tokens { return ((Value) token).value(); } else { throw new ConfigException.BugOrBroken( - "tried to get value of non-value token"); + "tried to get value of non-value token " + token); } } @@ -106,10 +147,43 @@ final class Tokens { return ((Line) token).lineNumber(); } else { throw new ConfigException.BugOrBroken( - "tried to get line number from non-newline"); + "tried to get line number from non-newline " + token); } } + static boolean isUnquotedText(Token token) { + return token instanceof UnquotedText; + } + + static String getUnquotedText(Token token) { + if (token instanceof UnquotedText) { + return ((UnquotedText) token).value(); + } else { + throw new ConfigException.BugOrBroken( + "tried to get unquoted text from " + token); + } + } + + static ConfigOrigin getUnquotedTextOrigin(Token token) { + if (token instanceof UnquotedText) { + return ((UnquotedText) token).origin(); + } else { + throw new ConfigException.BugOrBroken( + "tried to get unquoted text from " + token); + } + } + + /* + * static ConfigString newStringValueFromTokens(Token... tokens) { + * StringBuilder sb = new StringBuilder(); for (Token t : tokens) { if + * (isValue(t)) { ConfigValue v = getValue(t); if (v instanceof + * ConfigString) { sb.append(((ConfigString) v).unwrapped()); } else { // + * FIXME convert non-strings to string throw new + * ConfigException.BugOrBroken( "not handling non-strings here"); } } else + * if (isUnquotedText(t)) { String s = getUnquotedText(t); sb.append(s); } + * else { throw new ConfigException. } } } + */ + static Token START = new Token(TokenType.START); static Token END = new Token(TokenType.END); static Token COMMA = new Token(TokenType.COMMA); @@ -123,6 +197,10 @@ final class Tokens { return new Line(lineNumberJustEnded); } + static Token newUnquotedText(ConfigOrigin origin, String s) { + return new UnquotedText(origin, s); + } + static Token newValue(AbstractConfigValue value) { return new Value(value); } diff --git a/src/test/resources/equiv01/no-whitespace.json b/src/test/resources/equiv01/no-whitespace.json index 24caddef..72d0bcd7 100644 --- a/src/test/resources/equiv01/no-whitespace.json +++ b/src/test/resources/equiv01/no-whitespace.json @@ -1 +1 @@ -{"ints":{"fortyTwo":42,"fortyTwoAgain":42},"floats":{"fortyTwoPointOne":42.1,"fortyTwoPointOneAgain":42.1},"strings":{"abcd":"abcd","abcdAgain":"abcd","a":"a","b":"b","c":"c","d":"d"},"arrays":{"empty":[],"1":[1],"12":[1,2],"123":[1,2,3]},"booleans":{"true":true,"trueAgain":true,"false":false,"falseAgain":false},"nulls":{"null":null,"nullAgain":null}} \ No newline at end of file +{"ints":{"fortyTwo":42,"fortyTwoAgain":42},"floats":{"fortyTwoPointOne":42.1,"fortyTwoPointOneAgain":42.1},"strings":{"abcd":"abcd","abcdAgain":"abcd","a":"a","b":"b","c":"c","d":"d","concatenated":"null bar 42 baz true 3.14 hi"},"arrays":{"empty":[],"1":[1],"12":[1,2],"123":[1,2,3],"ofString":["a","b","c"]},"booleans":{"true":true,"trueAgain":true,"false":false,"falseAgain":false},"nulls":{"null":null,"nullAgain":null}} \ No newline at end of file diff --git a/src/test/resources/equiv01/original.json b/src/test/resources/equiv01/original.json index 33711c57..b02798cf 100644 --- a/src/test/resources/equiv01/original.json +++ b/src/test/resources/equiv01/original.json @@ -15,14 +15,16 @@ "a" : "a", "b" : "b", "c" : "c", - "d" : "d" + "d" : "d", + "concatenated" : "null bar 42 baz true 3.14 hi" }, "arrays" : { "empty" : [], "1" : [ 1 ], "12" : [1, 2], - "123" : [1, 2, 3] + "123" : [1, 2, 3], + "ofString" : [ "a", "b", "c" ] }, "booleans" : { diff --git a/src/test/resources/equiv01/unquoted.conf b/src/test/resources/equiv01/unquoted.conf new file mode 100644 index 00000000..b43cab90 --- /dev/null +++ b/src/test/resources/equiv01/unquoted.conf @@ -0,0 +1,41 @@ +{ + "ints" : { + "fortyTwo" : 42, + "fortyTwoAgain" : 42 + }, + + "floats" : { + "fortyTwoPointOne" : 42.1, + "fortyTwoPointOneAgain" : 42.1 + }, + + "strings" : { + "abcd" : abcd, + "abcdAgain" : abcd, + "a" : a, + "b" : b, + "c" : c, + "d" : d, + "concatenated" : null bar 42 "baz" true 3.14 hi + }, + + "arrays" : { + "empty" : [], + "1" : [ 1 ], + "12" : [1, 2], + "123" : [1, 2, 3], + "ofString" : [a, b, c] + }, + + "booleans" : { + "true" : true, + "trueAgain" : true, + "false" : false, + "falseAgain" : false + }, + + "nulls" : { + "null" : null, + "nullAgain" : null + } +} diff --git a/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala b/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala new file mode 100644 index 00000000..ba430a38 --- /dev/null +++ b/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala @@ -0,0 +1,51 @@ +package com.typesafe.config.impl + +import org.junit.Assert._ +import org.junit._ +import java.io.Reader +import java.io.StringReader +import com.typesafe.config._ +import java.util.HashMap + +class ConfParserTest extends TestUtils { + + @org.junit.Before + def setup() { + } + + def parse(s: String): ConfigValue = { + Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test conf string"), s) + } + + private def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = { + try { + body + } catch { + case t: Throwable => + throw new AssertionError(parserName + " parser did wrong thing on '" + s + "'", t) + } + } + + @Test + def invalidConfThrows(): Unit = { + // be sure we throw + for (invalid <- whitespaceVariations(invalidConf)) { + addOffendingJsonToException("config", invalid.test) { + intercept[ConfigException] { + parse(invalid.test) + } + } + } + } + + @Test + def validConfWorks(): Unit = { + // all we're checking here unfortunately is that it doesn't throw. + // for a more thorough check, use the EquivalentsTest stuff. + for (valid <- whitespaceVariations(validConf)) { + val ourAST = addOffendingJsonToException("config-conf", valid.test) { + parse(valid.test) + } + } + } +} diff --git a/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala b/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala index 4906237a..63400762 100644 --- a/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala +++ b/src/test/scala/com/typesafe/config/impl/EquivalentsTest.scala @@ -24,7 +24,7 @@ class EquivalentsTest extends TestUtils { private def filesForEquiv(equiv: File) = { val rawFiles = equiv.listFiles() - val files = rawFiles.filter({ f => f.getName().endsWith(".json") }) + val files = rawFiles.filter({ f => f.getName().endsWith(".json") || f.getName().endsWith(".conf") }) files } @@ -51,6 +51,15 @@ class EquivalentsTest extends TestUtils { describeFailure(testFile.getPath()) { assertEquals(original, value) } + + // check that all .json files can be parsed as .conf, + // i.e. .conf must be a superset of JSON + if (testFile.getName().endsWith(".json")) { + val parsedAsConf = Parser.parse(SyntaxFlavor.CONF, testFile) + describeFailure(testFile.getPath() + " parsed as .conf") { + assertEquals(original, parsedAsConf) + } + } } } diff --git a/src/test/scala/com/typesafe/config/impl/JsonTest.scala b/src/test/scala/com/typesafe/config/impl/JsonTest.scala index b7de4771..18a68ad5 100644 --- a/src/test/scala/com/typesafe/config/impl/JsonTest.scala +++ b/src/test/scala/com/typesafe/config/impl/JsonTest.scala @@ -8,14 +8,18 @@ import java.io.StringReader import com.typesafe.config._ import java.util.HashMap -class JsonTest extends TestUtils { +class ParseTest extends TestUtils { @org.junit.Before def setup() { } def parse(s: String): ConfigValue = { - Parser.parse(SyntaxFlavor.JSON, new SimpleConfigOrigin("test string"), s) + Parser.parse(SyntaxFlavor.JSON, new SimpleConfigOrigin("test json string"), s) + } + + def parseAsConf(s: String): ConfigValue = { + Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test conf string"), s) } private[this] def toLift(value: ConfigValue): lift.JValue = { @@ -88,90 +92,15 @@ class JsonTest extends TestUtils { withLiftExceptionsConverted(fromLift(lift.JsonParser.parse(json))) } - case class JsonTest(liftBehaviorUnexpected: Boolean, test: String) - implicit def string2jsontest(test: String): JsonTest = JsonTest(false, test) - - private val invalidJson = List[JsonTest]("", // empty document - "{", - "}", - "[", - "]", - "10", // value not in array or object - "\"foo\"", // value not in array or object - "\"", // single quote by itself - "{ \"foo\" : }", // no value in object - "{ : 10 }", // no key in object - // these two problems are ignored by the lift tokenizer - "[:\"foo\", \"bar\"]", // colon in an array; lift doesn't throw (tokenizer erases it) - "[\"foo\" : \"bar\"]", // colon in an array another way, lift ignores (tokenizer erases it) - "[ foo ]", // not a known token - "[ t ]", // start of "true" but ends wrong - "[ tx ]", - "[ tr ]", - "[ trx ]", - "[ tru ]", - "[ trux ]", - "[ truex ]", - "[ 10x ]", // number token with trailing junk - "[ 10e3e3 ]", // two exponents - "[ \"hello ]", // unterminated string - JsonTest(true, "{ \"foo\" , true }"), // comma instead of colon, lift is fine with this - JsonTest(true, "{ \"foo\" : true \"bar\" : false }"), // missing comma between fields, lift fine with this - "[ 10, }]", // array with } as an element - "[ 10, {]", // array with { as an element - "{}x", // trailing invalid token after the root object - "[]x", // trailing invalid token after the root array - JsonTest(true, "{}{}"), // trailing token after the root object - lift OK with it - "{}true", // trailing token after the root object - JsonTest(true, "[]{}"), // trailing valid token after the root array - "[]true", // trailing valid token after the root array - "") // empty document again, just for clean formatting of this list ;-) - - // We'll automatically try each of these with whitespace modifications - // so no need to add every possible whitespace variation - private val validJson = List[JsonTest]("{}", - "[]", - """{ "foo" : "bar" }""", - """["foo", "bar"]""", - """{ "foo" : 42 }""", - """[10, 11]""", - """[10,"foo"]""", - """{ "foo" : "bar", "baz" : "boo" }""", - """{ "foo" : { "bar" : "baz" }, "baz" : "boo" }""", - """{ "foo" : { "bar" : "baz", "woo" : "w00t" }, "baz" : "boo" }""", - """{ "foo" : [10,11,12], "baz" : "boo" }""", - JsonTest(true, """{ "foo" : "bar", "foo" : "bar2" }"""), // dup keys - lift just returns both, we use last one - """[{},{},{},{}]""", - """[[[[[[]]]]]]""", - """{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":42}}}}}}}}""", - // this long one is mostly to test rendering - """{ "foo" : { "bar" : "baz", "woo" : "w00t" }, "baz" : { "bar" : "baz", "woo" : [1,2,3,4], "w00t" : true, "a" : false, "b" : 3.14, "c" : null } }""", - "{}") - // For string quoting, check behavior of escaping a random character instead of one on the list; // lift-json seems to oddly treat that as a \ literal - private def whitespaceVariations(tests: Seq[JsonTest]): Seq[JsonTest] = { - val variations = List({ s: String => s }, // identity - { s: String => " " + s }, - { s: String => s + " " }, - { s: String => " " + s + " " }, - { s: String => s.replace(" ", "") }, // this would break with whitespace in a key or value - { s: String => s.replace(":", " : ") }, // could break with : in a key or value - { s: String => s.replace(",", " , ") } // could break with , in a key or value - ) - for { - t <- tests - v <- variations - } yield JsonTest(t.liftBehaviorUnexpected, v(t.test)) - } - private def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = { try { body } catch { case t: Throwable => - throw new AssertionError(parserName + " parser failed on '" + s + "'", t) + throw new AssertionError(parserName + " parser did wrong thing on '" + s + "'", t) } } @@ -211,9 +140,12 @@ class JsonTest extends TestUtils { val liftAST = addOffendingJsonToException("lift", valid.test) { fromJsonWithLiftParser(valid.test) } - val ourAST = addOffendingJsonToException("config", valid.test) { + val ourAST = addOffendingJsonToException("config-json", valid.test) { parse(valid.test) } + val ourConfAST = addOffendingJsonToException("config-conf", valid.test) { + parseAsConf(valid.test) + } if (valid.liftBehaviorUnexpected) { // ignore this for now } else { @@ -221,6 +153,12 @@ class JsonTest extends TestUtils { assertEquals(liftAST, ourAST) } } + + // check that our parser gives the same result in JSON mode and ".conf" mode. + // i.e. this tests that ".conf" format is a superset of JSON. + addOffendingJsonToException("config", valid.test) { + assertEquals(ourAST, ourConfAST) + } } } } diff --git a/src/test/scala/com/typesafe/config/impl/TestUtils.scala b/src/test/scala/com/typesafe/config/impl/TestUtils.scala index 7eaa24ce..0e7340da 100644 --- a/src/test/scala/com/typesafe/config/impl/TestUtils.scala +++ b/src/test/scala/com/typesafe/config/impl/TestUtils.scala @@ -47,4 +47,93 @@ abstract trait TestUtils { def fakeOrigin() = { new SimpleConfigOrigin("fake origin") } + + case class ParseTest(liftBehaviorUnexpected: Boolean, test: String) + implicit def string2jsontest(test: String): ParseTest = ParseTest(false, test) + + private val invalidJsonInvalidConf = List[ParseTest]("", // empty document + "{", + "}", + "[", + "]", + "10", // value not in array or object + "\"foo\"", // value not in array or object + "\"", // single quote by itself + "{ \"foo\" : }", // no value in object + "{ : 10 }", // no key in object + "{ foo : \"bar\" }", // no quotes on key + "{ foo : bar }", // no quotes on key or value + // these two problems are ignored by the lift tokenizer + "[:\"foo\", \"bar\"]", // colon in an array; lift doesn't throw (tokenizer erases it) + "[\"foo\" : \"bar\"]", // colon in an array another way, lift ignores (tokenizer erases it) + "[ 10e3e3 ]", // two exponents. ideally this might parse to a number plus string "e3" but it's hard to implement. + "[ \"hello ]", // unterminated string + ParseTest(true, "{ \"foo\" , true }"), // comma instead of colon, lift is fine with this + ParseTest(true, "{ \"foo\" : true \"bar\" : false }"), // missing comma between fields, lift fine with this + "[ 10, }]", // array with } as an element + "[ 10, {]", // array with { as an element + "{}x", // trailing invalid token after the root object + "[]x", // trailing invalid token after the root array + ParseTest(true, "{}{}"), // trailing token after the root object - lift OK with it + "{}true", // trailing token after the root object + ParseTest(true, "[]{}"), // trailing valid token after the root array + "[]true", // trailing valid token after the root array + "") // empty document again, just for clean formatting of this list ;-) + + // We'll automatically try each of these with whitespace modifications + // so no need to add every possible whitespace variation + protected val validJson = List[ParseTest]("{}", + "[]", + """{ "foo" : "bar" }""", + """["foo", "bar"]""", + """{ "foo" : 42 }""", + """[10, 11]""", + """[10,"foo"]""", + """{ "foo" : "bar", "baz" : "boo" }""", + """{ "foo" : { "bar" : "baz" }, "baz" : "boo" }""", + """{ "foo" : { "bar" : "baz", "woo" : "w00t" }, "baz" : "boo" }""", + """{ "foo" : [10,11,12], "baz" : "boo" }""", + ParseTest(true, """{ "foo" : "bar", "foo" : "bar2" }"""), // dup keys - lift just returns both, we use last one + """[{},{},{},{}]""", + """[[[[[[]]]]]]""", + """{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":42}}}}}}}}""", + // this long one is mostly to test rendering + """{ "foo" : { "bar" : "baz", "woo" : "w00t" }, "baz" : { "bar" : "baz", "woo" : [1,2,3,4], "w00t" : true, "a" : false, "b" : 3.14, "c" : null } }""", + "{}") + + private val validConfInvalidJson = List[ParseTest]( + """{ "foo" : bar }""", // no quotes on value + """{ "foo" : null bar 42 baz true 3.14 "hi" }""", // bunch of values to concat into a string + "[ foo ]", // not a known token in JSON + "[ t ]", // start of "true" but ends wrong in JSON + "[ tx ]", + "[ tr ]", + "[ trx ]", + "[ tru ]", + "[ trux ]", + "[ truex ]", + "[ 10x ]") // number token with trailing junk + + protected val invalidJson = validConfInvalidJson ++ invalidJsonInvalidConf; + + protected val invalidConf = invalidJsonInvalidConf; + + // .conf is a superset of JSON so validJson just goes in here + protected val validConf = validConfInvalidJson ++ validJson; + + protected def whitespaceVariations(tests: Seq[ParseTest]): Seq[ParseTest] = { + val variations = List({ s: String => s }, // identity + { s: String => " " + s }, + { s: String => s + " " }, + { s: String => " " + s + " " }, + { s: String => s.replace(" ", "") }, // this would break with whitespace in a key or value + { s: String => s.replace(":", " : ") }, // could break with : in a key or value + { s: String => s.replace(",", " , ") } // could break with , in a key or value + ) + for { + t <- tests + v <- variations + } yield ParseTest(t.liftBehaviorUnexpected, v(t.test)) + } + } diff --git a/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala b/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala index 166e30ff..01ecd078 100644 --- a/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala +++ b/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala @@ -44,12 +44,12 @@ class TokenizerTest extends TestUtils { def tokenizeAllTypesNoSpaces() { // all token types with no spaces (not sure JSON spec wants this to work, // but spec is unclear to me when spaces are required, and banning them - // is actually extra work) + // is actually extra work). val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"), - Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), - Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) - assertEquals(expected, tokenizeAsList(""",:}{]["foo"42true3.14falsenull""" + "\n")) + Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), Tokens.newBoolean(fakeOrigin(), false), + Tokens.newLong(fakeOrigin(), 42), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) + assertEquals(expected, tokenizeAsList(""",:}{]["foo"true3.14false42null""" + "\n")) } @Test @@ -76,6 +76,62 @@ class TokenizerTest extends TestUtils { assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n ")) } + @Test + def tokenizeTrueAndUnquotedText() { + val expected = List(Tokens.START, Tokens.newBoolean(fakeOrigin(), true), Tokens.newUnquotedText(fakeOrigin(), "foo"), Tokens.END) + assertEquals(expected, tokenizeAsList("""truefoo""")) + } + + @Test + def tokenizeFalseAndUnquotedText() { + val expected = List(Tokens.START, Tokens.newBoolean(fakeOrigin(), false), Tokens.newUnquotedText(fakeOrigin(), "foo"), Tokens.END) + assertEquals(expected, tokenizeAsList("""falsefoo""")) + } + + @Test + def tokenizeNullAndUnquotedText() { + val expected = List(Tokens.START, Tokens.newNull(fakeOrigin()), Tokens.newUnquotedText(fakeOrigin(), "foo"), Tokens.END) + assertEquals(expected, tokenizeAsList("""nullfoo""")) + } + + @Test + def tokenizeUnquotedTextContainingTrue() { + val expected = List(Tokens.START, Tokens.newUnquotedText(fakeOrigin(), "footrue"), Tokens.END) + assertEquals(expected, tokenizeAsList("""footrue""")) + } + + @Test + def tokenizeUnquotedTextContainingSpaceTrue() { + val expected = List(Tokens.START, Tokens.newUnquotedText(fakeOrigin(), "foo true"), Tokens.END) + assertEquals(expected, tokenizeAsList("""foo true""")) + } + + @Test + def tokenizeTrueAndSpaceAndUnquotedText() { + val expected = List(Tokens.START, Tokens.newBoolean(fakeOrigin(), true), Tokens.newUnquotedText(fakeOrigin(), "foo"), Tokens.END) + assertEquals(expected, tokenizeAsList("""true foo""")) + } + + @Test + def tokenizeUnquotedTextTrimsSpaces() { + val expected = List(Tokens.START, Tokens.newUnquotedText(fakeOrigin(), "foo"), Tokens.newLine(0), Tokens.END) + assertEquals(expected, tokenizeAsList(" foo \n")) + } + + @Test + def tokenizeUnquotedTextKeepsInternalSpaces() { + val expected = List(Tokens.START, Tokens.newUnquotedText(fakeOrigin(), "foo bar baz"), Tokens.newLine(0), Tokens.END) + assertEquals(expected, tokenizeAsList(" foo bar baz \n")) + } + + @Test + def tokenizeMixedUnquotedQuoted() { + val expected = List(Tokens.START, Tokens.newUnquotedText(fakeOrigin(), "foo"), + Tokens.newString(fakeOrigin(), "bar"), Tokens.newUnquotedText(fakeOrigin(), "baz"), + Tokens.newLine(0), Tokens.END) + assertEquals(expected, tokenizeAsList(" foo\"bar\"baz \n")) + } + @Test def tokenizerUnescapeStrings(): Unit = { case class UnescapeTest(escaped: String, result: ConfigString)