From e0a49c06daabd26c16dcd1ad6a72dd1ea1a85afa Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Thu, 10 Nov 2011 13:09:45 -0500 Subject: [PATCH] Generalize path handling in substitutions The new approach allows a mix of tokens in the substitution, so you can refer to path elements that contain characters that need quoting, including periods. --- SPEC.md | 24 +- .../com/typesafe/config/ConfigException.java | 4 + .../config/impl/AbstractConfigObject.java | 12 +- .../config/impl/AbstractConfigValue.java | 13 +- .../config/impl/ConfigSubstitution.java | 38 ++-- .../com/typesafe/config/impl/ConfigUtil.java | 49 +++++ .../java/com/typesafe/config/impl/Parser.java | 103 ++++++++- .../java/com/typesafe/config/impl/Path.java | 98 +++++++++ .../com/typesafe/config/impl/PathBuilder.java | 67 ++++++ .../typesafe/config/impl/Substitution.java | 45 ---- .../config/impl/SubstitutionStyle.java | 5 - .../com/typesafe/config/impl/Tokenizer.java | 207 +++++++++++------- .../java/com/typesafe/config/impl/Tokens.java | 45 +--- src/test/resources/test02.conf | 11 + .../typesafe/config/impl/ConfParserTest.scala | 68 +++++- .../config/impl/ConfigSubstitutionTest.scala | 2 +- .../com/typesafe/config/impl/ConfigTest.scala | 12 + .../config/impl/ConfigValueTest.scala | 13 -- .../com/typesafe/config/impl/JsonTest.scala | 19 +- .../com/typesafe/config/impl/PathTest.scala | 41 ++++ .../com/typesafe/config/impl/TestUtils.scala | 78 +++++-- .../com/typesafe/config/impl/TokenTest.scala | 8 - .../typesafe/config/impl/TokenizerTest.scala | 43 ++-- 23 files changed, 712 insertions(+), 293 deletions(-) create mode 100644 src/main/java/com/typesafe/config/impl/Path.java create mode 100644 src/main/java/com/typesafe/config/impl/PathBuilder.java delete mode 100644 src/main/java/com/typesafe/config/impl/Substitution.java delete mode 100644 src/main/java/com/typesafe/config/impl/SubstitutionStyle.java create mode 100644 src/test/resources/test02.conf create mode 100644 src/test/scala/com/typesafe/config/impl/PathTest.scala diff --git a/SPEC.md b/SPEC.md index b2a714ea..d184e997 100644 --- a/SPEC.md +++ b/SPEC.md @@ -156,6 +156,23 @@ Different from JSON: already then it refers to precisely that filename and the format is not flexible. +### Path expressions + +Path expressions are used to write out a path through the object +graph. They appear in two places; in substitutions, like +`${foo.bar}`, and as the keys in objects like `{ foo.bar : 42 }`. + +Path expressions work like a value concatenation, except that they +may not contain substitutions. This means that you can't nest +substitutions inside other substitutions, and you can't have +substitutions in keys. + +When concatenating the path expression, any `.` characters outside quoted +strings or numbers are understood as path separators, while inside quoted +strings `.` has no special meaning. So `foo.bar."hello.world"` would be +a path with three elements, looking up key `foo`, key `bar`, then key +`hello.world`. + ### Java properties mapping See the Java properties spec here: http://download.oracle.com/javase/7/docs/api/java/util/Properties.html#load%28java.io.Reader%29 @@ -191,11 +208,8 @@ simplified HOCON value when merged: Substitutions are a way of referring to other parts of the configuration tree. -The syntax is `${stringvalue}` where the `stringvalue` may be an unquoted or a -quoted string, following the usual rules. `stringvalue` may not be a value -concatenation, only a single string, so for example whitespace requires quoting. -`stringvalue` may not be a non-string value such as `true`, you would have to quote -it as `"true"`. +The syntax is `${stringvalue}` where the `stringvalue` is a path expression +(see above). Substitution processing is performed as the last parsing step, so a substitution can look forward in the configuration file and even retrieve a diff --git a/src/main/java/com/typesafe/config/ConfigException.java b/src/main/java/com/typesafe/config/ConfigException.java index 33dd56d6..ec49782d 100644 --- a/src/main/java/com/typesafe/config/ConfigException.java +++ b/src/main/java/com/typesafe/config/ConfigException.java @@ -143,6 +143,10 @@ public class ConfigException extends RuntimeException { public BadPath(String path, String message) { this(path, message, null); } + + public BadPath(ConfigOrigin origin, String message) { + super(origin, message); + } } /** diff --git a/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java b/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java index 9742efc9..67db777b 100644 --- a/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java +++ b/src/main/java/com/typesafe/config/impl/AbstractConfigObject.java @@ -49,24 +49,24 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements * Looks up the path with no transformation, type conversion, or exceptions * (just returns null if path not found). */ - protected ConfigValue peekPath(String path) { + protected ConfigValue peekPath(Path path) { return peekPath(this, path); } - protected ConfigValue peekPath(String path, SubstitutionResolver resolver, + protected ConfigValue peekPath(Path path, SubstitutionResolver resolver, int depth, boolean withFallbacks) { return peekPath(this, path, resolver, depth, withFallbacks); } - private static ConfigValue peekPath(AbstractConfigObject self, String path) { + private static ConfigValue peekPath(AbstractConfigObject self, Path path) { return peekPath(self, path, null, 0, false); } - private static ConfigValue peekPath(AbstractConfigObject self, String path, + private static ConfigValue peekPath(AbstractConfigObject self, Path path, SubstitutionResolver resolver, int depth, boolean withFallbacks) { - String key = ConfigUtil.firstElement(path); - String next = ConfigUtil.otherElements(path); + String key = path.first(); + Path next = path.remainder(); if (next == null) { ConfigValue v = self.peek(key, resolver, depth, withFallbacks); diff --git a/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java b/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java index 413abed3..74c1459f 100644 --- a/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java +++ b/src/main/java/com/typesafe/config/impl/AbstractConfigValue.java @@ -38,17 +38,6 @@ abstract class AbstractConfigValue implements ConfigValue { return other instanceof ConfigValue; } - protected static boolean equalsHandlingNull(Object a, Object b) { - if (a == null && b != null) - return false; - else if (a != null && b == null) - return false; - else if (a == b) // catches null == null plus optimizes identity case - return true; - else - return a.equals(b); - } - @Override public boolean equals(Object other) { // note that "origin" is deliberately NOT part of equality @@ -56,7 +45,7 @@ abstract class AbstractConfigValue implements ConfigValue { return canEqual(other) && (this.valueType() == ((ConfigValue) other).valueType()) - && equalsHandlingNull(this.unwrapped(), + && ConfigUtil.equalsHandlingNull(this.unwrapped(), ((ConfigValue) other).unwrapped()); } else { return false; diff --git a/src/main/java/com/typesafe/config/impl/ConfigSubstitution.java b/src/main/java/com/typesafe/config/impl/ConfigSubstitution.java index 627321fb..1166a230 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigSubstitution.java +++ b/src/main/java/com/typesafe/config/impl/ConfigSubstitution.java @@ -7,9 +7,16 @@ import com.typesafe.config.ConfigOrigin; import com.typesafe.config.ConfigValue; import com.typesafe.config.ConfigValueType; +/** + * A ConfigSubstitution represents a value with one or more substitutions in it; + * it can resolve to a value of any type, though if the substitution has more + * than one piece it always resolves to a string via value concatenation. + */ final class ConfigSubstitution extends AbstractConfigValue { - // this is a list of String and Substitution + // this is a list of String and Path where the Path + // have to be resolved to values, then if there's more + // than one piece everything is stringified and concatenated private List pieces; ConfigSubstitution(ConfigOrigin origin, List pieces) { @@ -29,27 +36,25 @@ final class ConfigSubstitution extends AbstractConfigValue { "tried to unwrap a ConfigSubstitution; need to resolve substitution first"); } + List pieces() { + return pieces; + } + // larger than anyone would ever want private static final int MAX_DEPTH = 100; private ConfigValue findInObject(AbstractConfigObject root, SubstitutionResolver resolver, /* null if we should not have refs */ - Substitution subst, int depth, + Path subst, int depth, boolean withFallbacks) { if (depth > MAX_DEPTH) { - throw new ConfigException.BadValue(origin(), subst.reference(), - "Substitution ${" + subst.reference() + throw new ConfigException.BadValue(origin(), subst.render(), + "Substitution ${" + subst.render() + "} is part of a cycle of substitutions"); } - ConfigValue result = null; - if (subst.isPath()) { - result = root.peekPath(subst.reference(), resolver, depth, + ConfigValue result = root.peekPath(subst, resolver, depth, withFallbacks); - } else { - result = root.peek(subst.reference(), resolver, depth, - withFallbacks); - } if (result instanceof ConfigSubstitution) { throw new ConfigException.BugOrBroken( @@ -63,8 +68,7 @@ final class ConfigSubstitution extends AbstractConfigValue { return result; } - private ConfigValue resolve(SubstitutionResolver resolver, - Substitution subst, + private ConfigValue resolve(SubstitutionResolver resolver, Path subst, int depth, boolean withFallbacks) { ConfigValue result = findInObject(resolver.root(), resolver, subst, depth, withFallbacks); @@ -95,7 +99,7 @@ final class ConfigSubstitution extends AbstractConfigValue { if (p instanceof String) { sb.append((String) p); } else { - ConfigValue v = resolve(resolver, (Substitution) p, + ConfigValue v = resolve(resolver, (Path) p, depth, withFallbacks); switch (v.valueType()) { case NULL: @@ -105,7 +109,7 @@ final class ConfigSubstitution extends AbstractConfigValue { case OBJECT: // cannot substitute lists and objects into strings throw new ConfigException.WrongType(v.origin(), - ((Substitution) p).reference(), + ((Path) p).render(), "not a list or object", v.valueType().name()); default: sb.append(((AbstractConfigValue) v).transformToString()); @@ -114,10 +118,10 @@ final class ConfigSubstitution extends AbstractConfigValue { } return new ConfigString(origin(), sb.toString()); } else { - if (!(pieces.get(0) instanceof Substitution)) + if (!(pieces.get(0) instanceof Path)) throw new ConfigException.BugOrBroken( "ConfigSubstitution should never contain a single String piece"); - return resolve(resolver, (Substitution) pieces.get(0), depth, + return resolve(resolver, (Path) pieces.get(0), depth, withFallbacks); } } diff --git a/src/main/java/com/typesafe/config/impl/ConfigUtil.java b/src/main/java/com/typesafe/config/impl/ConfigUtil.java index 37bbcdde..19673519 100644 --- a/src/main/java/com/typesafe/config/impl/ConfigUtil.java +++ b/src/main/java/com/typesafe/config/impl/ConfigUtil.java @@ -48,4 +48,53 @@ final class ConfigUtil { else return path.substring(0, i); } + + static boolean equalsHandlingNull(Object a, Object b) { + if (a == null && b != null) + return false; + else if (a != null && b == null) + return false; + else if (a == b) // catches null == null plus optimizes identity case + return true; + else + return a.equals(b); + } + + static String renderJsonString(String s) { + StringBuilder sb = new StringBuilder(); + sb.append('"'); + for (int i = 0; i < s.length(); ++i) { + char c = s.charAt(i); + switch (c) { + case '"': + sb.append("\\\""); + break; + case '\\': + sb.append("\\\\"); + break; + case '\n': + sb.append("\\n"); + break; + case '\b': + sb.append("\\b"); + break; + case '\f': + sb.append("\\f"); + break; + case '\r': + sb.append("\\r"); + break; + case '\t': + sb.append("\\t"); + break; + default: + if (Character.isISOControl(c)) + sb.append(String.format("\\u%04x", (int) c)); + else + sb.append(c); + } + } + sb.append('"'); + return sb.toString(); + } } diff --git a/src/main/java/com/typesafe/config/impl/Parser.java b/src/main/java/com/typesafe/config/impl/Parser.java index 1dd7caeb..45ed18ef 100644 --- a/src/main/java/com/typesafe/config/impl/Parser.java +++ b/src/main/java/com/typesafe/config/impl/Parser.java @@ -155,6 +155,92 @@ final class Parser { return t; } + static class Element { + StringBuilder sb; + // an element can be empty if it has a quoted empty string "" in it + boolean canBeEmpty; + + Element(String initial, boolean canBeEmpty) { + this.canBeEmpty = canBeEmpty; + this.sb = new StringBuilder(initial); + } + } + + private void addPathText(List buf, boolean wasQuoted, + String newText) { + int i = wasQuoted ? -1 : newText.indexOf('.'); + Element current = buf.get(buf.size() - 1); + if (i < 0) { + // add to current path element + current.sb.append(newText); + // any empty quoted string means this element can + // now be empty. + if (wasQuoted && current.sb.length() == 0) + current.canBeEmpty = true; + } else { + // "buf" plus up to the period is an element + current.sb.append(newText.substring(0, i)); + // then start a new element + buf.add(new Element("", false)); + // recurse to consume remainder of newText + addPathText(buf, false, newText.substring(i + 1)); + } + } + + private Path parsePathExpression(List expression) { + // each builder in "buf" is an element in the path. + List buf = new ArrayList(); + buf.add(new Element("", false)); + + for (Token t : expression) { + if (Tokens.isValueWithType(t, ConfigValueType.STRING)) { + AbstractConfigValue v = Tokens.getValue(t); + // this is a quoted string; so any periods + // in here don't count as path separators + String s = v.transformToString(); + + addPathText(buf, true, s); + } else { + // any periods outside of a quoted string count as + // separators + String text; + if (Tokens.isValue(t)) { + // appending a number here may add + // a period, but we _do_ count those as path + // separators, because we basically want + // "foo 3.0bar" to parse as a string even + // though there's a number in it. The fact that + // we tokenize non-string values is largely an + // implementation detail. + AbstractConfigValue v = Tokens.getValue(t); + text = v.transformToString(); + } else if (Tokens.isUnquotedText(t)) { + text = Tokens.getUnquotedText(t); + } else { + throw new ConfigException.BadPath(lineOrigin(), + "Token not allowed in path expression: " + + t); + } + + addPathText(buf, false, text); + } + } + + PathBuilder pb = new PathBuilder(); + for (Element e : buf) { + if (e.sb.length() == 0 && !e.canBeEmpty) { + throw new ConfigException.BadPath( + lineOrigin(), + buf.toString(), + "path has a leading, trailing, or two adjacent period '.' (use \"\" empty string if you want an empty element)"); + } else { + pb.appendKey(e.sb.toString()); + } + } + + return pb.result(); + } + // merge a bunch of adjacent values into one // value; change unquoted text into a string // value. @@ -184,7 +270,7 @@ final class Parser { return; } - // this will be a list of String and Substitution + // this will be a list of String and Path List minimized = new ArrayList(); // we have multiple value tokens or one unquoted text token; @@ -212,10 +298,10 @@ final class Parser { sb.setLength(0); } // now save substitution - String reference = Tokens.getSubstitution(valueToken); - SubstitutionStyle style = Tokens - .getSubstitutionStyle(valueToken); - minimized.add(new Substitution(reference, style)); + List expression = Tokens + .getSubstitutionPathExpression(valueToken); + Path path = parsePathExpression(expression); + minimized.add(path); } else { throw new ConfigException.BugOrBroken( "should not be trying to consolidate token: " @@ -276,6 +362,7 @@ final class Parser { // invoked just after the OPEN_CURLY Map values = new HashMap(); ConfigOrigin objectOrigin = lineOrigin(); + boolean afterComma = false; while (true) { Token t = nextTokenIgnoringNewline(); if (Tokens.isValueWithType(t, ConfigValueType.STRING)) { @@ -295,7 +382,12 @@ final class Parser { // our custom config language, they should be merged if the // value is an object. values.put(key, parseValue(valueToken)); + + afterComma = false; } else if (t == Tokens.CLOSE_CURLY) { + if (afterComma) { + throw parseError("expecting a field name after comma, got a close brace }"); + } break; } else { throw parseError("Expecting close brace } or a field name, got " @@ -306,6 +398,7 @@ final class Parser { break; } else if (t == Tokens.COMMA) { // continue looping + afterComma = true; } else { throw parseError("Expecting close brace } or a comma, got " + t); diff --git a/src/main/java/com/typesafe/config/impl/Path.java b/src/main/java/com/typesafe/config/impl/Path.java new file mode 100644 index 00000000..d5364f3e --- /dev/null +++ b/src/main/java/com/typesafe/config/impl/Path.java @@ -0,0 +1,98 @@ +package com.typesafe.config.impl; + +import com.typesafe.config.ConfigException; + +final class Path { + + private String first; + private Path remainder; + + Path(String first, Path remainder) { + this.first = first; + this.remainder = remainder; + } + + Path(String... elements) { + if (elements.length == 0) + throw new ConfigException.BugOrBroken("empty path"); + this.first = elements[0]; + if (elements.length > 1) { + PathBuilder pb = new PathBuilder(); + for (int i = 1; i < elements.length; ++i) { + pb.appendKey(elements[i]); + } + this.remainder = pb.result(); + } else { + this.remainder = null; + } + } + + String first() { + return first; + } + + Path remainder() { + return remainder; + } + + @Override + public boolean equals(Object other) { + if (other instanceof Path) { + Path that = (Path) other; + return this.first.equals(that.first) + && ConfigUtil.equalsHandlingNull(this.remainder, + that.remainder); + } else { + return false; + } + } + + @Override + public int hashCode() { + return 41 * (41 + first.hashCode()) + + (remainder == null ? 0 : remainder.hashCode()); + } + + // this doesn't have a very precise meaning, just to reduce + // noise from quotes in the rendered path + static boolean hasFunkyChars(String s) { + for (int i = 0; i < s.length(); ++i) { + char c = s.charAt(i); + if (Character.isLetterOrDigit(c) || c == ' ') + continue; + else + return true; + } + return false; + } + + private void appendToStringBuilder(StringBuilder sb) { + if (hasFunkyChars(first)) + sb.append(ConfigUtil.renderJsonString(first)); + else + sb.append(first); + if (remainder != null) { + sb.append("."); + remainder.appendToStringBuilder(sb); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Path("); + appendToStringBuilder(sb); + sb.append(")"); + return sb.toString(); + } + + /** + * toString() is a debugging-oriented version while this is an + * error-message-oriented human-readable one. + */ + String render() { + StringBuilder sb = new StringBuilder(); + appendToStringBuilder(sb); + return sb.toString(); + } +} diff --git a/src/main/java/com/typesafe/config/impl/PathBuilder.java b/src/main/java/com/typesafe/config/impl/PathBuilder.java new file mode 100644 index 00000000..8b0a2e61 --- /dev/null +++ b/src/main/java/com/typesafe/config/impl/PathBuilder.java @@ -0,0 +1,67 @@ +package com.typesafe.config.impl; + +import java.util.Stack; + +import com.typesafe.config.ConfigException; + +final class PathBuilder { + // the keys are kept "backward" (top of stack is end of path) + private Stack keys; + private Path result; + + PathBuilder() { + keys = new Stack(); + } + + private void checkCanAppend() { + if (result != null) + throw new ConfigException.BugOrBroken( + "Adding to PathBuilder after getting result"); + } + + void appendPath(String path) { + checkCanAppend(); + ConfigUtil.verifyPath(path); + + String next = ConfigUtil.firstElement(path); + String remainder = ConfigUtil.otherElements(path); + + while (next != null) { + keys.push(next); + if (remainder != null) { + next = ConfigUtil.firstElement(remainder); + remainder = ConfigUtil.otherElements(remainder); + } else { + next = null; + } + } + } + + void appendKey(String key) { + checkCanAppend(); + + keys.push(key); + } + + Path result() { + if (result == null) { + Path remainder = null; + while (!keys.isEmpty()) { + String key = keys.pop(); + remainder = new Path(key, remainder); + } + result = remainder; + } + return result; + } + + static Path newPath(String path) { + PathBuilder pb = new PathBuilder(); + pb.appendPath(path); + return pb.result(); + } + + static Path newKey(String key) { + return new Path(key, null); + } +} diff --git a/src/main/java/com/typesafe/config/impl/Substitution.java b/src/main/java/com/typesafe/config/impl/Substitution.java deleted file mode 100644 index 0ae20d19..00000000 --- a/src/main/java/com/typesafe/config/impl/Substitution.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.typesafe.config.impl; - - -final class Substitution { - private SubstitutionStyle style; - private String reference; - - Substitution(String reference, SubstitutionStyle style) { - this.style = style; - this.reference = reference; - } - - SubstitutionStyle style() { - return style; - } - - String reference() { - return reference; - } - - boolean isPath() { - return style == SubstitutionStyle.PATH; - } - - @Override - public boolean equals(Object other) { - if (other instanceof Substitution) { - Substitution that = (Substitution) other; - return this.reference.equals(that.reference) - && this.style == that.style; - } else { - return false; - } - } - - @Override - public int hashCode() { - return 41 * (41 + reference.hashCode()) + style.hashCode(); - } - - @Override - public String toString() { - return "Substitution(" + reference + "," + style.name() + ")"; - } -} diff --git a/src/main/java/com/typesafe/config/impl/SubstitutionStyle.java b/src/main/java/com/typesafe/config/impl/SubstitutionStyle.java deleted file mode 100644 index 4729b3cd..00000000 --- a/src/main/java/com/typesafe/config/impl/SubstitutionStyle.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.typesafe.config.impl; - -enum SubstitutionStyle { - PATH, KEY -} diff --git a/src/main/java/com/typesafe/config/impl/Tokenizer.java b/src/main/java/com/typesafe/config/impl/Tokenizer.java index 1d07d88d..8998235b 100644 --- a/src/main/java/com/typesafe/config/impl/Tokenizer.java +++ b/src/main/java/com/typesafe/config/impl/Tokenizer.java @@ -2,8 +2,10 @@ package com.typesafe.config.impl; import java.io.IOException; import java.io.Reader; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; +import java.util.List; import java.util.Queue; import com.typesafe.config.ConfigException; @@ -20,15 +22,71 @@ final class Tokenizer { private static class TokenIterator implements Iterator { + private static class WhitespaceSaver { + // has to be saved inside value concatenations + private StringBuilder whitespace; + // may need to value-concat with next value + private boolean lastTokenWasSimpleValue; + + WhitespaceSaver() { + whitespace = new StringBuilder(); + lastTokenWasSimpleValue = false; + } + + void add(int c) { + if (lastTokenWasSimpleValue) + whitespace.appendCodePoint(c); + } + + Token check(Token t, ConfigOrigin baseOrigin, int lineNumber) { + if (isSimpleValue(t)) { + return nextIsASimpleValue(baseOrigin, lineNumber); + } else { + nextIsNotASimpleValue(); + return null; + } + } + + // called if the next token is not a simple value; + // discards any whitespace we were saving between + // simple values. + private void nextIsNotASimpleValue() { + lastTokenWasSimpleValue = false; + whitespace.setLength(0); + } + + // called if the next token IS a simple value, + // so creates a whitespace token if the previous + // token also was. + private Token nextIsASimpleValue(ConfigOrigin baseOrigin, + int lineNumber) { + if (lastTokenWasSimpleValue) { + // need to save whitespace between the two so + // the parser has the option to concatenate it. + if (whitespace.length() > 0) { + Token t = Tokens.newUnquotedText( + lineOrigin(baseOrigin, lineNumber), + whitespace.toString()); + whitespace.setLength(0); // reset + return t; + } else { + // lastTokenWasSimpleValue = true still + return null; + } + } else { + lastTokenWasSimpleValue = true; + whitespace.setLength(0); + return null; + } + } + } + private ConfigOrigin origin; private Reader input; private int oneCharBuffer; private int lineNumber; private Queue tokens; - // has to be saved inside value concatenations - private StringBuilder whitespace; - // may need to value-concat with next value - private boolean lastTokenWasSimpleValue; + private WhitespaceSaver whitespaceSaver; TokenIterator(ConfigOrigin origin, Reader input) { this.origin = origin; @@ -37,8 +95,7 @@ final class Tokenizer { lineNumber = 0; tokens = new LinkedList(); tokens.add(Tokens.START); - whitespace = new StringBuilder(); - lastTokenWasSimpleValue = false; + whitespaceSaver = new WhitespaceSaver(); } @@ -76,15 +133,14 @@ final class Tokenizer { } // get next char, skipping non-newline whitespace - private int nextCharAfterWhitespace() { + private int nextCharAfterWhitespace(WhitespaceSaver saver) { for (;;) { int c = nextChar(); if (c == -1) { return -1; } else if (isWhitespaceNotNewline(c)) { - if (lastTokenWasSimpleValue) - whitespace.appendCodePoint(c); + saver.add(c); continue; } else { return c; @@ -97,11 +153,27 @@ final class Tokenizer { } private ConfigException parseError(String message, Throwable cause) { - return new ConfigException.Parse(lineOrigin(), message, cause); + return parseError(lineOrigin(), message, cause); + } + + private static ConfigException parseError(ConfigOrigin origin, + String message, + Throwable cause) { + return new ConfigException.Parse(origin, message, cause); + } + + private static ConfigException parseError(ConfigOrigin origin, + String message) { + return parseError(origin, message, null); } private ConfigOrigin lineOrigin() { - return new SimpleConfigOrigin(origin.description() + ": line " + return lineOrigin(origin, lineNumber); + } + + private static ConfigOrigin lineOrigin(ConfigOrigin baseOrigin, + int lineNumber) { + return new SimpleConfigOrigin(baseOrigin.description() + ": line " + lineNumber); } @@ -270,91 +342,49 @@ final class Tokenizer { throw parseError("'$' not followed by {"); } - String reference = null; - boolean wasQuoted = false; + WhitespaceSaver saver = new WhitespaceSaver(); + List expression = new ArrayList(); + Token t; do { - c = nextChar(); - if (c == -1) - throw parseError("End of input but substitution was still open"); + t = pullNextToken(saver); - if (c == '"') { - if (reference != null) - throw parseError("Substitution contains multiple string values"); - Token t = pullQuotedString(); - AbstractConfigValue v = Tokens.getValue(t); - reference = ((ConfigString) v).unwrapped(); - wasQuoted = true; - } else if (c == '}') { + // note that we avoid validating the allowed tokens inside + // the substitution here; we even allow nested substitutions + // in the tokenizer. The parser sorts it out. + if (t == Tokens.CLOSE_CURLY) { // end the loop, done! + break; + } else if (t == Tokens.END) { + throw parseError(origin, + "Substitution ${ was not closed with a }"); } else { - if (reference != null || notInUnquotedText.indexOf(c) >= 0 - || isWhitespace(c)) - throw parseError("Substitution contains multiple string values or invalid char: '" - + ((char) c) + "'"); - putBack(c); - Token t = pullUnquotedText(); - if (!Tokens.isUnquotedText(t)) { - throw parseError("Substitution contains non-string token, try quoting it: " - + t); - } - reference = Tokens.getUnquotedText(t); + Token whitespace = saver.check(t, origin, lineNumber); + if (whitespace != null) + expression.add(whitespace); + expression.add(t); } - } while (c != '}'); + } while (true); - SubstitutionStyle style = ((!wasQuoted) && reference.indexOf('.') >= 0) ? SubstitutionStyle.PATH - : SubstitutionStyle.KEY; - return Tokens.newSubstitution(origin, reference, style); + return Tokens.newSubstitution(origin, expression); } - // called if the next token is not a simple value; - // discards any whitespace we were saving between - // simple values. - private void nextIsNotASimpleValue() { - lastTokenWasSimpleValue = false; - whitespace.setLength(0); - } - - // called if the next token IS a simple value, - // so creates a whitespace token if the previous - // token also was. - private void nextIsASimpleValue() { - if (lastTokenWasSimpleValue) { - // need to save whitespace between the two so - // the parser has the option to concatenate it. - if (whitespace.length() > 0) { - tokens.add(Tokens.newUnquotedText(lineOrigin(), - whitespace.toString())); - whitespace.setLength(0); // reset - } - // lastTokenWasSimpleValue = true still - } else { - lastTokenWasSimpleValue = true; - whitespace.setLength(0); - } - } - - private void queueNextToken() { - int c = nextCharAfterWhitespace(); + private Token pullNextToken(WhitespaceSaver saver) { + int c = nextCharAfterWhitespace(saver); if (c == -1) { - nextIsNotASimpleValue(); - tokens.add(Tokens.END); + return Tokens.END; } else if (c == '\n') { // newline tokens have the just-ended line number - nextIsNotASimpleValue(); - tokens.add(Tokens.newLine(lineNumber)); lineNumber += 1; + return Tokens.newLine(lineNumber - 1); } else { Token t = null; - boolean tIsSimpleValue = false; switch (c) { case '"': t = pullQuotedString(); - tIsSimpleValue = true; break; case '$': t = pullSubstitution(); - tIsSimpleValue = true; break; case ':': t = Tokens.COLON; @@ -379,7 +409,6 @@ final class Tokenizer { if (t == null) { if (firstNumberChars.indexOf(c) >= 0) { t = pullNumber(c); - tIsSimpleValue = true; } else if (notInUnquotedText.indexOf(c) >= 0) { throw parseError(String .format("Character '%c' is not the start of any valid token", @@ -387,7 +416,6 @@ final class Tokenizer { } else { putBack(c); t = pullUnquotedText(); - tIsSimpleValue = true; } } @@ -395,16 +423,27 @@ final class Tokenizer { throw new ConfigException.BugOrBroken( "bug: failed to generate next token"); - if (tIsSimpleValue) { - nextIsASimpleValue(); - } else { - nextIsNotASimpleValue(); - } - - tokens.add(t); + return t; } } + private static boolean isSimpleValue(Token t) { + if (Tokens.isSubstitution(t) || Tokens.isUnquotedText(t) + || Tokens.isValue(t)) { + return true; + } else { + return false; + } + } + + private void queueNextToken() { + Token t = pullNextToken(whitespaceSaver); + Token whitespace = whitespaceSaver.check(t, origin, lineNumber); + if (whitespace != null) + tokens.add(whitespace); + tokens.add(t); + } + @Override public boolean hasNext() { return !tokens.isEmpty(); diff --git a/src/main/java/com/typesafe/config/impl/Tokens.java b/src/main/java/com/typesafe/config/impl/Tokens.java index d53bf55b..06bb0008 100644 --- a/src/main/java/com/typesafe/config/impl/Tokens.java +++ b/src/main/java/com/typesafe/config/impl/Tokens.java @@ -1,5 +1,7 @@ package com.typesafe.config.impl; +import java.util.List; + import com.typesafe.config.ConfigException; import com.typesafe.config.ConfigOrigin; import com.typesafe.config.ConfigValueType; @@ -120,35 +122,25 @@ final class Tokens { // This is not a Value, because it requires special processing static private class Substitution extends Token { private ConfigOrigin origin; - private String value; - private boolean isPath; + private List value; - Substitution(ConfigOrigin origin, String s, SubstitutionStyle style) { + Substitution(ConfigOrigin origin, List expression) { super(TokenType.SUBSTITUTION); this.origin = origin; - this.value = s; - // if the string is not quoted and contains '.' then - // it's a path rather than just a key name. - - this.isPath = style == SubstitutionStyle.PATH; + this.value = expression; } ConfigOrigin origin() { return origin; } - String value() { + List value() { return value; } - boolean isPath() { - return isPath; - } - @Override public String toString() { - return tokenType().name() + "(" + value + ",isPath=" + isPath - + ")"; + return tokenType().name() + "(" + value.toString() + ")"; } @Override @@ -159,14 +151,12 @@ final class Tokens { @Override public boolean equals(Object other) { return super.equals(other) - && ((Substitution) other).value.equals(value) - && ((Substitution) other).isPath() == this.isPath(); + && ((Substitution) other).value.equals(value); } @Override public int hashCode() { - return 41 * (41 * (41 + super.hashCode()) + value.hashCode()) - + Boolean.valueOf(isPath()).hashCode(); + return 41 * (41 + super.hashCode()) + value.hashCode(); } } @@ -226,7 +216,7 @@ final class Tokens { return token instanceof Substitution; } - static String getSubstitution(Token token) { + static List getSubstitutionPathExpression(Token token) { if (token instanceof Substitution) { return ((Substitution) token).value(); } else { @@ -244,16 +234,6 @@ final class Tokens { } } - static SubstitutionStyle getSubstitutionStyle(Token token) { - if (token instanceof Substitution) { - return ((Substitution) token).isPath() ? SubstitutionStyle.PATH - : SubstitutionStyle.KEY; - } else { - throw new ConfigException.BugOrBroken( - "tried to get substitution style from " + token); - } - } - static Token START = new Token(TokenType.START); static Token END = new Token(TokenType.END); static Token COMMA = new Token(TokenType.COMMA); @@ -271,9 +251,8 @@ final class Tokens { return new UnquotedText(origin, s); } - static Token newSubstitution(ConfigOrigin origin, String s, - SubstitutionStyle style) { - return new Substitution(origin, s, style); + static Token newSubstitution(ConfigOrigin origin, List expression) { + return new Substitution(origin, expression); } static Token newValue(AbstractConfigValue value) { diff --git a/src/test/resources/test02.conf b/src/test/resources/test02.conf new file mode 100644 index 00000000..b64c9bd7 --- /dev/null +++ b/src/test/resources/test02.conf @@ -0,0 +1,11 @@ +{ + "" : { "" : { "" : 42 } }, + "42_a" : ${""."".""}, + "42_b" : ${""""."""".""""}, + "42_c" : ${ """".""""."""" }, + "a" : { "b" : { "c" : 57 } }, + "57_a" : ${a.b.c}, + "57_b" : ${"a"."b"."c"}, + "a.b.c" : 103, + "103_a" : ${"a.b.c"} +} diff --git a/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala b/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala index 49f61f55..db7c4748 100644 --- a/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala +++ b/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala @@ -6,19 +6,24 @@ import java.io.Reader import java.io.StringReader import com.typesafe.config._ import java.util.HashMap +import scala.collection.JavaConverters._ class ConfParserTest extends TestUtils { - def parse(s: String): ConfigValue = { + def parseWithoutResolving(s: String) = { Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test conf string"), s, includer()) } - private def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = { - try { - body - } catch { - case t: Throwable => - throw new AssertionError(parserName + " parser did wrong thing on '" + s + "'", t) + def parse(s: String) = { + val tree = parseWithoutResolving(s) + + // resolve substitutions so we can test problems with that, like cycles or + // interpolating arrays into strings + tree match { + case obj: AbstractConfigObject => + SubstitutionResolver.resolveWithoutFallbacks(tree, obj) + case _ => + tree } } @@ -44,4 +49,53 @@ class ConfParserTest extends TestUtils { } } } + + private def parsePath(s: String): Path = { + val tree = parseWithoutResolving("[${" + s + "}]") + tree match { + case list: ConfigList => + list.asJavaList().get(0) match { + case subst: ConfigSubstitution => + subst.pieces().get(0) match { + case p: Path => p + } + } + } + } + + @Test + def pathParsing() { + assertEquals(path("a"), parsePath("a")) + assertEquals(path("a", "b"), parsePath("a.b")) + assertEquals(path("a.b"), parsePath("\"a.b\"")) + assertEquals(path("a."), parsePath("\"a.\"")) + assertEquals(path(".b"), parsePath("\".b\"")) + assertEquals(path("true"), parsePath("true")) + assertEquals(path("a"), parsePath(" a ")) + assertEquals(path("a ", "b"), parsePath(" a .b")) + assertEquals(path("a ", " b"), parsePath(" a . b")) + assertEquals(path("a b"), parsePath(" a b")) + assertEquals(path("a", "b.c", "d"), parsePath("a.\"b.c\".d")) + assertEquals(path("3", "14"), parsePath("3.14")) + assertEquals(path("a3", "14"), parsePath("a3.14")) + assertEquals(path(""), parsePath("\"\"")) + assertEquals(path("a", "", "b"), parsePath("a.\"\".b")) + assertEquals(path("a", ""), parsePath("a.\"\"")) + assertEquals(path("", "b"), parsePath("\"\".b")) + assertEquals(path(""), parsePath("\"\"\"\"")) + assertEquals(path("a", ""), parsePath("a.\"\"\"\"")) + assertEquals(path("", "b"), parsePath("\"\"\"\".b")) + + for (invalid <- Seq("a.", ".b", "a..b", "a${b}c", "\"\".", ".\"\"")) { + try { + intercept[ConfigException.BadPath] { + parsePath(invalid) + } + } catch { + case e => + System.err.println("failed on: " + invalid); + throw e; + } + } + } } diff --git a/src/test/scala/com/typesafe/config/impl/ConfigSubstitutionTest.scala b/src/test/scala/com/typesafe/config/impl/ConfigSubstitutionTest.scala index b01af373..3fb4d25c 100644 --- a/src/test/scala/com/typesafe/config/impl/ConfigSubstitutionTest.scala +++ b/src/test/scala/com/typesafe/config/impl/ConfigSubstitutionTest.scala @@ -38,7 +38,7 @@ class ConfigSubstitutionTest extends TestUtils { @Test def resolveTrivialKey() { - val s = subst("foo", SubstitutionStyle.KEY) + val s = subst("foo") val v = resolveWithoutFallbacks(s, simpleObject) assertEquals(intValue(42), v) } diff --git a/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index eb4754af..bd0b2070 100644 --- a/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -368,4 +368,16 @@ class ConfigTest extends TestUtils { def test01LoadWithConfigConfig() { val conf = Config.load(new ConfigConfig("test01")) } + + @Test + def test02WeirdPaths() { + val conf = Config.load("test02") + + assertEquals(42, conf.getInt("42_a")) + assertEquals(42, conf.getInt("42_b")) + assertEquals(42, conf.getInt("42_c")) + assertEquals(57, conf.getInt("57_a")) + assertEquals(57, conf.getInt("57_b")) + assertEquals(103, conf.getInt("103_a")) + } } diff --git a/src/test/scala/com/typesafe/config/impl/ConfigValueTest.scala b/src/test/scala/com/typesafe/config/impl/ConfigValueTest.scala index 939486b5..3f3da1bd 100644 --- a/src/test/scala/com/typesafe/config/impl/ConfigValueTest.scala +++ b/src/test/scala/com/typesafe/config/impl/ConfigValueTest.scala @@ -64,19 +64,6 @@ class ConfigValueTest extends TestUtils { checkNotEqualObjects(a, b) } - @Test - def substitutionEquality() { - val a = new Substitution("foo", SubstitutionStyle.KEY); - val sameAsA = new Substitution("foo", SubstitutionStyle.KEY); - val differentRef = new Substitution("bar", SubstitutionStyle.KEY); - val differentStyle = new Substitution("foo", SubstitutionStyle.PATH); - - checkEqualObjects(a, a) - checkEqualObjects(a, sameAsA) - checkNotEqualObjects(a, differentRef) - checkNotEqualObjects(a, differentStyle) - } - @Test def configSubstitutionEquality() { val a = subst("foo") diff --git a/src/test/scala/com/typesafe/config/impl/JsonTest.scala b/src/test/scala/com/typesafe/config/impl/JsonTest.scala index 0979ac3b..716377c3 100644 --- a/src/test/scala/com/typesafe/config/impl/JsonTest.scala +++ b/src/test/scala/com/typesafe/config/impl/JsonTest.scala @@ -92,15 +92,6 @@ class JsonTest extends TestUtils { // For string quoting, check behavior of escaping a random character instead of one on the list; // lift-json seems to oddly treat that as a \ literal - private def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = { - try { - body - } catch { - case t: Throwable => - throw new AssertionError(parserName + " parser did wrong thing on '" + s + "'", t) - } - } - @Test def invalidJsonThrows(): Unit = { // be sure Lift throws on the string @@ -162,4 +153,14 @@ class JsonTest extends TestUtils { } } } + + @Test + def renderingJsonStrings() { + def r(s: String) = ConfigUtil.renderJsonString(s) + assertEquals(""""abcdefg"""", r("""abcdefg""")) + assertEquals(""""\" \\ \n \b \f \r \t"""", r("\" \\ \n \b \f \r \t")) + // control characters are escaped. Remember that unicode escapes + // are weird and happen on the source file before doing other processing. + assertEquals("\"\\" + "u001f\"", r("\u001f")) + } } diff --git a/src/test/scala/com/typesafe/config/impl/PathTest.scala b/src/test/scala/com/typesafe/config/impl/PathTest.scala new file mode 100644 index 00000000..e77b70cb --- /dev/null +++ b/src/test/scala/com/typesafe/config/impl/PathTest.scala @@ -0,0 +1,41 @@ +package com.typesafe.config.impl + +import org.junit.Assert._ +import org.junit._ + +class PathTest extends TestUtils { + + @Test + def pathEquality() { + // note: foo.bar is a single key here + val a = PathBuilder.newKey("foo.bar") + val sameAsA = PathBuilder.newKey("foo.bar") + val differentKey = PathBuilder.newKey("hello") + // here foo.bar is two elements + val twoElements = PathBuilder.newPath("foo.bar") + val sameAsTwoElements = PathBuilder.newPath("foo.bar") + + checkEqualObjects(a, a) + checkEqualObjects(a, sameAsA) + checkNotEqualObjects(a, differentKey) + checkNotEqualObjects(a, twoElements) + checkEqualObjects(twoElements, sameAsTwoElements) + } + + @Test + def pathToString() { + assertEquals("Path(foo)", PathBuilder.newPath("foo").toString()) + assertEquals("Path(foo.bar)", PathBuilder.newPath("foo.bar").toString()) + assertEquals("Path(foo.\"bar*\")", PathBuilder.newPath("foo.bar*").toString()) + assertEquals("Path(\"foo.bar\")", PathBuilder.newKey("foo.bar").toString()) + } + + @Test + def pathRender() { + assertEquals("foo", PathBuilder.newPath("foo").render()) + assertEquals("foo.bar", PathBuilder.newPath("foo.bar").render()) + assertEquals("foo.\"bar*\"", PathBuilder.newPath("foo.bar*").render()) + assertEquals("\"foo.bar\"", PathBuilder.newKey("foo.bar").render()) + assertEquals("foo bar", PathBuilder.newKey("foo bar").render()) + } +} diff --git a/src/test/scala/com/typesafe/config/impl/TestUtils.scala b/src/test/scala/com/typesafe/config/impl/TestUtils.scala index 479f1e5a..64086dad 100644 --- a/src/test/scala/com/typesafe/config/impl/TestUtils.scala +++ b/src/test/scala/com/typesafe/config/impl/TestUtils.scala @@ -2,6 +2,9 @@ package com.typesafe.config.impl import org.junit.Assert._ import org.junit._ +import com.typesafe.config.ConfigOrigin +import java.io.Reader +import java.io.StringReader abstract trait TestUtils { protected def intercept[E <: Throwable: Manifest](block: => Unit): E = { @@ -126,14 +129,10 @@ abstract trait TestUtils { "[${]", // unclosed substitution "[$]", // '$' by itself "[$ ]", // '$' by itself with spaces after - """${"foo""bar"}""", // multiple strings in substitution """{ "a" : [1,2], "b" : y${a}z }""", // trying to interpolate an array in a string """{ "a" : { "c" : 2 }, "b" : y${a}z }""", // trying to interpolate an object in a string - ParseTest(false, true, "[${ foo.bar}]"), // substitution with leading spaces - ParseTest(false, true, "[${foo.bar }]"), // substitution with trailing spaces - ParseTest(false, true, "[${ \"foo.bar\"}]"), // substitution with leading spaces and quoted - ParseTest(false, true, "[${\"foo.bar\" }]"), // substitution with trailing spaces and quoted - "[${true}]", // substitution with unquoted true token + """{ "a" : ${a} }""", // simple cycle + """[ { "a" : 2, "b" : ${${a}} } ]""", // nested substitution "[ = ]", // = is not a valid token "") // empty document again, just for clean formatting of this list ;-) @@ -175,7 +174,14 @@ abstract trait TestUtils { "[ ${foo.bar} ]", "[ abc xyz ${foo.bar} qrs tuv ]", // value concatenation "[ 1, 2, 3, blah ]", - "[ ${\"foo.bar\"} ]") + "[ ${\"foo.bar\"} ]", + ParseTest(false, true, "[${ foo.bar}]"), // substitution with leading spaces + ParseTest(false, true, "[${foo.bar }]"), // substitution with trailing spaces + ParseTest(false, true, "[${ \"foo.bar\"}]"), // substitution with leading spaces and quoted + ParseTest(false, true, "[${\"foo.bar\" }]"), // substitution with trailing spaces and quoted + """${"foo""bar"}""", // multiple strings in substitution + """${foo "bar" baz}""", // multiple strings and whitespace in substitution + "[${true}]") // substitution with unquoted true token protected val invalidJson = validConfInvalidJson ++ invalidJsonInvalidConf; @@ -184,6 +190,21 @@ abstract trait TestUtils { // .conf is a superset of JSON so validJson just goes in here protected val validConf = validConfInvalidJson ++ validJson; + protected def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = { + try { + body + } catch { + case t: Throwable => + val tokens = try { + "tokens: " + tokenizeAsList(s) + } catch { + case e => + "tokenizer failed: " + e.getMessage(); + } + throw new AssertionError(parserName + " parser did wrong thing on '" + s + "', " + tokens, t) + } + } + protected def whitespaceVariations(tests: Seq[ParseTest]): Seq[ParseTest] = { val variations = List({ s: String => s }, // identity { s: String => " " + s }, @@ -216,14 +237,14 @@ abstract trait TestUtils { Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test string"), s, includer()).asInstanceOf[AbstractConfigObject] } - protected def subst(ref: String, style: SubstitutionStyle = SubstitutionStyle.PATH) = { - val pieces = java.util.Collections.singletonList[Object](new Substitution(ref, style)) + protected def subst(ref: String) = { + val pieces = java.util.Collections.singletonList[Object](PathBuilder.newPath(ref)) new ConfigSubstitution(fakeOrigin(), pieces) } - protected def substInString(ref: String, style: SubstitutionStyle = SubstitutionStyle.PATH) = { + protected def substInString(ref: String) = { import scala.collection.JavaConverters._ - val pieces = List("start<", new Substitution(ref, style), ">end") + val pieces = List("start<", PathBuilder.newPath(ref), ">end") new ConfigSubstitution(fakeOrigin(), pieces.asJava) } @@ -231,10 +252,41 @@ abstract trait TestUtils { def tokenFalse = Tokens.newBoolean(fakeOrigin(), false) def tokenNull = Tokens.newNull(fakeOrigin()) def tokenUnquoted(s: String) = Tokens.newUnquotedText(fakeOrigin(), s) - def tokenKeySubstitution(s: String) = Tokens.newSubstitution(fakeOrigin(), s, SubstitutionStyle.KEY) - def tokenPathSubstitution(s: String) = Tokens.newSubstitution(fakeOrigin(), s, SubstitutionStyle.PATH) def tokenString(s: String) = Tokens.newString(fakeOrigin(), s) def tokenDouble(d: Double) = Tokens.newDouble(fakeOrigin(), d) def tokenInt(i: Int) = Tokens.newInt(fakeOrigin(), i) def tokenLong(l: Long) = Tokens.newLong(fakeOrigin(), l) + + def tokenSubstitution(expression: Token*) = { + val l = new java.util.ArrayList[Token] + for (t <- expression) { + l.add(t); + } + Tokens.newSubstitution(fakeOrigin(), l); + } + + // quoted string substitution (no interpretation of periods) + def tokenKeySubstitution(s: String) = tokenSubstitution(tokenString(s)) + + def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = { + Tokenizer.tokenize(origin, input) + } + + def tokenize(input: Reader): java.util.Iterator[Token] = { + tokenize(new SimpleConfigOrigin("anonymous Reader"), input) + } + + def tokenize(s: String): java.util.Iterator[Token] = { + val reader = new StringReader(s) + val result = tokenize(reader) + // reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken + result + } + + def tokenizeAsList(s: String) = { + import scala.collection.JavaConverters._ + tokenize(s).asScala.toList + } + + def path(elements: String*) = new Path(elements: _*) } diff --git a/src/test/scala/com/typesafe/config/impl/TokenTest.scala b/src/test/scala/com/typesafe/config/impl/TokenTest.scala index dde57d86..f4152540 100644 --- a/src/test/scala/com/typesafe/config/impl/TokenTest.scala +++ b/src/test/scala/com/typesafe/config/impl/TokenTest.scala @@ -43,13 +43,6 @@ class TokenTest extends TestUtils { checkEqualObjects(tokenKeySubstitution("foo"), tokenKeySubstitution("foo")) checkNotEqualObjects(tokenKeySubstitution("foo"), tokenKeySubstitution("bar")) - // path subst - checkEqualObjects(tokenPathSubstitution("foo"), tokenPathSubstitution("foo")) - checkNotEqualObjects(tokenPathSubstitution("foo"), tokenPathSubstitution("bar")) - - // key and path not equal - checkNotEqualObjects(tokenKeySubstitution("foo"), tokenPathSubstitution("foo")) - // null checkEqualObjects(tokenNull, tokenNull) @@ -75,7 +68,6 @@ class TokenTest extends TestUtils { tokenUnquoted("foo").toString() tokenString("bar").toString() tokenKeySubstitution("a").toString() - tokenPathSubstitution("b").toString() Tokens.newLine(10).toString() Tokens.START.toString() Tokens.END.toString() diff --git a/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala b/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala index 4b37947b..c4fa79bf 100644 --- a/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala +++ b/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala @@ -1,41 +1,24 @@ package com.typesafe.config.impl -import org.junit.Assert._ -import org.junit._ -import net.liftweb.{ json => lift } -import java.io.Reader -import java.io.StringReader -import com.typesafe.config._ -import java.util.HashMap +import org.junit.Assert.assertEquals +import org.junit.Test + +import com.typesafe.config.ConfigException class TokenizerTest extends TestUtils { - def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = { - Tokenizer.tokenize(origin, input) - } - - def tokenize(input: Reader): java.util.Iterator[Token] = { - tokenize(new SimpleConfigOrigin("anonymous Reader"), input) - } - - def tokenize(s: String): java.util.Iterator[Token] = { - val reader = new StringReader(s) - val result = tokenize(reader) - // reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken - result - } - - def tokenizeAsList(s: String) = { - import scala.collection.JavaConverters._ - tokenize(s).asScala.toList - } - @Test def tokenizeEmptyString() { assertEquals(List(Tokens.START, Tokens.END), tokenizeAsList("")) } + @Test + def tokenizeNewlines() { + assertEquals(List(Tokens.START, Tokens.newLine(0), Tokens.newLine(1), Tokens.END), + tokenizeAsList("\n\n")) + } + @Test def tokenizeAllTypesNoSpaces() { // all token types with no spaces (not sure JSON spec wants this to work, @@ -44,7 +27,7 @@ class TokenizerTest extends TestUtils { val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"), tokenTrue, tokenDouble(3.14), tokenFalse, - tokenLong(42), tokenNull, tokenPathSubstitution("a.b"), + tokenLong(42), tokenNull, tokenSubstitution(tokenUnquoted("a.b")), tokenKeySubstitution("c.d"), Tokens.newLine(0), Tokens.END) assertEquals(expected, tokenizeAsList(""",:}{]["foo"true3.14false42null${a.b}${"c.d"}""" + "\n")) } @@ -55,7 +38,7 @@ class TokenizerTest extends TestUtils { Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"), tokenUnquoted(" "), tokenLong(42), tokenUnquoted(" "), tokenTrue, tokenUnquoted(" "), tokenDouble(3.14), tokenUnquoted(" "), tokenFalse, tokenUnquoted(" "), tokenNull, - tokenUnquoted(" "), tokenPathSubstitution("a.b"), tokenUnquoted(" "), tokenKeySubstitution("c.d"), + tokenUnquoted(" "), tokenSubstitution(tokenUnquoted("a.b")), tokenUnquoted(" "), tokenKeySubstitution("c.d"), Tokens.newLine(0), Tokens.END) assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null ${a.b} ${"c.d"} """ + "\n ")) } @@ -66,7 +49,7 @@ class TokenizerTest extends TestUtils { Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"), tokenUnquoted(" "), tokenLong(42), tokenUnquoted(" "), tokenTrue, tokenUnquoted(" "), tokenDouble(3.14), tokenUnquoted(" "), tokenFalse, tokenUnquoted(" "), tokenNull, - tokenUnquoted(" "), tokenPathSubstitution("a.b"), tokenUnquoted(" "), + tokenUnquoted(" "), tokenSubstitution(tokenUnquoted("a.b")), tokenUnquoted(" "), tokenKeySubstitution("c.d"), Tokens.newLine(0), Tokens.END) assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null ${a.b} ${"c.d"} """ + "\n "))