github-repos/config
1
0
Fork 0
mirror of https://github.com/lightbend/config.git synced 2025-01-15 23:01:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Generalize path handling in substitutions

Browse Source 
The new approach allows a mix of tokens in the substitution,
so you can refer to path elements that contain characters
that need quoting, including periods.
This commit is contained in:
Havoc Pennington 2011-11-10 13:09:45 -05:00
parent cdcde9eebc
commit e0a49c06da
23 changed files with 712 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
SPEC.md
View File

@ -156,6 +156,23 @@ Different from JSON:
already then it refers to precisely that filename and the format
is not flexible.
### Path expressions
Path expressions are used to write out a path through the object
graph. They appear in two places; in substitutions, like
`${foo.bar}`, and as the keys in objects like `{ foo.bar : 42 }`.
Path expressions work like a value concatenation, except that they
may not contain substitutions. This means that you can't nest
substitutions inside other substitutions, and you can't have
substitutions in keys.
When concatenating the path expression, any `.` characters outside quoted
strings or numbers are understood as path separators, while inside quoted
strings `.` has no special meaning. So `foo.bar."hello.world"` would be
a path with three elements, looking up key `foo`, key `bar`, then key
`hello.world`.
### Java properties mapping
See the Java properties spec here: http://download.oracle.com/javase/7/docs/api/java/util/Properties.html#load%28java.io.Reader%29
@ -191,11 +208,8 @@ simplified HOCON value when merged:
Substitutions are a way of referring to other parts of the configuration
tree.
The syntax is `${stringvalue}` where the `stringvalue` may be an unquoted or a
quoted string, following the usual rules. `stringvalue` may not be a value
concatenation, only a single string, so for example whitespace requires quoting.
`stringvalue` may not be a non-string value such as `true`, you would have to quote
it as `"true"`.
The syntax is `${stringvalue}` where the `stringvalue` is a path expression
(see above).
Substitution processing is performed as the last parsing step, so a
substitution can look forward in the configuration file and even retrieve a

4
src/main/java/com/typesafe/config/ConfigException.java
View File

@ -143,6 +143,10 @@ public class ConfigException extends RuntimeException {
public BadPath(String path, String message) {
this(path, message, null);
}
public BadPath(ConfigOrigin origin, String message) {
super(origin, message);
}
}
/**

12
src/main/java/com/typesafe/config/impl/AbstractConfigObject.java
View File

@ -49,24 +49,24 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements
* Looks up the path with no transformation, type conversion, or exceptions
* (just returns null if path not found).
*/
protected ConfigValue peekPath(String path) {
protected ConfigValue peekPath(Path path) {
return peekPath(this, path);
}
protected ConfigValue peekPath(String path, SubstitutionResolver resolver,
protected ConfigValue peekPath(Path path, SubstitutionResolver resolver,
int depth,
boolean withFallbacks) {
return peekPath(this, path, resolver, depth, withFallbacks);
}
private static ConfigValue peekPath(AbstractConfigObject self, String path) {
private static ConfigValue peekPath(AbstractConfigObject self, Path path) {
return peekPath(self, path, null, 0, false);
}
private static ConfigValue peekPath(AbstractConfigObject self, String path,
private static ConfigValue peekPath(AbstractConfigObject self, Path path,
SubstitutionResolver resolver, int depth, boolean withFallbacks) {
String key = ConfigUtil.firstElement(path);
String next = ConfigUtil.otherElements(path);
String key = path.first();
Path next = path.remainder();
if (next == null) {
ConfigValue v = self.peek(key, resolver, depth, withFallbacks);

13
src/main/java/com/typesafe/config/impl/AbstractConfigValue.java
View File

@ -38,17 +38,6 @@ abstract class AbstractConfigValue implements ConfigValue {
return other instanceof ConfigValue;
}
protected static boolean equalsHandlingNull(Object a, Object b) {
if (a == null && b != null)
return false;
else if (a != null && b == null)
return false;
else if (a == b) // catches null == null plus optimizes identity case
return true;
else
return a.equals(b);
}
@Override
public boolean equals(Object other) {
// note that "origin" is deliberately NOT part of equality
@ -56,7 +45,7 @@ abstract class AbstractConfigValue implements ConfigValue {
return canEqual(other)
&& (this.valueType() ==
((ConfigValue) other).valueType())
&& equalsHandlingNull(this.unwrapped(),
&& ConfigUtil.equalsHandlingNull(this.unwrapped(),
((ConfigValue) other).unwrapped());
} else {
return false;

38
src/main/java/com/typesafe/config/impl/ConfigSubstitution.java
View File

@ -7,9 +7,16 @@ import com.typesafe.config.ConfigOrigin;
import com.typesafe.config.ConfigValue;
import com.typesafe.config.ConfigValueType;
/**
* A ConfigSubstitution represents a value with one or more substitutions in it;
* it can resolve to a value of any type, though if the substitution has more
* than one piece it always resolves to a string via value concatenation.
*/
final class ConfigSubstitution extends AbstractConfigValue {
// this is a list of String and Substitution
// this is a list of String and Path where the Path
// have to be resolved to values, then if there's more
// than one piece everything is stringified and concatenated
private List<Object> pieces;
ConfigSubstitution(ConfigOrigin origin, List<Object> pieces) {
@ -29,27 +36,25 @@ final class ConfigSubstitution extends AbstractConfigValue {
"tried to unwrap a ConfigSubstitution; need to resolve substitution first");
}
List<Object> pieces() {
return pieces;
}
// larger than anyone would ever want
private static final int MAX_DEPTH = 100;
private ConfigValue findInObject(AbstractConfigObject root,
SubstitutionResolver resolver, /* null if we should not have refs */
Substitution subst, int depth,
Path subst, int depth,
boolean withFallbacks) {
if (depth > MAX_DEPTH) {
throw new ConfigException.BadValue(origin(), subst.reference(),
"Substitution ${" + subst.reference()
throw new ConfigException.BadValue(origin(), subst.render(),
"Substitution ${" + subst.render()
+ "} is part of a cycle of substitutions");
}
ConfigValue result = null;
if (subst.isPath()) {
result = root.peekPath(subst.reference(), resolver, depth,
ConfigValue result = root.peekPath(subst, resolver, depth,
withFallbacks);
} else {
result = root.peek(subst.reference(), resolver, depth,
withFallbacks);
}
if (result instanceof ConfigSubstitution) {
throw new ConfigException.BugOrBroken(
@ -63,8 +68,7 @@ final class ConfigSubstitution extends AbstractConfigValue {
return result;
}
private ConfigValue resolve(SubstitutionResolver resolver,
Substitution subst,
private ConfigValue resolve(SubstitutionResolver resolver, Path subst,
int depth, boolean withFallbacks) {
ConfigValue result = findInObject(resolver.root(), resolver, subst,
depth, withFallbacks);
@ -95,7 +99,7 @@ final class ConfigSubstitution extends AbstractConfigValue {
if (p instanceof String) {
sb.append((String) p);
} else {
ConfigValue v = resolve(resolver, (Substitution) p,
ConfigValue v = resolve(resolver, (Path) p,
depth, withFallbacks);
switch (v.valueType()) {
case NULL:
@ -105,7 +109,7 @@ final class ConfigSubstitution extends AbstractConfigValue {
case OBJECT:
// cannot substitute lists and objects into strings
throw new ConfigException.WrongType(v.origin(),
((Substitution) p).reference(),
((Path) p).render(),
"not a list or object", v.valueType().name());
default:
sb.append(((AbstractConfigValue) v).transformToString());
@ -114,10 +118,10 @@ final class ConfigSubstitution extends AbstractConfigValue {
}
return new ConfigString(origin(), sb.toString());
} else {
if (!(pieces.get(0) instanceof Substitution))
if (!(pieces.get(0) instanceof Path))
throw new ConfigException.BugOrBroken(
"ConfigSubstitution should never contain a single String piece");
return resolve(resolver, (Substitution) pieces.get(0), depth,
return resolve(resolver, (Path) pieces.get(0), depth,
withFallbacks);
}
}

49
src/main/java/com/typesafe/config/impl/ConfigUtil.java
View File

@ -48,4 +48,53 @@ final class ConfigUtil {
else
return path.substring(0, i);
}
static boolean equalsHandlingNull(Object a, Object b) {
if (a == null && b != null)
return false;
else if (a != null && b == null)
return false;
else if (a == b) // catches null == null plus optimizes identity case
return true;
else
return a.equals(b);
}
static String renderJsonString(String s) {
StringBuilder sb = new StringBuilder();
sb.append('"');
for (int i = 0; i < s.length(); ++i) {
char c = s.charAt(i);
switch (c) {
case '"':
sb.append("\\\"");
break;
case '\\':
sb.append("\\\\");
break;
case '\n':
sb.append("\\n");
break;
case '\b':
sb.append("\\b");
break;
case '\f':
sb.append("\\f");
break;
case '\r':
sb.append("\\r");
break;
case '\t':
sb.append("\\t");
break;
default:
if (Character.isISOControl(c))
sb.append(String.format("\\u%04x", (int) c));
else
sb.append(c);
}
}
sb.append('"');
return sb.toString();
}
}

103
src/main/java/com/typesafe/config/impl/Parser.java
View File

@ -155,6 +155,92 @@ final class Parser {
return t;
}
static class Element {
StringBuilder sb;
// an element can be empty if it has a quoted empty string "" in it
boolean canBeEmpty;
Element(String initial, boolean canBeEmpty) {
this.canBeEmpty = canBeEmpty;
this.sb = new StringBuilder(initial);
}
}
private void addPathText(List<Element> buf, boolean wasQuoted,
String newText) {
int i = wasQuoted ? -1 : newText.indexOf('.');
Element current = buf.get(buf.size() - 1);
if (i < 0) {
// add to current path element
current.sb.append(newText);
// any empty quoted string means this element can
// now be empty.
if (wasQuoted && current.sb.length() == 0)
current.canBeEmpty = true;
} else {
// "buf" plus up to the period is an element
current.sb.append(newText.substring(0, i));
// then start a new element
buf.add(new Element("", false));
// recurse to consume remainder of newText
addPathText(buf, false, newText.substring(i + 1));
}
}
private Path parsePathExpression(List<Token> expression) {
// each builder in "buf" is an element in the path.
List<Element> buf = new ArrayList<Element>();
buf.add(new Element("", false));
for (Token t : expression) {
if (Tokens.isValueWithType(t, ConfigValueType.STRING)) {
AbstractConfigValue v = Tokens.getValue(t);
// this is a quoted string; so any periods
// in here don't count as path separators
String s = v.transformToString();
addPathText(buf, true, s);
} else {
// any periods outside of a quoted string count as
// separators
String text;
if (Tokens.isValue(t)) {
// appending a number here may add
// a period, but we _do_ count those as path
// separators, because we basically want
// "foo 3.0bar" to parse as a string even
// though there's a number in it. The fact that
// we tokenize non-string values is largely an
// implementation detail.
AbstractConfigValue v = Tokens.getValue(t);
text = v.transformToString();
} else if (Tokens.isUnquotedText(t)) {
text = Tokens.getUnquotedText(t);
} else {
throw new ConfigException.BadPath(lineOrigin(),
"Token not allowed in path expression: "
+ t);
}
addPathText(buf, false, text);
}
}
PathBuilder pb = new PathBuilder();
for (Element e : buf) {
if (e.sb.length() == 0 && !e.canBeEmpty) {
throw new ConfigException.BadPath(
lineOrigin(),
buf.toString(),
"path has a leading, trailing, or two adjacent period '.' (use \"\" empty string if you want an empty element)");
} else {
pb.appendKey(e.sb.toString());
}
}
return pb.result();
}
// merge a bunch of adjacent values into one
// value; change unquoted text into a string
// value.
@ -184,7 +270,7 @@ final class Parser {
return;
}
// this will be a list of String and Substitution
// this will be a list of String and Path
List<Object> minimized = new ArrayList<Object>();
// we have multiple value tokens or one unquoted text token;
@ -212,10 +298,10 @@ final class Parser {
sb.setLength(0);
}
// now save substitution
String reference = Tokens.getSubstitution(valueToken);
SubstitutionStyle style = Tokens
.getSubstitutionStyle(valueToken);
minimized.add(new Substitution(reference, style));
List<Token> expression = Tokens
.getSubstitutionPathExpression(valueToken);
Path path = parsePathExpression(expression);
minimized.add(path);
} else {
throw new ConfigException.BugOrBroken(
"should not be trying to consolidate token: "
@ -276,6 +362,7 @@ final class Parser {
// invoked just after the OPEN_CURLY
Map<String, AbstractConfigValue> values = new HashMap<String, AbstractConfigValue>();
ConfigOrigin objectOrigin = lineOrigin();
boolean afterComma = false;
while (true) {
Token t = nextTokenIgnoringNewline();
if (Tokens.isValueWithType(t, ConfigValueType.STRING)) {
@ -295,7 +382,12 @@ final class Parser {
// our custom config language, they should be merged if the
// value is an object.
values.put(key, parseValue(valueToken));
afterComma = false;
} else if (t == Tokens.CLOSE_CURLY) {
if (afterComma) {
throw parseError("expecting a field name after comma, got a close brace }");
}
break;
} else {
throw parseError("Expecting close brace } or a field name, got "
@ -306,6 +398,7 @@ final class Parser {
break;
} else if (t == Tokens.COMMA) {
// continue looping
afterComma = true;
} else {
throw parseError("Expecting close brace } or a comma, got "
+ t);

98
src/main/java/com/typesafe/config/impl/Path.java Normal file
View File

@ -0,0 +1,98 @@
package com.typesafe.config.impl;
import com.typesafe.config.ConfigException;
final class Path {
private String first;
private Path remainder;
Path(String first, Path remainder) {
this.first = first;
this.remainder = remainder;
}
Path(String... elements) {
if (elements.length == 0)
throw new ConfigException.BugOrBroken("empty path");
this.first = elements[0];
if (elements.length > 1) {
PathBuilder pb = new PathBuilder();
for (int i = 1; i < elements.length; ++i) {
pb.appendKey(elements[i]);
}
this.remainder = pb.result();
} else {
this.remainder = null;
}
}
String first() {
return first;
}
Path remainder() {
return remainder;
}
@Override
public boolean equals(Object other) {
if (other instanceof Path) {
Path that = (Path) other;
return this.first.equals(that.first)
&& ConfigUtil.equalsHandlingNull(this.remainder,
that.remainder);
} else {
return false;
}
}
@Override
public int hashCode() {
return 41 * (41 + first.hashCode())
+ (remainder == null ? 0 : remainder.hashCode());
}
// this doesn't have a very precise meaning, just to reduce
// noise from quotes in the rendered path
static boolean hasFunkyChars(String s) {
for (int i = 0; i < s.length(); ++i) {
char c = s.charAt(i);
if (Character.isLetterOrDigit(c) || c == ' ')
continue;
else
return true;
}
return false;
}
private void appendToStringBuilder(StringBuilder sb) {
if (hasFunkyChars(first))
sb.append(ConfigUtil.renderJsonString(first));
else
sb.append(first);
if (remainder != null) {
sb.append(".");
remainder.appendToStringBuilder(sb);
}
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("Path(");
appendToStringBuilder(sb);
sb.append(")");
return sb.toString();
}
/**
* toString() is a debugging-oriented version while this is an
* error-message-oriented human-readable one.
*/
String render() {
StringBuilder sb = new StringBuilder();
appendToStringBuilder(sb);
return sb.toString();
}
}

67
src/main/java/com/typesafe/config/impl/PathBuilder.java Normal file
View File

@ -0,0 +1,67 @@
package com.typesafe.config.impl;
import java.util.Stack;
import com.typesafe.config.ConfigException;
final class PathBuilder {
// the keys are kept "backward" (top of stack is end of path)
private Stack<String> keys;
private Path result;
PathBuilder() {
keys = new Stack<String>();
}
private void checkCanAppend() {
if (result != null)
throw new ConfigException.BugOrBroken(
"Adding to PathBuilder after getting result");
}
void appendPath(String path) {
checkCanAppend();
ConfigUtil.verifyPath(path);
String next = ConfigUtil.firstElement(path);
String remainder = ConfigUtil.otherElements(path);
while (next != null) {
keys.push(next);
if (remainder != null) {
next = ConfigUtil.firstElement(remainder);
remainder = ConfigUtil.otherElements(remainder);
} else {
next = null;
}
}
}
void appendKey(String key) {
checkCanAppend();
keys.push(key);
}
Path result() {
if (result == null) {
Path remainder = null;
while (!keys.isEmpty()) {
String key = keys.pop();
remainder = new Path(key, remainder);
}
result = remainder;
}
return result;
}
static Path newPath(String path) {
PathBuilder pb = new PathBuilder();
pb.appendPath(path);
return pb.result();
}
static Path newKey(String key) {
return new Path(key, null);
}
}

45
src/main/java/com/typesafe/config/impl/Substitution.java
View File

@ -1,45 +0,0 @@
package com.typesafe.config.impl;
final class Substitution {
private SubstitutionStyle style;
private String reference;
Substitution(String reference, SubstitutionStyle style) {
this.style = style;
this.reference = reference;
}
SubstitutionStyle style() {
return style;
}
String reference() {
return reference;
}
boolean isPath() {
return style == SubstitutionStyle.PATH;
}
@Override
public boolean equals(Object other) {
if (other instanceof Substitution) {
Substitution that = (Substitution) other;
return this.reference.equals(that.reference)
&& this.style == that.style;
} else {
return false;
}
}
@Override
public int hashCode() {
return 41 * (41 + reference.hashCode()) + style.hashCode();
}
@Override
public String toString() {
return "Substitution(" + reference + "," + style.name() + ")";
}
}

5
src/main/java/com/typesafe/config/impl/SubstitutionStyle.java
View File

@ -1,5 +0,0 @@
package com.typesafe.config.impl;
enum SubstitutionStyle {
PATH, KEY
}

207
src/main/java/com/typesafe/config/impl/Tokenizer.java
View File

@ -2,8 +2,10 @@ package com.typesafe.config.impl;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import com.typesafe.config.ConfigException;
@ -20,15 +22,71 @@ final class Tokenizer {
private static class TokenIterator implements Iterator<Token> {
private static class WhitespaceSaver {
// has to be saved inside value concatenations
private StringBuilder whitespace;
// may need to value-concat with next value
private boolean lastTokenWasSimpleValue;
WhitespaceSaver() {
whitespace = new StringBuilder();
lastTokenWasSimpleValue = false;
}
void add(int c) {
if (lastTokenWasSimpleValue)
whitespace.appendCodePoint(c);
}
Token check(Token t, ConfigOrigin baseOrigin, int lineNumber) {
if (isSimpleValue(t)) {
return nextIsASimpleValue(baseOrigin, lineNumber);
} else {
nextIsNotASimpleValue();
return null;
}
}
// called if the next token is not a simple value;
// discards any whitespace we were saving between
// simple values.
private void nextIsNotASimpleValue() {
lastTokenWasSimpleValue = false;
whitespace.setLength(0);
}
// called if the next token IS a simple value,
// so creates a whitespace token if the previous
// token also was.
private Token nextIsASimpleValue(ConfigOrigin baseOrigin,
int lineNumber) {
if (lastTokenWasSimpleValue) {
// need to save whitespace between the two so
// the parser has the option to concatenate it.
if (whitespace.length() > 0) {
Token t = Tokens.newUnquotedText(
lineOrigin(baseOrigin, lineNumber),
whitespace.toString());
whitespace.setLength(0); // reset
return t;
} else {
// lastTokenWasSimpleValue = true still
return null;
}
} else {
lastTokenWasSimpleValue = true;
whitespace.setLength(0);
return null;
}
}
}
private ConfigOrigin origin;
private Reader input;
private int oneCharBuffer;
private int lineNumber;
private Queue<Token> tokens;
// has to be saved inside value concatenations
private StringBuilder whitespace;
// may need to value-concat with next value
private boolean lastTokenWasSimpleValue;
private WhitespaceSaver whitespaceSaver;
TokenIterator(ConfigOrigin origin, Reader input) {
this.origin = origin;
@ -37,8 +95,7 @@ final class Tokenizer {
lineNumber = 0;
tokens = new LinkedList<Token>();
tokens.add(Tokens.START);
whitespace = new StringBuilder();
lastTokenWasSimpleValue = false;
whitespaceSaver = new WhitespaceSaver();
}
@ -76,15 +133,14 @@ final class Tokenizer {
}
// get next char, skipping non-newline whitespace
private int nextCharAfterWhitespace() {
private int nextCharAfterWhitespace(WhitespaceSaver saver) {
for (;;) {
int c = nextChar();
if (c == -1) {
return -1;
} else if (isWhitespaceNotNewline(c)) {
if (lastTokenWasSimpleValue)
whitespace.appendCodePoint(c);
saver.add(c);
continue;
} else {
return c;
@ -97,11 +153,27 @@ final class Tokenizer {
}
private ConfigException parseError(String message, Throwable cause) {
return new ConfigException.Parse(lineOrigin(), message, cause);
return parseError(lineOrigin(), message, cause);
}
private static ConfigException parseError(ConfigOrigin origin,
String message,
Throwable cause) {
return new ConfigException.Parse(origin, message, cause);
}
private static ConfigException parseError(ConfigOrigin origin,
String message) {
return parseError(origin, message, null);
}
private ConfigOrigin lineOrigin() {
return new SimpleConfigOrigin(origin.description() + ": line "
return lineOrigin(origin, lineNumber);
}
private static ConfigOrigin lineOrigin(ConfigOrigin baseOrigin,
int lineNumber) {
return new SimpleConfigOrigin(baseOrigin.description() + ": line "
+ lineNumber);
}
@ -270,91 +342,49 @@ final class Tokenizer {
throw parseError("'$' not followed by {");
}
String reference = null;
boolean wasQuoted = false;
WhitespaceSaver saver = new WhitespaceSaver();
List<Token> expression = new ArrayList<Token>();
Token t;
do {
c = nextChar();
if (c == -1)
throw parseError("End of input but substitution was still open");
t = pullNextToken(saver);
if (c == '"') {
if (reference != null)
throw parseError("Substitution contains multiple string values");
Token t = pullQuotedString();
AbstractConfigValue v = Tokens.getValue(t);
reference = ((ConfigString) v).unwrapped();
wasQuoted = true;
} else if (c == '}') {
// note that we avoid validating the allowed tokens inside
// the substitution here; we even allow nested substitutions
// in the tokenizer. The parser sorts it out.
if (t == Tokens.CLOSE_CURLY) {
// end the loop, done!
break;
} else if (t == Tokens.END) {
throw parseError(origin,
"Substitution ${ was not closed with a }");
} else {
if (reference != null || notInUnquotedText.indexOf(c) >= 0
|| isWhitespace(c))
throw parseError("Substitution contains multiple string values or invalid char: '"
+ ((char) c) + "'");
putBack(c);
Token t = pullUnquotedText();
if (!Tokens.isUnquotedText(t)) {
throw parseError("Substitution contains non-string token, try quoting it: "
+ t);
}
reference = Tokens.getUnquotedText(t);
Token whitespace = saver.check(t, origin, lineNumber);
if (whitespace != null)
expression.add(whitespace);
expression.add(t);
}
} while (c != '}');
} while (true);
SubstitutionStyle style = ((!wasQuoted) && reference.indexOf('.') >= 0) ? SubstitutionStyle.PATH
: SubstitutionStyle.KEY;
return Tokens.newSubstitution(origin, reference, style);
return Tokens.newSubstitution(origin, expression);
}
// called if the next token is not a simple value;
// discards any whitespace we were saving between
// simple values.
private void nextIsNotASimpleValue() {
lastTokenWasSimpleValue = false;
whitespace.setLength(0);
}
// called if the next token IS a simple value,
// so creates a whitespace token if the previous
// token also was.
private void nextIsASimpleValue() {
if (lastTokenWasSimpleValue) {
// need to save whitespace between the two so
// the parser has the option to concatenate it.
if (whitespace.length() > 0) {
tokens.add(Tokens.newUnquotedText(lineOrigin(),
whitespace.toString()));
whitespace.setLength(0); // reset
}
// lastTokenWasSimpleValue = true still
} else {
lastTokenWasSimpleValue = true;
whitespace.setLength(0);
}
}
private void queueNextToken() {
int c = nextCharAfterWhitespace();
private Token pullNextToken(WhitespaceSaver saver) {
int c = nextCharAfterWhitespace(saver);
if (c == -1) {
nextIsNotASimpleValue();
tokens.add(Tokens.END);
return Tokens.END;
} else if (c == '\n') {
// newline tokens have the just-ended line number
nextIsNotASimpleValue();
tokens.add(Tokens.newLine(lineNumber));
lineNumber += 1;
return Tokens.newLine(lineNumber - 1);
} else {
Token t = null;
boolean tIsSimpleValue = false;
switch (c) {
case '"':
t = pullQuotedString();
tIsSimpleValue = true;
break;
case '$':
t = pullSubstitution();
tIsSimpleValue = true;
break;
case ':':
t = Tokens.COLON;
@ -379,7 +409,6 @@ final class Tokenizer {
if (t == null) {
if (firstNumberChars.indexOf(c) >= 0) {
t = pullNumber(c);
tIsSimpleValue = true;
} else if (notInUnquotedText.indexOf(c) >= 0) {
throw parseError(String
.format("Character '%c' is not the start of any valid token",
@ -387,7 +416,6 @@ final class Tokenizer {
} else {
putBack(c);
t = pullUnquotedText();
tIsSimpleValue = true;
}
}
@ -395,16 +423,27 @@ final class Tokenizer {
throw new ConfigException.BugOrBroken(
"bug: failed to generate next token");
if (tIsSimpleValue) {
nextIsASimpleValue();
} else {
nextIsNotASimpleValue();
}
tokens.add(t);
return t;
}
}
private static boolean isSimpleValue(Token t) {
if (Tokens.isSubstitution(t) || Tokens.isUnquotedText(t)
|| Tokens.isValue(t)) {
return true;
} else {
return false;
}
}
private void queueNextToken() {
Token t = pullNextToken(whitespaceSaver);
Token whitespace = whitespaceSaver.check(t, origin, lineNumber);
if (whitespace != null)
tokens.add(whitespace);
tokens.add(t);
}
@Override
public boolean hasNext() {
return !tokens.isEmpty();

45
src/main/java/com/typesafe/config/impl/Tokens.java
View File

@ -1,5 +1,7 @@
package com.typesafe.config.impl;
import java.util.List;
import com.typesafe.config.ConfigException;
import com.typesafe.config.ConfigOrigin;
import com.typesafe.config.ConfigValueType;
@ -120,35 +122,25 @@ final class Tokens {
// This is not a Value, because it requires special processing
static private class Substitution extends Token {
private ConfigOrigin origin;
private String value;
private boolean isPath;
private List<Token> value;
Substitution(ConfigOrigin origin, String s, SubstitutionStyle style) {
Substitution(ConfigOrigin origin, List<Token> expression) {
super(TokenType.SUBSTITUTION);
this.origin = origin;
this.value = s;
// if the string is not quoted and contains '.' then
// it's a path rather than just a key name.
this.isPath = style == SubstitutionStyle.PATH;
this.value = expression;
}
ConfigOrigin origin() {
return origin;
}
String value() {
List<Token> value() {
return value;
}
boolean isPath() {
return isPath;
}
@Override
public String toString() {
return tokenType().name() + "(" + value + ",isPath=" + isPath
+ ")";
return tokenType().name() + "(" + value.toString() + ")";
}
@Override
@ -159,14 +151,12 @@ final class Tokens {
@Override
public boolean equals(Object other) {
return super.equals(other)
&& ((Substitution) other).value.equals(value)
&& ((Substitution) other).isPath() == this.isPath();
&& ((Substitution) other).value.equals(value);
}
@Override
public int hashCode() {
return 41 * (41 * (41 + super.hashCode()) + value.hashCode())
+ Boolean.valueOf(isPath()).hashCode();
return 41 * (41 + super.hashCode()) + value.hashCode();
}
}
@ -226,7 +216,7 @@ final class Tokens {
return token instanceof Substitution;
}
static String getSubstitution(Token token) {
static List<Token> getSubstitutionPathExpression(Token token) {
if (token instanceof Substitution) {
return ((Substitution) token).value();
} else {
@ -244,16 +234,6 @@ final class Tokens {
}
}
static SubstitutionStyle getSubstitutionStyle(Token token) {
if (token instanceof Substitution) {
return ((Substitution) token).isPath() ? SubstitutionStyle.PATH
: SubstitutionStyle.KEY;
} else {
throw new ConfigException.BugOrBroken(
"tried to get substitution style from " + token);
}
}
static Token START = new Token(TokenType.START);
static Token END = new Token(TokenType.END);
static Token COMMA = new Token(TokenType.COMMA);
@ -271,9 +251,8 @@ final class Tokens {
return new UnquotedText(origin, s);
}
static Token newSubstitution(ConfigOrigin origin, String s,
SubstitutionStyle style) {
return new Substitution(origin, s, style);
static Token newSubstitution(ConfigOrigin origin, List<Token> expression) {
return new Substitution(origin, expression);
}
static Token newValue(AbstractConfigValue value) {

11
src/test/resources/test02.conf Normal file
View File

@ -0,0 +1,11 @@
{
"" : { "" : { "" : 42 } },
"42_a" : ${""."".""},
"42_b" : ${""""."""".""""},
"42_c" : ${ """".""""."""" },
"a" : { "b" : { "c" : 57 } },
"57_a" : ${a.b.c},
"57_b" : ${"a"."b"."c"},
"a.b.c" : 103,
"103_a" : ${"a.b.c"}
}

68
src/test/scala/com/typesafe/config/impl/ConfParserTest.scala
View File

@ -6,19 +6,24 @@ import java.io.Reader
import java.io.StringReader
import com.typesafe.config._
import java.util.HashMap
import scala.collection.JavaConverters._
class ConfParserTest extends TestUtils {
def parse(s: String): ConfigValue = {
def parseWithoutResolving(s: String) = {
Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test conf string"), s, includer())
}
private def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = {
try {
body
} catch {
case t: Throwable =>
throw new AssertionError(parserName + " parser did wrong thing on '" + s + "'", t)
def parse(s: String) = {
val tree = parseWithoutResolving(s)
// resolve substitutions so we can test problems with that, like cycles or
// interpolating arrays into strings
tree match {
case obj: AbstractConfigObject =>
SubstitutionResolver.resolveWithoutFallbacks(tree, obj)
case _ =>
tree
}
}
@ -44,4 +49,53 @@ class ConfParserTest extends TestUtils {
}
}
}
private def parsePath(s: String): Path = {
val tree = parseWithoutResolving("[${" + s + "}]")
tree match {
case list: ConfigList =>
list.asJavaList().get(0) match {
case subst: ConfigSubstitution =>
subst.pieces().get(0) match {
case p: Path => p
}
}
}
}
@Test
def pathParsing() {
assertEquals(path("a"), parsePath("a"))
assertEquals(path("a", "b"), parsePath("a.b"))
assertEquals(path("a.b"), parsePath("\"a.b\""))
assertEquals(path("a."), parsePath("\"a.\""))
assertEquals(path(".b"), parsePath("\".b\""))
assertEquals(path("true"), parsePath("true"))
assertEquals(path("a"), parsePath(" a "))
assertEquals(path("a ", "b"), parsePath(" a .b"))
assertEquals(path("a ", " b"), parsePath(" a . b"))
assertEquals(path("a b"), parsePath(" a b"))
assertEquals(path("a", "b.c", "d"), parsePath("a.\"b.c\".d"))
assertEquals(path("3", "14"), parsePath("3.14"))
assertEquals(path("a3", "14"), parsePath("a3.14"))
assertEquals(path(""), parsePath("\"\""))
assertEquals(path("a", "", "b"), parsePath("a.\"\".b"))
assertEquals(path("a", ""), parsePath("a.\"\""))
assertEquals(path("", "b"), parsePath("\"\".b"))
assertEquals(path(""), parsePath("\"\"\"\""))
assertEquals(path("a", ""), parsePath("a.\"\"\"\""))
assertEquals(path("", "b"), parsePath("\"\"\"\".b"))
for (invalid <- Seq("a.", ".b", "a..b", "a${b}c", "\"\".", ".\"\"")) {
try {
intercept[ConfigException.BadPath] {
parsePath(invalid)
}
} catch {
case e =>
System.err.println("failed on: " + invalid);
throw e;
}
}
}
}

2
src/test/scala/com/typesafe/config/impl/ConfigSubstitutionTest.scala
View File

@ -38,7 +38,7 @@ class ConfigSubstitutionTest extends TestUtils {
@Test
def resolveTrivialKey() {
val s = subst("foo", SubstitutionStyle.KEY)
val s = subst("foo")
val v = resolveWithoutFallbacks(s, simpleObject)
assertEquals(intValue(42), v)
}

12
src/test/scala/com/typesafe/config/impl/ConfigTest.scala
View File

@ -368,4 +368,16 @@ class ConfigTest extends TestUtils {
def test01LoadWithConfigConfig() {
val conf = Config.load(new ConfigConfig("test01"))
}
@Test
def test02WeirdPaths() {
val conf = Config.load("test02")
assertEquals(42, conf.getInt("42_a"))
assertEquals(42, conf.getInt("42_b"))
assertEquals(42, conf.getInt("42_c"))
assertEquals(57, conf.getInt("57_a"))
assertEquals(57, conf.getInt("57_b"))
assertEquals(103, conf.getInt("103_a"))
}
}

13
src/test/scala/com/typesafe/config/impl/ConfigValueTest.scala
View File

@ -64,19 +64,6 @@ class ConfigValueTest extends TestUtils {
checkNotEqualObjects(a, b)
}
@Test
def substitutionEquality() {
val a = new Substitution("foo", SubstitutionStyle.KEY);
val sameAsA = new Substitution("foo", SubstitutionStyle.KEY);
val differentRef = new Substitution("bar", SubstitutionStyle.KEY);
val differentStyle = new Substitution("foo", SubstitutionStyle.PATH);
checkEqualObjects(a, a)
checkEqualObjects(a, sameAsA)
checkNotEqualObjects(a, differentRef)
checkNotEqualObjects(a, differentStyle)
}
@Test
def configSubstitutionEquality() {
val a = subst("foo")

19
src/test/scala/com/typesafe/config/impl/JsonTest.scala
View File

@ -92,15 +92,6 @@ class JsonTest extends TestUtils {
// For string quoting, check behavior of escaping a random character instead of one on the list;
// lift-json seems to oddly treat that as a \ literal
private def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = {
try {
body
} catch {
case t: Throwable =>
throw new AssertionError(parserName + " parser did wrong thing on '" + s + "'", t)
}
}
@Test
def invalidJsonThrows(): Unit = {
// be sure Lift throws on the string
@ -162,4 +153,14 @@ class JsonTest extends TestUtils {
}
}
}
@Test
def renderingJsonStrings() {
def r(s: String) = ConfigUtil.renderJsonString(s)
assertEquals(""""abcdefg"""", r("""abcdefg"""))
assertEquals(""""\" \\ \n \b \f \r \t"""", r("\" \\ \n \b \f \r \t"))
// control characters are escaped. Remember that unicode escapes
// are weird and happen on the source file before doing other processing.
assertEquals("\"\\" + "u001f\"", r("\u001f"))
}
}

41
src/test/scala/com/typesafe/config/impl/PathTest.scala Normal file
View File

@ -0,0 +1,41 @@
package com.typesafe.config.impl
import org.junit.Assert._
import org.junit._
class PathTest extends TestUtils {
@Test
def pathEquality() {
// note: foo.bar is a single key here
val a = PathBuilder.newKey("foo.bar")
val sameAsA = PathBuilder.newKey("foo.bar")
val differentKey = PathBuilder.newKey("hello")
// here foo.bar is two elements
val twoElements = PathBuilder.newPath("foo.bar")
val sameAsTwoElements = PathBuilder.newPath("foo.bar")
checkEqualObjects(a, a)
checkEqualObjects(a, sameAsA)
checkNotEqualObjects(a, differentKey)
checkNotEqualObjects(a, twoElements)
checkEqualObjects(twoElements, sameAsTwoElements)
}
@Test
def pathToString() {
assertEquals("Path(foo)", PathBuilder.newPath("foo").toString())
assertEquals("Path(foo.bar)", PathBuilder.newPath("foo.bar").toString())
assertEquals("Path(foo.\"bar*\")", PathBuilder.newPath("foo.bar*").toString())
assertEquals("Path(\"foo.bar\")", PathBuilder.newKey("foo.bar").toString())
}
@Test
def pathRender() {
assertEquals("foo", PathBuilder.newPath("foo").render())
assertEquals("foo.bar", PathBuilder.newPath("foo.bar").render())
assertEquals("foo.\"bar*\"", PathBuilder.newPath("foo.bar*").render())
assertEquals("\"foo.bar\"", PathBuilder.newKey("foo.bar").render())
assertEquals("foo bar", PathBuilder.newKey("foo bar").render())
}
}

78
src/test/scala/com/typesafe/config/impl/TestUtils.scala
View File

@ -2,6 +2,9 @@ package com.typesafe.config.impl
import org.junit.Assert._
import org.junit._
import com.typesafe.config.ConfigOrigin
import java.io.Reader
import java.io.StringReader
abstract trait TestUtils {
protected def intercept[E <: Throwable: Manifest](block: => Unit): E = {
@ -126,14 +129,10 @@ abstract trait TestUtils {
"[${]", // unclosed substitution
"[$]", // '$' by itself
"[$ ]", // '$' by itself with spaces after
"""${"foo""bar"}""", // multiple strings in substitution
"""{ "a" : [1,2], "b" : y${a}z }""", // trying to interpolate an array in a string
"""{ "a" : { "c" : 2 }, "b" : y${a}z }""", // trying to interpolate an object in a string
ParseTest(false, true, "[${ foo.bar}]"), // substitution with leading spaces
ParseTest(false, true, "[${foo.bar }]"), // substitution with trailing spaces
ParseTest(false, true, "[${ \"foo.bar\"}]"), // substitution with leading spaces and quoted
ParseTest(false, true, "[${\"foo.bar\" }]"), // substitution with trailing spaces and quoted
"[${true}]", // substitution with unquoted true token
"""{ "a" : ${a} }""", // simple cycle
"""[ { "a" : 2, "b" : ${${a}} } ]""", // nested substitution
"[ = ]", // = is not a valid token
"") // empty document again, just for clean formatting of this list ;-)
@ -175,7 +174,14 @@ abstract trait TestUtils {
"[ ${foo.bar} ]",
"[ abc xyz ${foo.bar} qrs tuv ]", // value concatenation
"[ 1, 2, 3, blah ]",
"[ ${\"foo.bar\"} ]")
"[ ${\"foo.bar\"} ]",
ParseTest(false, true, "[${ foo.bar}]"), // substitution with leading spaces
ParseTest(false, true, "[${foo.bar }]"), // substitution with trailing spaces
ParseTest(false, true, "[${ \"foo.bar\"}]"), // substitution with leading spaces and quoted
ParseTest(false, true, "[${\"foo.bar\" }]"), // substitution with trailing spaces and quoted
"""${"foo""bar"}""", // multiple strings in substitution
"""${foo "bar" baz}""", // multiple strings and whitespace in substitution
"[${true}]") // substitution with unquoted true token
protected val invalidJson = validConfInvalidJson ++ invalidJsonInvalidConf;
@ -184,6 +190,21 @@ abstract trait TestUtils {
// .conf is a superset of JSON so validJson just goes in here
protected val validConf = validConfInvalidJson ++ validJson;
protected def addOffendingJsonToException[R](parserName: String, s: String)(body: => R) = {
try {
body
} catch {
case t: Throwable =>
val tokens = try {
"tokens: " + tokenizeAsList(s)
} catch {
case e =>
"tokenizer failed: " + e.getMessage();
}
throw new AssertionError(parserName + " parser did wrong thing on '" + s + "', " + tokens, t)
}
}
protected def whitespaceVariations(tests: Seq[ParseTest]): Seq[ParseTest] = {
val variations = List({ s: String => s }, // identity
{ s: String => " " + s },
@ -216,14 +237,14 @@ abstract trait TestUtils {
Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test string"), s, includer()).asInstanceOf[AbstractConfigObject]
}
protected def subst(ref: String, style: SubstitutionStyle = SubstitutionStyle.PATH) = {
val pieces = java.util.Collections.singletonList[Object](new Substitution(ref, style))
protected def subst(ref: String) = {
val pieces = java.util.Collections.singletonList[Object](PathBuilder.newPath(ref))
new ConfigSubstitution(fakeOrigin(), pieces)
}
protected def substInString(ref: String, style: SubstitutionStyle = SubstitutionStyle.PATH) = {
protected def substInString(ref: String) = {
import scala.collection.JavaConverters._
val pieces = List("start<", new Substitution(ref, style), ">end")
val pieces = List("start<", PathBuilder.newPath(ref), ">end")
new ConfigSubstitution(fakeOrigin(), pieces.asJava)
}
@ -231,10 +252,41 @@ abstract trait TestUtils {
def tokenFalse = Tokens.newBoolean(fakeOrigin(), false)
def tokenNull = Tokens.newNull(fakeOrigin())
def tokenUnquoted(s: String) = Tokens.newUnquotedText(fakeOrigin(), s)
def tokenKeySubstitution(s: String) = Tokens.newSubstitution(fakeOrigin(), s, SubstitutionStyle.KEY)
def tokenPathSubstitution(s: String) = Tokens.newSubstitution(fakeOrigin(), s, SubstitutionStyle.PATH)
def tokenString(s: String) = Tokens.newString(fakeOrigin(), s)
def tokenDouble(d: Double) = Tokens.newDouble(fakeOrigin(), d)
def tokenInt(i: Int) = Tokens.newInt(fakeOrigin(), i)
def tokenLong(l: Long) = Tokens.newLong(fakeOrigin(), l)
def tokenSubstitution(expression: Token*) = {
val l = new java.util.ArrayList[Token]
for (t <- expression) {
l.add(t);
}
Tokens.newSubstitution(fakeOrigin(), l);
}
// quoted string substitution (no interpretation of periods)
def tokenKeySubstitution(s: String) = tokenSubstitution(tokenString(s))
def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = {
Tokenizer.tokenize(origin, input)
}
def tokenize(input: Reader): java.util.Iterator[Token] = {
tokenize(new SimpleConfigOrigin("anonymous Reader"), input)
}
def tokenize(s: String): java.util.Iterator[Token] = {
val reader = new StringReader(s)
val result = tokenize(reader)
// reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken
result
}
def tokenizeAsList(s: String) = {
import scala.collection.JavaConverters._
tokenize(s).asScala.toList
}
def path(elements: String*) = new Path(elements: _*)
}

8
src/test/scala/com/typesafe/config/impl/TokenTest.scala
View File

@ -43,13 +43,6 @@ class TokenTest extends TestUtils {
checkEqualObjects(tokenKeySubstitution("foo"), tokenKeySubstitution("foo"))
checkNotEqualObjects(tokenKeySubstitution("foo"), tokenKeySubstitution("bar"))
// path subst
checkEqualObjects(tokenPathSubstitution("foo"), tokenPathSubstitution("foo"))
checkNotEqualObjects(tokenPathSubstitution("foo"), tokenPathSubstitution("bar"))
// key and path not equal
checkNotEqualObjects(tokenKeySubstitution("foo"), tokenPathSubstitution("foo"))
// null
checkEqualObjects(tokenNull, tokenNull)
@ -75,7 +68,6 @@ class TokenTest extends TestUtils {
tokenUnquoted("foo").toString()
tokenString("bar").toString()
tokenKeySubstitution("a").toString()
tokenPathSubstitution("b").toString()
Tokens.newLine(10).toString()
Tokens.START.toString()
Tokens.END.toString()

43
src/test/scala/com/typesafe/config/impl/TokenizerTest.scala
View File

@ -1,41 +1,24 @@
package com.typesafe.config.impl
import org.junit.Assert._
import org.junit._
import net.liftweb.{ json => lift }
import java.io.Reader
import java.io.StringReader
import com.typesafe.config._
import java.util.HashMap
import org.junit.Assert.assertEquals
import org.junit.Test
import com.typesafe.config.ConfigException
class TokenizerTest extends TestUtils {
def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = {
Tokenizer.tokenize(origin, input)
}
def tokenize(input: Reader): java.util.Iterator[Token] = {
tokenize(new SimpleConfigOrigin("anonymous Reader"), input)
}
def tokenize(s: String): java.util.Iterator[Token] = {
val reader = new StringReader(s)
val result = tokenize(reader)
// reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken
result
}
def tokenizeAsList(s: String) = {
import scala.collection.JavaConverters._
tokenize(s).asScala.toList
}
@Test
def tokenizeEmptyString() {
assertEquals(List(Tokens.START, Tokens.END),
tokenizeAsList(""))
}
@Test
def tokenizeNewlines() {
assertEquals(List(Tokens.START, Tokens.newLine(0), Tokens.newLine(1), Tokens.END),
tokenizeAsList("\n\n"))
}
@Test
def tokenizeAllTypesNoSpaces() {
// all token types with no spaces (not sure JSON spec wants this to work,
@ -44,7 +27,7 @@ class TokenizerTest extends TestUtils {
val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"),
tokenTrue, tokenDouble(3.14), tokenFalse,
tokenLong(42), tokenNull, tokenPathSubstitution("a.b"),
tokenLong(42), tokenNull, tokenSubstitution(tokenUnquoted("a.b")),
tokenKeySubstitution("c.d"), Tokens.newLine(0), Tokens.END)
assertEquals(expected, tokenizeAsList(""",:}{]["foo"true3.14false42null${a.b}${"c.d"}""" + "\n"))
}
@ -55,7 +38,7 @@ class TokenizerTest extends TestUtils {
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"),
tokenUnquoted(" "), tokenLong(42), tokenUnquoted(" "), tokenTrue, tokenUnquoted(" "),
tokenDouble(3.14), tokenUnquoted(" "), tokenFalse, tokenUnquoted(" "), tokenNull,
tokenUnquoted(" "), tokenPathSubstitution("a.b"), tokenUnquoted(" "), tokenKeySubstitution("c.d"),
tokenUnquoted(" "), tokenSubstitution(tokenUnquoted("a.b")), tokenUnquoted(" "), tokenKeySubstitution("c.d"),
Tokens.newLine(0), Tokens.END)
assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null ${a.b} ${"c.d"} """ + "\n "))
}
@ -66,7 +49,7 @@ class TokenizerTest extends TestUtils {
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, tokenString("foo"),
tokenUnquoted(" "), tokenLong(42), tokenUnquoted(" "), tokenTrue, tokenUnquoted(" "),
tokenDouble(3.14), tokenUnquoted(" "), tokenFalse, tokenUnquoted(" "), tokenNull,
tokenUnquoted(" "), tokenPathSubstitution("a.b"), tokenUnquoted(" "),
tokenUnquoted(" "), tokenSubstitution(tokenUnquoted("a.b")), tokenUnquoted(" "),
tokenKeySubstitution("c.d"),
Tokens.newLine(0), Tokens.END)
assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null ${a.b} ${"c.d"} """ + "\n "))