Refactor Parser to parse ConfigNodes

Rename the Parser class to ConfigParser. Refactor ConfigParser
to parse ConfigNodes rather than Tokens.

config/src/main/java/com/typesafe/config/impl/ConfigDocumentParser.java

@@ -11,7 +11,7 @@ import com.typesafe.config.ConfigSyntax;
 import com.typesafe.config.ConfigValueType;
 
 final class ConfigDocumentParser {
-    static ConfigNodeComplexValue parse(Iterator<Token> tokens, ConfigParseOptions options) {
+    static ConfigNodeRoot parse(Iterator<Token> tokens, ConfigParseOptions options) {
         ParseContext context = new ParseContext(options.getSyntax(), tokens);
         return context.parse();
     }
@@ -74,11 +74,13 @@ final class ConfigDocumentParser {
         private Token nextTokenIgnoringWhitespace(Collection<AbstractConfigNode> nodes) {
             while (true) {
                 Token t = nextToken();
-                if (Tokens.isIgnoredWhitespace(t) || Tokens.isComment(t) || Tokens.isNewline(t) || isUnquotedWhitespace(t)) {
+                if (Tokens.isIgnoredWhitespace(t) || Tokens.isNewline(t) || isUnquotedWhitespace(t)) {
                     nodes.add(new ConfigNodeSingleToken(t));
                     if (Tokens.isNewline(t)) {
                         lineNumber = t.lineNumber() + 1;
                     }
+                } else if (Tokens.isComment(t)) {
+                    nodes.add(new ConfigNodeComment(t));
                 } else {
                     int newNumber = t.lineNumber();
                     if (newNumber >= 0)
@@ -112,8 +114,12 @@ final class ConfigDocumentParser {
                 boolean sawSeparatorOrNewline = false;
                 Token t = nextToken();
                 while (true) {
-                    if (Tokens.isIgnoredWhitespace(t) || isUnquotedWhitespace(t) || Tokens.isComment(t)) {
+                    if (Tokens.isIgnoredWhitespace(t) || isUnquotedWhitespace(t)) {
                         //do nothing
+                    } else if (Tokens.isComment(t)) {
+                        nodes.add(new ConfigNodeComment(t));
+                        t = nextToken();
+                        continue;
                     } else if (Tokens.isNewline(t)) {
                         sawSeparatorOrNewline = true;
                         lineNumber++;
@@ -344,7 +350,7 @@ final class ConfigDocumentParser {
             }
         }
 
-        private void parseInclude(ArrayList<AbstractConfigNode> children) {
+        private ConfigNodeInclude parseInclude(ArrayList<AbstractConfigNode> children) {
             Token t = nextTokenIgnoringWhitespace(children);
 
             // we either have a quoted string or the "file()" syntax
@@ -374,7 +380,7 @@ final class ConfigDocumentParser {
                     throw parseError("expecting a quoted string inside file(), classpath(), or url(), rather than: "
                             + t);
                 }
-                children.add(new ConfigNodeSingleToken(t));
+                children.add(new ConfigNodeSimpleValue(t));
                 // skip space after string, inside parens
                 t = nextTokenIgnoringWhitespace(children);
 
@@ -389,6 +395,7 @@ final class ConfigDocumentParser {
             } else {
                 throw parseError("include keyword is not followed by a quoted string, but by: " + t);
             }
+            return new ConfigNodeInclude(children);
         }
 
         private ConfigNodeComplexValue parseObject(boolean hadOpenCurly) {
@@ -418,9 +425,9 @@ final class ConfigDocumentParser {
                     putBack(t);
                     break;
                 } else if (flavor != ConfigSyntax.JSON && isIncludeKeyword(t)) {
-                    objectNodes.add(new ConfigNodeSingleToken(t));
+                    ArrayList<AbstractConfigNode> includeNodes = new ArrayList<AbstractConfigNode>();
-                    parseInclude(objectNodes);
+                    includeNodes.add(new ConfigNodeSingleToken(t));
-
+                    objectNodes.add(parseInclude(includeNodes));
                     afterComma = false;
                 } else {
                     keyValueNodes = new ArrayList<AbstractConfigNode>();
@@ -602,7 +609,7 @@ final class ConfigDocumentParser {
             }
         }
 
-        ConfigNodeComplexValue parse() {
+        ConfigNodeRoot parse() {
             ArrayList<AbstractConfigNode> children = new ArrayList<AbstractConfigNode>();
             Token t = nextToken();
             if (t == Tokens.START) {
@@ -614,6 +621,7 @@ final class ConfigDocumentParser {
 
             t = nextTokenIgnoringWhitespace(children);
             AbstractConfigNode result = null;
+            boolean missingCurly = false;
             if (t == Tokens.OPEN_CURLY || t == Tokens.OPEN_SQUARE) {
                 result = parseValue(t);
             } else {
@@ -629,18 +637,26 @@ final class ConfigDocumentParser {
                     // this token should be the first field's key, or part
                     // of it, so put it back.
                     putBack(t);
+                    missingCurly = true;
                     result = parseObject(false);
                 }
             }
             // Need to pull the children out of the resulting node so we can keep leading
-            // and trailing whitespace
+            // and trailing whitespace if this was a no-brace object. Otherwise, we need to add
-            children.addAll(((ConfigNodeComplexValue)result).children());
+            // the result into the list of children.
+            if (result instanceof ConfigNodeObject && missingCurly) {
+                children.addAll(((ConfigNodeComplexValue) result).children());
+            } else {
+                children.add(result);
+            }
             t = nextTokenIgnoringWhitespace(children);
             if (t == Tokens.END) {
-                if (result instanceof ConfigNodeArray) {
+                if (missingCurly) {
-                    return new ConfigNodeArray(children);
+                    // If there were no braces, the entire document should be treated as a single object
+                    return new ConfigNodeRoot(Collections.singletonList(new ConfigNodeObject(children)));
+                } else {
+                    return new ConfigNodeRoot(children);
                 }
-                return new ConfigNodeObject(children);
             } else {
                 throw parseError("Document has trailing tokens after first object or array: "
                         + t);

config/src/main/java/com/typesafe/config/impl/ConfigNodeComment.java

@@ -0,0 +1,17 @@
+package com.typesafe.config.impl;
+
+
+import com.typesafe.config.ConfigException;
+
+final class ConfigNodeComment extends ConfigNodeSingleToken {
+    ConfigNodeComment(Token comment) {
+        super(comment);
+        if (!Tokens.isComment(super.token)) {
+            throw new ConfigException.BugOrBroken("Tried to create a ConfigNodeComment from a non-comment token");
+        }
+    }
+
+    protected String commentText() {
+        return Tokens.getCommentText(super.token);
+    }
+}

config/src/main/java/com/typesafe/config/impl/ConfigNodeField.java

@@ -7,6 +7,7 @@ import com.typesafe.config.ConfigException;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 final class ConfigNodeField extends AbstractConfigNode {
     final private ArrayList<AbstractConfigNode> children;
@@ -52,4 +53,26 @@ final class ConfigNodeField extends AbstractConfigNode {
         }
         throw new ConfigException.BugOrBroken("Field node doesn't have a path");
     }
+
+    protected Token separator() {
+        for (AbstractConfigNode child : children) {
+            if (child instanceof ConfigNodeSingleToken) {
+                Token t = ((ConfigNodeSingleToken) child).token();
+                if (t == Tokens.PLUS_EQUALS || t == Tokens.COLON || t == Tokens.EQUALS) {
+                    return t;
+                }
+            }
+        }
+        return null;
+    }
+
+    protected List<String> comments() {
+        List<String> comments = new ArrayList<String>();
+        for (AbstractConfigNode child : children) {
+            if (child instanceof ConfigNodeComment) {
+                comments.add(((ConfigNodeComment) child).commentText());
+            }
+        }
+        return comments;
+    }
 }

config/src/main/java/com/typesafe/config/impl/ConfigNodeInclude.java

@@ -0,0 +1,31 @@
+package com.typesafe.config.impl;
+
+import java.util.Collection;
+
+final class ConfigNodeInclude extends ConfigNodeComplexValue {
+    ConfigNodeInclude(Collection<AbstractConfigNode> children) {
+        super(children);
+    }
+
+    protected String kind() {
+        for (AbstractConfigNode n : children) {
+            if (n instanceof ConfigNodeSingleToken) {
+                Token t = ((ConfigNodeSingleToken) n).token();
+                if (Tokens.isUnquotedText(t) && !t.tokenText().equals("include") &&
+                        t.tokenText().matches(".*\\w.*")) {
+                    return t.tokenText();
+                }
+            }
+        }
+        return null;
+    }
+
+    protected String name() {
+        for (AbstractConfigNode n : children) {
+            if (n instanceof ConfigNodeSimpleValue) {
+                return (String)Tokens.getValue(((ConfigNodeSimpleValue) n).token()).unwrapped();
+            }
+        }
+        return null;
+    }
+}

config/src/main/java/com/typesafe/config/impl/ConfigNodeRoot.java

@@ -0,0 +1,38 @@
+package com.typesafe.config.impl;
+
+import com.typesafe.config.ConfigException;
+import com.typesafe.config.ConfigSyntax;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+final class ConfigNodeRoot extends ConfigNodeComplexValue {
+    ConfigNodeRoot(Collection<AbstractConfigNode> children) {
+        super(children);
+    }
+
+    protected ConfigNodeComplexValue value() {
+        for (AbstractConfigNode node : children) {
+            if (node instanceof ConfigNodeComplexValue) {
+                return (ConfigNodeComplexValue)node;
+            }
+        }
+        throw new ConfigException.BugOrBroken("ConfigNodeRoot did not contain a value");
+    }
+
+    protected ConfigNodeRoot setValue(String desiredPath, AbstractConfigNodeValue value, ConfigSyntax flavor) {
+        ArrayList<AbstractConfigNode> childrenCopy = new ArrayList(children);
+        for (int i = 0; i < childrenCopy.size(); i++) {
+            AbstractConfigNode node = childrenCopy.get(i);
+            if (node instanceof ConfigNodeComplexValue) {
+                if (node instanceof ConfigNodeArray) {
+                    throw new ConfigException.Generic("The ConfigDocument had an array at the root level, and values cannot be replaced inside an array.");
+                } else if (node instanceof ConfigNodeObject) {
+                    childrenCopy.set(i, ((ConfigNodeObject) node).setValueOnPath(desiredPath, value, flavor));
+                    return new ConfigNodeRoot(childrenCopy);
+                }
+            }
+        }
+        throw new ConfigException.BugOrBroken("ConfigNodeRoot did not contain a value");
+    }
+}

config/src/main/java/com/typesafe/config/impl/ConfigNodeSimpleValue.java

@@ -5,8 +5,9 @@ package com.typesafe.config.impl;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
-class ConfigNodeSimpleValue extends AbstractConfigNodeValue {
+final class ConfigNodeSimpleValue extends AbstractConfigNodeValue {
     final Token token;
     ConfigNodeSimpleValue(Token value) {
         token = value;
@@ -16,4 +17,21 @@ class ConfigNodeSimpleValue extends AbstractConfigNodeValue {
     protected Collection<Token> tokens() {
         return Collections.singletonList(token);
     }
+
+    protected Token token() { return token; }
+
+    protected AbstractConfigValue value() {
+        if (Tokens.isValue(token))
+            return Tokens.getValue(token);
+        else if (Tokens.isUnquotedText(token))
+            return new ConfigString.Unquoted(token.origin(), Tokens.getUnquotedText(token));
+        else if (Tokens.isSubstitution(token)) {
+            List<Token> expression = Tokens.getSubstitutionPathExpression(token);
+            Path path = PathParser.parsePathExpression(expression.iterator(), token.origin());
+            boolean optional = Tokens.getSubstitutionOptional(token);
+
+            return new ConfigReference(token.origin(), new SubstitutionExpression(path, optional));
+        }
+        return null;
+    }
 }

config/src/main/java/com/typesafe/config/impl/ConfigNodeSingleToken.java

@@ -6,7 +6,7 @@ package com.typesafe.config.impl;
 import java.util.Collection;
 import java.util.Collections;
 
-final class ConfigNodeSingleToken extends AbstractConfigNode{
+class ConfigNodeSingleToken extends AbstractConfigNode {
     final Token token;
     ConfigNodeSingleToken(Token t) {
         token = t;

config/src/main/java/com/typesafe/config/impl/ConfigParser.java

@@ -0,0 +1,435 @@
+/**
+ *   Copyright (C) 2011-2012 Typesafe Inc. <http://typesafe.com>
+ */
+package com.typesafe.config.impl;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Stack;
+
+import com.typesafe.config.*;
+
+final class ConfigParser {
+    static AbstractConfigValue parse(ConfigNodeRoot document,
+                                     ConfigOrigin origin, ConfigParseOptions options,
+                                     ConfigIncludeContext includeContext) {
+        ParseContext context = new ParseContext(options.getSyntax(), origin, document,
+                SimpleIncluder.makeFull(options.getIncluder()), includeContext);
+        return context.parse();
+    }
+
+    static private final class ParseContext {
+        private int lineNumber;
+        final private ConfigNodeRoot document;
+        final private FullIncluder includer;
+        final private ConfigIncludeContext includeContext;
+        final private ConfigSyntax flavor;
+        final private ConfigOrigin baseOrigin;
+        final private LinkedList<Path> pathStack;
+        // this is the number of "equals" we are inside,
+        // used to modify the error message to reflect that
+        // someone may think this is .properties format.
+        int equalsCount;
+        // the number of lists we are inside; this is used to detect the "cannot
+        // generate a reference to a list element" problem, and once we fix that
+        // problem we should be able to get rid of this variable.
+        int arrayCount;
+
+        ParseContext(ConfigSyntax flavor, ConfigOrigin origin, ConfigNodeRoot document,
+                FullIncluder includer, ConfigIncludeContext includeContext) {
+            lineNumber = 1;
+            this.document = document;
+            this.flavor = flavor;
+            this.baseOrigin = origin;
+            this.includer = includer;
+            this.includeContext = includeContext;
+            this.pathStack = new LinkedList<Path>();
+            this.equalsCount = 0;
+            this.arrayCount = 0;
+        }
+
+        // merge a bunch of adjacent values into one
+        // value; change unquoted text into a string
+        // value.
+        private AbstractConfigValue parseConcatenation(ConfigNodeConcatenation n) {
+            // this trick is not done in JSON
+            if (flavor == ConfigSyntax.JSON)
+                return null;
+
+            // create only if we have value tokens
+            List<AbstractConfigValue> values = new ArrayList<AbstractConfigValue>();
+
+            for (AbstractConfigNode node : n.children()) {
+                AbstractConfigValue v = null;
+                if (node instanceof AbstractConfigNodeValue) {
+                    v = parseValue((AbstractConfigNodeValue)node, null);
+                    values.add(v);
+                }
+            }
+
+            return ConfigConcatenation.concatenate(values);
+        }
+
+        private SimpleConfigOrigin lineOrigin() {
+            return ((SimpleConfigOrigin) baseOrigin).withLineNumber(lineNumber);
+        }
+
+        private ConfigException parseError(String message) {
+            return parseError(message, null);
+        }
+
+        private ConfigException parseError(String message, Throwable cause) {
+            return new ConfigException.Parse(lineOrigin(), message, cause);
+        }
+
+        private Path fullCurrentPath() {
+            // pathStack has top of stack at front
+            if (pathStack.isEmpty())
+                throw new ConfigException.BugOrBroken("Bug in parser; tried to get current path when at root");
+            else
+                return new Path(pathStack.descendingIterator());
+        }
+
+        private AbstractConfigValue parseValue(AbstractConfigNodeValue n, List<String> comments) {
+            AbstractConfigValue v;
+
+            int startingArrayCount = arrayCount;
+            int startingEqualsCount = equalsCount;
+
+            if (n instanceof ConfigNodeSimpleValue) {
+                v = ((ConfigNodeSimpleValue) n).value();
+            } else if (n instanceof ConfigNodeObject) {
+                v = parseObject((ConfigNodeObject)n);
+            } else if (n instanceof ConfigNodeArray) {
+                v = parseArray((ConfigNodeArray)n);
+            } else if (n instanceof ConfigNodeConcatenation) {
+                v = parseConcatenation((ConfigNodeConcatenation)n);
+            } else {
+                throw parseError("Expecting a value but got wrong node type: " + n.getClass());
+            }
+
+            if (comments != null && !comments.isEmpty()) {
+                v = v.withOrigin(v.origin().prependComments(new ArrayList(comments)));
+                comments.clear();
+            }
+
+            if (arrayCount != startingArrayCount)
+                throw new ConfigException.BugOrBroken("Bug in config parser: unbalanced array count");
+            if (equalsCount != startingEqualsCount)
+                throw new ConfigException.BugOrBroken("Bug in config parser: unbalanced equals count");
+
+            return v;
+        }
+
+        private static AbstractConfigObject createValueUnderPath(Path path,
+                AbstractConfigValue value) {
+            // for path foo.bar, we are creating
+            // { "foo" : { "bar" : value } }
+            List<String> keys = new ArrayList<String>();
+
+            String key = path.first();
+            Path remaining = path.remainder();
+            while (key != null) {
+                keys.add(key);
+                if (remaining == null) {
+                    break;
+                } else {
+                    key = remaining.first();
+                    remaining = remaining.remainder();
+                }
+            }
+
+            // the withComments(null) is to ensure comments are only
+            // on the exact leaf node they apply to.
+            // a comment before "foo.bar" applies to the full setting
+            // "foo.bar" not also to "foo"
+            ListIterator<String> i = keys.listIterator(keys.size());
+            String deepest = i.previous();
+            AbstractConfigObject o = new SimpleConfigObject(value.origin().withComments(null),
+                    Collections.<String, AbstractConfigValue> singletonMap(
+                            deepest, value));
+            while (i.hasPrevious()) {
+                Map<String, AbstractConfigValue> m = Collections.<String, AbstractConfigValue> singletonMap(
+                        i.previous(), o);
+                o = new SimpleConfigObject(value.origin().withComments(null), m);
+            }
+
+            return o;
+        }
+
+        private void parseInclude(Map<String, AbstractConfigValue> values, ConfigNodeInclude n) {
+            AbstractConfigObject obj;
+
+            if (n.kind() != null) {
+                if (n.kind().equals("url(")) {
+                    URL url;
+                    try {
+                        url = new URL(n.name());
+                    } catch (MalformedURLException e) {
+                        throw parseError("include url() specifies an invalid URL: " + n.name(), e);
+                    }
+                    obj = (AbstractConfigObject) includer.includeURL(includeContext, url);
+                } else if (n.kind().equals("file(")) {
+                    obj = (AbstractConfigObject) includer.includeFile(includeContext,
+                            new File(n.name()));
+                } else if (n.kind().equals("classpath(")) {
+                    obj = (AbstractConfigObject) includer.includeResources(includeContext, n.name());
+                } else {
+                    throw new ConfigException.BugOrBroken("should not be reached");
+                }
+            } else {
+                obj = (AbstractConfigObject) includer
+                        .include(includeContext, n.name());
+            }
+
+            // we really should make this work, but for now throwing an
+            // exception is better than producing an incorrect result.
+            // See https://github.com/typesafehub/config/issues/160
+            if (arrayCount > 0 && obj.resolveStatus() != ResolveStatus.RESOLVED)
+                throw parseError("Due to current limitations of the config parser, when an include statement is nested inside a list value, "
+                        + "${} substitutions inside the included file cannot be resolved correctly. Either move the include outside of the list value or "
+                        + "remove the ${} statements from the included file.");
+
+            if (!pathStack.isEmpty()) {
+                Path prefix = fullCurrentPath();
+                obj = obj.relativized(prefix);
+            }
+
+            for (String key : obj.keySet()) {
+                AbstractConfigValue v = obj.get(key);
+                AbstractConfigValue existing = values.get(key);
+                if (existing != null) {
+                    values.put(key, v.withFallback(existing));
+                } else {
+                    values.put(key, v);
+                }
+            }
+        }
+
+        private AbstractConfigObject parseObject(ConfigNodeObject n) {
+            // invoked just after the OPEN_CURLY (or START, if !hadOpenCurly)
+            Map<String, AbstractConfigValue> values = new HashMap<String, AbstractConfigValue>();
+            SimpleConfigOrigin objectOrigin = lineOrigin();
+            boolean lastWasNewline = false;
+
+            ArrayList<AbstractConfigNode> nodes = new ArrayList(n.children());
+            List<String> comments = new ArrayList<String>();
+            for (int i = 0; i < nodes.size(); i++) {
+                AbstractConfigNode node = nodes.get(i);
+                if (node instanceof ConfigNodeComment) {
+                    lastWasNewline = false;
+                    comments.add(((ConfigNodeComment) node).commentText());
+                } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) {
+                    lineNumber++;
+                    if (lastWasNewline) {
+                        // Drop all comments if there was a blank line and start a new comment block
+                        comments.clear();
+                    }
+                    lastWasNewline = true;
+                } else if (flavor != ConfigSyntax.JSON && node instanceof ConfigNodeInclude) {
+                    parseInclude(values, (ConfigNodeInclude)node);
+                    lastWasNewline = false;
+                } else if (node instanceof ConfigNodeField) {
+                    lastWasNewline = false;
+                    Path path = ((ConfigNodeField) node).path().value();
+                    AbstractConfigNodeValue nodeValue = ((ConfigNodeField) node).value();
+                    comments.addAll(((ConfigNodeField) node).comments());
+                    boolean insideEquals = false;
+
+                    // path must be on-stack while we parse the value
+                    pathStack.push(path);
+                    if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) {
+                        // we really should make this work, but for now throwing
+                        // an exception is better than producing an incorrect
+                        // result. See
+                        // https://github.com/typesafehub/config/issues/160
+                        if (arrayCount > 0)
+                            throw parseError("Due to current limitations of the config parser, += does not work nested inside a list. "
+                                    + "+= expands to a ${} substitution and the path in ${} cannot currently refer to list elements. "
+                                    + "You might be able to move the += outside of the list and then refer to it from inside the list with ${}.");
+
+                        // because we will put it in an array after the fact so
+                        // we want this to be incremented during the parseValue
+                        // below in order to throw the above exception.
+                        arrayCount += 1;
+                    }
+
+                    AbstractConfigNodeValue valueNode;
+                    AbstractConfigValue newValue;
+
+                    if (((ConfigNodeField) node).separator() == Tokens.EQUALS) {
+                        insideEquals = true;
+                        equalsCount += 1;
+                    }
+
+                    valueNode = ((ConfigNodeField) node).value();
+
+                    // comments from the key token go to the value token
+                    //newValue = parseValue(valueToken.prepend(keyToken.comments));
+                    newValue = parseValue(valueNode, comments);
+
+                    if (((ConfigNodeField) node).separator() == Tokens.PLUS_EQUALS) {
+                        arrayCount -= 1;
+
+                        List<AbstractConfigValue> concat = new ArrayList<AbstractConfigValue>(2);
+                        AbstractConfigValue previousRef = new ConfigReference(newValue.origin(),
+                                new SubstitutionExpression(fullCurrentPath(), true /* optional */));
+                        AbstractConfigValue list = new SimpleConfigList(newValue.origin(),
+                                Collections.singletonList(newValue));
+                        concat.add(previousRef);
+                        concat.add(list);
+                        newValue = ConfigConcatenation.concatenate(concat);
+                    }
+
+                    // Grab any trailing comments on the same line
+                    if (i < nodes.size() - 1) {
+                        i++;
+                        while (i < nodes.size()) {
+                            if (nodes.get(i) instanceof ConfigNodeComment) {
+                                ConfigNodeComment comment = (ConfigNodeComment) nodes.get(i);
+                                newValue = newValue.withOrigin(newValue.origin().appendComments(
+                                            Collections.singletonList(comment.commentText())));
+                                break;
+                            } else if (nodes.get(i) instanceof ConfigNodeSingleToken) {
+                                ConfigNodeSingleToken curr = (ConfigNodeSingleToken) nodes.get(i);
+                                if (curr.token() == Tokens.COMMA || Tokens.isIgnoredWhitespace(curr.token())) {
+                                    // keep searching, as there could still be a comment
+                                } else {
+                                    i--;
+                                    break;
+                                }
+                            } else {
+                                i--;
+                                break;
+                            }
+                            i++;
+                        }
+                    }
+
+                    pathStack.pop();
+                    if (insideEquals) {
+                        equalsCount -= 1;
+                    }
+
+                    String key = path.first();
+                    Path remaining = path.remainder();
+
+                    if (remaining == null) {
+                        AbstractConfigValue existing = values.get(key);
+                        if (existing != null) {
+                            // In strict JSON, dups should be an error; while in
+                            // our custom config language, they should be merged
+                            // if the value is an object (or substitution that
+                            // could become an object).
+
+                            if (flavor == ConfigSyntax.JSON) {
+                                throw parseError("JSON does not allow duplicate fields: '"
+                                    + key
+                                    + "' was already seen at "
+                                    + existing.origin().description());
+                            } else {
+                                newValue = newValue.withFallback(existing);
+                            }
+                        }
+                        values.put(key, newValue);
+                    } else {
+                        if (flavor == ConfigSyntax.JSON) {
+                            throw new ConfigException.BugOrBroken(
+                                    "somehow got multi-element path in JSON mode");
+                        }
+
+                        AbstractConfigObject obj = createValueUnderPath(
+                                remaining, newValue);
+                        AbstractConfigValue existing = values.get(key);
+                        if (existing != null) {
+                            obj = obj.withFallback(existing);
+                        }
+                        values.put(key, obj);
+                    }
+                }
+            }
+
+            return new SimpleConfigObject(objectOrigin, values);
+        }
+
+        private SimpleConfigList parseArray(ConfigNodeArray n) {
+            // invoked just after the OPEN_SQUARE
+            arrayCount += 1;
+
+            SimpleConfigOrigin arrayOrigin = lineOrigin();
+            List<AbstractConfigValue> values = new ArrayList<AbstractConfigValue>();
+
+            boolean lastWasNewLine = false;
+            List<String> comments = new ArrayList<String>();
+
+            AbstractConfigValue v = null;
+            // now remaining elements
+            for (AbstractConfigNode node : n.children()) {
+                if (node instanceof ConfigNodeComment) {
+                    comments.add(((ConfigNodeComment) node).commentText());
+                    lastWasNewLine = false;
+                } else if (node instanceof ConfigNodeSingleToken && Tokens.isNewline(((ConfigNodeSingleToken) node).token())) {
+                    lineNumber++;
+                    if (lastWasNewLine && v == null) {
+                        comments.clear();
+                    } else if (v != null) {
+                        values.add(v.withOrigin(v.origin().appendComments(new ArrayList(comments))));
+                        comments.clear();
+                        v = null;
+                    }
+                    lastWasNewLine = true;
+                } else if (node instanceof AbstractConfigNodeValue) {
+                    lastWasNewLine = false;
+                    if (v != null) {
+                        values.add(v.withOrigin(v.origin().appendComments(new ArrayList(comments))));
+                        comments.clear();
+                    }
+                    v = parseValue((AbstractConfigNodeValue)node, comments);
+                }
+            }
+            // There shouldn't be any comments at this point, but add them just in case
+            if (v != null) {
+                values.add(v.withOrigin(v.origin().appendComments(new ArrayList(comments))));
+            }
+            arrayCount -= 1;
+            return new SimpleConfigList(arrayOrigin, values);
+        }
+
+        AbstractConfigValue parse() {
+            AbstractConfigValue result = null;
+            ArrayList<String> comments = new ArrayList<String>();
+            boolean lastWasNewLine = false;
+            for (AbstractConfigNode node : document.children()) {
+                if (node instanceof ConfigNodeComment) {
+                    comments.add(((ConfigNodeComment) node).commentText());
+                    lastWasNewLine = false;
+                } else if (node instanceof ConfigNodeSingleToken) {
+                    Token t = ((ConfigNodeSingleToken) node).token();
+                    if (Tokens.isNewline(t)) {
+                        lineNumber++;
+                        if (lastWasNewLine && result == null) {
+                            comments.clear();
+                        } else if (result != null) {
+                            result = result.withOrigin(result.origin().appendComments(new ArrayList(comments)));
+                            comments.clear();
+                            break;
+                        }
+                        lastWasNewLine = true;
+                    }
+                } else if (node instanceof ConfigNodeComplexValue) {
+                    result = parseValue((ConfigNodeComplexValue)node, comments);
+                    lastWasNewLine = false;
+                }
+            }
+            return result;
+        }
+    }
+}

config/src/main/java/com/typesafe/config/impl/Parseable.java

@@ -211,7 +211,7 @@ public abstract class Parseable implements ConfigParseable {
             return rawParseDocument(origin, finalOptions);
         } catch (IOException e) {
             if (finalOptions.getAllowMissing()) {
-                return new SimpleConfigDocument(new ConfigNodeObject(new ArrayList<AbstractConfigNode>()), finalOptions);
+                return new SimpleConfigDocument(new ConfigNodeRoot(Collections.singletonList(new ConfigNodeObject(new ArrayList<AbstractConfigNode>()))), finalOptions);
             } else {
                 trace("exception loading " + origin.description() + ": " + e.getClass().getName()
                         + ": " + e.getMessage());
@@ -254,7 +254,9 @@ public abstract class Parseable implements ConfigParseable {
             return PropertiesParser.parse(reader, origin);
         } else {
             Iterator<Token> tokens = Tokenizer.tokenize(origin, reader, finalOptions.getSyntax());
-            return Parser.parse(tokens, origin, finalOptions, includeContext());
+            ConfigNodeRoot document = ConfigDocumentParser.parse(tokens, finalOptions);
+            return ConfigParser.parse(document, origin, finalOptions, includeContext());
+            //return Parser.parse(tokens, origin, finalOptions, includeContext());
         }
     }
 

config/src/main/java/com/typesafe/config/impl/SimpleConfigDocument.java

@@ -9,25 +9,22 @@ import java.io.StringReader;
 import java.util.Iterator;
 
 final class SimpleConfigDocument implements ConfigDocument {
-    private ConfigNodeComplexValue configNodeTree;
+    private ConfigNodeRoot configNodeTree;
     private ConfigParseOptions parseOptions;
 
-    SimpleConfigDocument(ConfigNodeComplexValue parsedNode, ConfigParseOptions parseOptions) {
+    SimpleConfigDocument(ConfigNodeRoot parsedNode, ConfigParseOptions parseOptions) {
         configNodeTree = parsedNode;
         this.parseOptions = parseOptions;
     }
 
     public ConfigDocument setValue(String path, String newValue) {
-        if (configNodeTree instanceof ConfigNodeArray) {
-            throw new ConfigException.Generic("The ConfigDocument had an array at the root level, and values cannot be replaced inside an array.");
-        }
         SimpleConfigOrigin origin = SimpleConfigOrigin.newSimple("single value parsing");
         StringReader reader = new StringReader(newValue);
         Iterator<Token> tokens = Tokenizer.tokenize(origin, reader, parseOptions.getSyntax());
         AbstractConfigNodeValue parsedValue = ConfigDocumentParser.parseValue(tokens, parseOptions);
         reader.close();
 
-        return new SimpleConfigDocument(((ConfigNodeObject)configNodeTree).setValueOnPath(path, parsedValue, parseOptions.getSyntax()), parseOptions);
+        return new SimpleConfigDocument(configNodeTree.setValue(path, parsedValue, parseOptions.getSyntax()), parseOptions);
     }
 
     public ConfigDocument setValue(String path, ConfigValue newValue) {

config/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala

@@ -567,6 +567,14 @@ class ConfParserTest extends TestUtils {
                 ]
                 """)
         assertComments(Seq(" BeforeCommaElementSameLine"), conf22, "foo", 0)
+
+        // comment with a line containing only whitespace after is dropped
+        val conf23 = parseConfig("""
+                { # BlankAfter
+
+                foo=10 }
+                              """)
+        assertComments(Seq(), conf23, "foo")
     }
 
     @Test