Implement parsing url() file() classpath() includes

config/src/main/java/com/typesafe/config/impl/Parser.java

@@ -3,7 +3,10 @@
  */
 package com.typesafe.config.impl;
 
+import java.io.File;
 import java.io.StringReader;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -496,29 +499,87 @@ final class Parser {
                 t = nextTokenIgnoringNewline();
             }
 
-            if (Tokens.isValueWithType(t.token, ConfigValueType.STRING)) {
-                String name = (String) Tokens.getValue(t.token).unwrapped();
-                AbstractConfigObject obj = (AbstractConfigObject) includer
-                        .include(includeContext, name);
+            AbstractConfigObject obj;
 
-                if (!pathStack.isEmpty()) {
-                    Path prefix = new Path(pathStack);
-                    obj = obj.relativized(prefix);
+            // we either have a quoted string or the "file()" syntax
+            if (Tokens.isUnquotedText(t.token)) {
+                // get foo(
+                String kind = Tokens.getUnquotedText(t.token);
+
+                if (kind.equals("url(")) {
+
+                } else if (kind.equals("file(")) {
+
+                } else if (kind.equals("classpath(")) {
+
+                } else {
+                    throw parseError("expecting include parameter to be quoted filename, file(), classpath(), or url(). No spaces are allowed before the open paren. Not expecting: "
+                            + t);
                 }
 
-                for (String key : obj.keySet()) {
-                    AbstractConfigValue v = obj.get(key);
-                    AbstractConfigValue existing = values.get(key);
-                    if (existing != null) {
-                        values.put(key, v.withFallback(existing));
-                    } else {
-                        values.put(key, v);
+                // skip space inside parens
+                t = nextTokenIgnoringNewline();
+                while (isUnquotedWhitespace(t.token)) {
+                    t = nextTokenIgnoringNewline();
+                }
+
+                // quoted string
+                String name;
+                if (Tokens.isValueWithType(t.token, ConfigValueType.STRING)) {
+                    name = (String) Tokens.getValue(t.token).unwrapped();
+                } else {
+                    throw parseError("expecting a quoted string inside file(), classpath(), or url(), rather than: "
+                            + t);
+                }
+                // skip space after string, inside parens
+                t = nextTokenIgnoringNewline();
+                while (isUnquotedWhitespace(t.token)) {
+                    t = nextTokenIgnoringNewline();
+                }
+
+                if (Tokens.isUnquotedText(t.token) && Tokens.getUnquotedText(t.token).equals(")")) {
+                    // OK, close paren
+                } else {
+                    throw parseError("expecting a close parentheses ')' here, not: " + t);
+                }
+
+                if (kind.equals("url(")) {
+                    URL url;
+                    try {
+                        url = new URL(name);
+                    } catch (MalformedURLException e) {
+                        throw parseError("include url() specifies an invalid URL: " + name, e);
                     }
+                    obj = (AbstractConfigObject) includer.includeURL(includeContext, url);
+                } else if (kind.equals("file(")) {
+                    obj = (AbstractConfigObject) includer.includeFile(includeContext,
+                            new File(name));
+                } else if (kind.equals("classpath(")) {
+                    obj = (AbstractConfigObject) includer.includeResources(includeContext, name);
+                } else {
+                    throw new ConfigException.BugOrBroken("should not be reached");
                 }
-
+            } else if (Tokens.isValueWithType(t.token, ConfigValueType.STRING)) {
+                String name = (String) Tokens.getValue(t.token).unwrapped();
+                obj = (AbstractConfigObject) includer
+                        .include(includeContext, name);
             } else {
-                throw parseError("include keyword is not followed by a quoted string, but by: "
-                        + t);
+                throw parseError("include keyword is not followed by a quoted string, but by: " + t);
+            }
+
+            if (!pathStack.isEmpty()) {
+                Path prefix = new Path(pathStack);
+                obj = obj.relativized(prefix);
+            }
+
+            for (String key : obj.keySet()) {
+                AbstractConfigValue v = obj.get(key);
+                AbstractConfigValue existing = values.get(key);
+                if (existing != null) {
+                    values.put(key, v.withFallback(existing));
+                } else {
+                    values.put(key, v);
+                }
             }
         }
 

config/src/test/scala/com/typesafe/config/impl/ConfParserTest.scala

@@ -489,4 +489,121 @@ class ConfParserTest extends TestUtils {
         assertComments(Seq(), conf8, "x")
         assertComments(Seq(), conf8, "a")
     }
+
+    @Test
+    def includeFile() {
+        val conf = ConfigFactory.parseString("include file(\"" + resourceFile("test01") + "\")")
+
+        // should have loaded conf, json, properties
+        assertEquals(42, conf.getInt("ints.fortyTwo"))
+        assertEquals(1, conf.getInt("fromJson1"))
+        assertEquals("abc", conf.getString("fromProps.abc"))
+    }
+
+    @Test
+    def includeFileWithExtension() {
+        val conf = ConfigFactory.parseString("include file(\"" + resourceFile("test01.conf") + "\")")
+
+        assertEquals(42, conf.getInt("ints.fortyTwo"))
+        assertFalse(conf.hasPath("fromJson1"))
+        assertFalse(conf.hasPath("fromProps.abc"))
+    }
+
+    @Test
+    def includeFileWhitespaceInsideParens() {
+        val conf = ConfigFactory.parseString("include file(  \n  \"" + resourceFile("test01") + "\"  \n  )")
+
+        // should have loaded conf, json, properties
+        assertEquals(42, conf.getInt("ints.fortyTwo"))
+        assertEquals(1, conf.getInt("fromJson1"))
+        assertEquals("abc", conf.getString("fromProps.abc"))
+    }
+
+    @Test
+    def includeFileNoWhitespaceOutsideParens() {
+        val e = intercept[ConfigException.Parse] {
+            ConfigFactory.parseString("include file (\"" + resourceFile("test01") + "\")")
+        }
+        assertTrue("wrong exception: " + e.getMessage, e.getMessage.contains("expecting include parameter"))
+    }
+
+    @Test
+    def includeFileNotQuoted() {
+        val e = intercept[ConfigException.Parse] {
+            ConfigFactory.parseString("include file(" + resourceFile("test01") + ")")
+        }
+        assertTrue("wrong exception: " + e.getMessage, e.getMessage.contains("expecting include parameter"))
+    }
+
+    @Test
+    def includeFileNotQuotedAndSpecialChar() {
+        val e = intercept[ConfigException.Parse] {
+            ConfigFactory.parseString("include file(:" + resourceFile("test01") + ")")
+        }
+        assertTrue("wrong exception: " + e.getMessage, e.getMessage.contains("expecting a quoted string"))
+    }
+
+    @Test
+    def includeFileUnclosedParens() {
+        val e = intercept[ConfigException.Parse] {
+            ConfigFactory.parseString("include file(\"" + resourceFile("test01") + "\" something")
+        }
+        assertTrue("wrong exception: " + e.getMessage, e.getMessage.contains("expecting a close paren"))
+    }
+
+    @Test
+    def includeURLBasename() {
+        // "AnySyntax" trick doesn't work for url() includes
+        val url = resourceFile("test01").toURI().toURL().toExternalForm()
+        val conf = ConfigFactory.parseString("include url(\"" + url + "\")")
+
+        assertTrue("including basename URL doesn't load anything", conf.isEmpty())
+    }
+
+    @Test
+    def includeURLWithExtension() {
+        val url = resourceFile("test01.conf").toURI().toURL().toExternalForm()
+        val conf = ConfigFactory.parseString("include url(\"" + url + "\")")
+
+        assertEquals(42, conf.getInt("ints.fortyTwo"))
+        assertFalse(conf.hasPath("fromJson1"))
+        assertFalse(conf.hasPath("fromProps.abc"))
+    }
+
+    @Test
+    def includeURLInvalid() {
+        val e = intercept[ConfigException.Parse] {
+            ConfigFactory.parseString("include url(\"junk:junk:junk\")")
+        }
+        assertTrue("wrong exception: " + e.getMessage, e.getMessage.contains("invalid URL"))
+    }
+
+    @Test
+    def includeResources() {
+        val conf = ConfigFactory.parseString("include classpath(\"test01\")")
+
+        // should have loaded conf, json, properties
+        assertEquals(42, conf.getInt("ints.fortyTwo"))
+        assertEquals(1, conf.getInt("fromJson1"))
+        assertEquals("abc", conf.getString("fromProps.abc"))
+    }
+
+    @Test
+    def includeURLHeuristically() {
+        val url = resourceFile("test01.conf").toURI().toURL().toExternalForm()
+        val conf = ConfigFactory.parseString("include \"" + url + "\"")
+
+        assertEquals(42, conf.getInt("ints.fortyTwo"))
+        assertFalse(conf.hasPath("fromJson1"))
+        assertFalse(conf.hasPath("fromProps.abc"))
+    }
+
+    @Test
+    def includeURLBasenameHeuristically() {
+        // "AnySyntax" trick doesn't work for url includes
+        val url = resourceFile("test01").toURI().toURL().toExternalForm()
+        val conf = ConfigFactory.parseString("include \"" + url + "\"")
+
+        assertTrue("including basename URL doesn't load anything", conf.isEmpty())
+    }
 }

config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala

@@ -13,6 +13,7 @@ import java.io.File
 import scala.collection.mutable
 import equiv03.SomethingInEquiv03
 import java.io.StringReader
+import java.net.URL
 
 class PublicApiTest extends TestUtils {
     @Test
@@ -282,11 +283,17 @@ class PublicApiTest extends TestUtils {
         assertEquals(conf, conf2)
     }
 
-    case class Included(name: String, fallback: ConfigIncluder)
+    sealed trait IncludeKind
+    case object IncludeKindHeuristic extends IncludeKind;
+    case object IncludeKindFile extends IncludeKind;
+    case object IncludeKindURL extends IncludeKind;
+    case object IncludeKindClasspath extends IncludeKind;
+
+    case class Included(name: String, fallback: ConfigIncluder, kind: IncludeKind)
 
     class RecordingIncluder(val fallback: ConfigIncluder, val included: mutable.ListBuffer[Included]) extends ConfigIncluder {
         override def include(context: ConfigIncludeContext, name: String): ConfigObject = {
-            included += Included(name, fallback)
+            included += Included(name, fallback, IncludeKindHeuristic)
             fallback.include(context, name)
         }
 
@@ -301,6 +308,35 @@ class PublicApiTest extends TestUtils {
         }
     }
 
+    class RecordingFullIncluder(fallback: ConfigIncluder, included: mutable.ListBuffer[Included])
+        extends RecordingIncluder(fallback, included)
+        with ConfigIncluderFile with ConfigIncluderURL with ConfigIncluderClasspath {
+        override def includeFile(context: ConfigIncludeContext, file: File) = {
+            included += Included("file(" + file.getName() + ")", fallback, IncludeKindFile)
+            fallback.asInstanceOf[ConfigIncluderFile].includeFile(context, file)
+        }
+
+        override def includeURL(context: ConfigIncludeContext, url: URL) = {
+            included += Included("url(" + url.toExternalForm() + ")", fallback, IncludeKindURL)
+            fallback.asInstanceOf[ConfigIncluderURL].includeURL(context, url)
+        }
+
+        override def includeResources(context: ConfigIncludeContext, name: String) = {
+            included += Included("classpath(" + name + ")", fallback, IncludeKindFile)
+            fallback.asInstanceOf[ConfigIncluderClasspath].includeResources(context, name)
+        }
+
+        override def withFallback(fallback: ConfigIncluder) = {
+            if (this.fallback == fallback) {
+                this;
+            } else if (this.fallback == null) {
+                new RecordingFullIncluder(fallback, included);
+            } else {
+                new RecordingFullIncluder(this.fallback.withFallback(fallback), included)
+            }
+        }
+    }
+
     private def whatWasIncluded(parser: ConfigParseOptions => Config): List[Included] = {
         val included = mutable.ListBuffer[Included]()
         val includer = new RecordingIncluder(null, included)
@@ -310,6 +346,15 @@ class PublicApiTest extends TestUtils {
         included.toList
     }
 
+    private def whatWasIncludedFull(parser: ConfigParseOptions => Config): List[Included] = {
+        val included = mutable.ListBuffer[Included]()
+        val includer = new RecordingFullIncluder(null, included)
+
+        val conf = parser(ConfigParseOptions.defaults().setIncluder(includer).setAllowMissing(false))
+
+        included.toList
+    }
+
     @Test
     def includersAreUsedWithFiles() {
         val included = whatWasIncluded(ConfigFactory.parseFile(resourceFile("test03.conf"), _))
@@ -337,6 +382,18 @@ class PublicApiTest extends TestUtils {
             included.map(_.name))
     }
 
+    // full includer should only be used with the file(), url(), classpath() syntax.
+    @Test
+    def fullIncluderNotUsedWithoutNewSyntax() {
+        val included = whatWasIncluded(ConfigFactory.parseFile(resourceFile("equiv03/includes.conf"), _))
+
+        assertEquals(List("letters/a.conf", "numbers/1.conf", "numbers/2", "letters/b.json", "letters/c", "root/foo.conf"),
+            included.map(_.name))
+
+        val includedFull = whatWasIncludedFull(ConfigFactory.parseFile(resourceFile("equiv03/includes.conf"), _))
+        assertEquals(included, includedFull)
+    }
+
     @Test
     def includersAreUsedWithClasspath() {
         val included = whatWasIncluded(ConfigFactory.parseResources(classOf[PublicApiTest], "/test03.conf", _))
@@ -377,6 +434,33 @@ class PublicApiTest extends TestUtils {
             included.map(_.name))
     }
 
+    @Test
+    def fullIncluderUsed() {
+        val included = whatWasIncludedFull(ConfigFactory.parseString("""
+                    include "equiv03/includes.conf"
+                    include file("nonexistent")
+                    include url("file:/nonexistent")
+                    include classpath("nonexistent")
+                """, _))
+        assertEquals(List("equiv03/includes.conf", "letters/a.conf", "numbers/1.conf",
+            "numbers/2", "letters/b.json", "letters/c", "root/foo.conf",
+            "file(nonexistent)", "url(file:/nonexistent)", "classpath(nonexistent)"),
+            included.map(_.name))
+    }
+
+    @Test
+    def nonFullIncluderSurvivesNewStyleIncludes() {
+        val included = whatWasIncluded(ConfigFactory.parseString("""
+                    include "equiv03/includes.conf"
+                    include file("nonexistent")
+                    include url("file:/nonexistent")
+                    include classpath("nonexistent")
+                """, _))
+        assertEquals(List("equiv03/includes.conf", "letters/a.conf", "numbers/1.conf",
+            "numbers/2", "letters/b.json", "letters/c", "root/foo.conf"),
+            included.map(_.name))
+    }
+
     @Test
     def stringParsing() {
         val conf = ConfigFactory.parseString("{ a : b }", ConfigParseOptions.defaults())