test and fix up the merged config object loaded from resources

src/main/java/com/typesafe/config/ConfigObject.java

@@ -90,6 +90,8 @@ public interface ConfigObject extends ConfigValue {
 
     List<Double> getDoubleList(String path);
 
+    List<String> getStringList(String path);
+
     List<? extends ConfigObject> getObjectList(String path);
 
     List<? extends Object> getAnyList(String path);

src/main/java/com/typesafe/config/impl/AbstractConfigObject.java

@@ -146,10 +146,9 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements
     }
 
     /**
-     * Stack should be from overrides to fallbacks (earlier items win). Test
-     * suite should check: merging of objects with a non-object in the middle.
-     * Override of object with non-object, override of non-object with object.
-     * Merging 0, 1, N objects.
+     * Stack should be from overrides to fallbacks (earlier items win). Objects
+     * have their keys combined into a new object, while other kinds of value
+     * are just first-one-wins.
      */
     static AbstractConfigObject merge(ConfigOrigin origin,
             List<AbstractConfigObject> stack,
@@ -175,6 +174,7 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements
                                 stackForKey = objects.get(key);
                             } else {
                                 stackForKey = new ArrayList<AbstractConfigObject>();
+                                objects.put(key, stackForKey);
                             }
                             stackForKey.add(transformed(
                                     (AbstractConfigObject) v,
@@ -187,6 +187,7 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements
                     }
                 }
             }
+
             for (Map.Entry<String, List<AbstractConfigObject>> entry : objects
                     .entrySet()) {
                 List<AbstractConfigObject> stackForKey = entry.getValue();
@@ -373,6 +374,11 @@ abstract class AbstractConfigObject extends AbstractConfigValue implements
         return l;
     }
 
+    @Override
+    public List<String> getStringList(String path) {
+        return getHomogeneousUnwrappedList(path, ConfigValueType.STRING);
+    }
+
     @Override
     public List<ConfigObject> getObjectList(String path) {
         List<ConfigObject> l = new ArrayList<ConfigObject>();

src/main/java/com/typesafe/config/impl/ConfigImpl.java

@@ -1,5 +1,8 @@
 package com.typesafe.config.impl;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Enumeration;
@@ -11,6 +14,7 @@ import java.util.Properties;
 import com.typesafe.config.ConfigConfig;
 import com.typesafe.config.ConfigException;
 import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigOrigin;
 
 /** This is public but is only supposed to be used by the "config" package */
 public class ConfigImpl {
@@ -28,13 +32,29 @@ public class ConfigImpl {
         if (system != null)
             stack.add(system);
 
+        // now try to load a resource for each extension
+        addResource(configConfig.rootPath() + ".conf", stack);
+        addResource(configConfig.rootPath() + ".json", stack);
+        addResource(configConfig.rootPath() + ".properties", stack);
+
         ConfigTransformer transformer = withExtraTransformer(null);
 
         AbstractConfigObject merged = AbstractConfigObject
                 .merge(new SimpleConfigOrigin("config for "
                         + configConfig.rootPath()), stack, transformer);
 
-        return merged;
+        AbstractConfigValue resolved = SubstitutionResolver.resolve(merged,
+                merged);
+
+        return (AbstractConfigObject) resolved;
+    }
+
+    private static void addResource(String name,
+            List<AbstractConfigObject> stack) {
+        URL url = ConfigImpl.class.getResource("/" + name);
+        if (url != null) {
+            stack.add(loadURL(url));
+        }
     }
 
     static ConfigObject getEnvironmentAsConfig() {
@@ -51,6 +71,39 @@ public class ConfigImpl {
                 withExtraTransformer(null));
     }
 
+    static AbstractConfigObject loadURL(URL url) {
+        if (url.getPath().endsWith(".properties")) {
+            ConfigOrigin origin = new SimpleConfigOrigin(url.toExternalForm());
+            Properties props = new Properties();
+            InputStream stream = null;
+            try {
+                stream = url.openStream();
+                props.load(stream);
+            } catch (IOException e) {
+                throw new ConfigException.IO(origin, "failed to open url", e);
+            } finally {
+                if (stream != null) {
+                    try {
+                        stream.close();
+                    } catch (IOException e) {
+                    }
+                }
+            }
+            return fromProperties(url.toExternalForm(), props);
+        } else {
+            return forceParsedToObject(Parser.parse(url));
+        }
+    }
+
+    static AbstractConfigObject forceParsedToObject(AbstractConfigValue value) {
+        if (value instanceof AbstractConfigObject) {
+            return (AbstractConfigObject) value;
+        } else {
+            throw new ConfigException.WrongType(value.origin(), "",
+                    "object at file root", value.valueType().name());
+        }
+    }
+
     private static ConfigTransformer withExtraTransformer(
             ConfigTransformer extraTransformer) {
         // idea is to avoid creating a new, unique transformer if there's no

src/main/java/com/typesafe/config/impl/Parser.java

@@ -2,13 +2,14 @@ package com.typesafe.config.impl;
 
 import java.io.BufferedInputStream;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.StringReader;
 import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -48,28 +49,53 @@ final class Parser {
         return parse(flavor, origin, new StringReader(input));
     }
 
-    static AbstractConfigValue parse(File f) {
-        ConfigOrigin origin = new SimpleConfigOrigin(f.getPath());
-        SyntaxFlavor flavor = null;
-        if (f.getName().endsWith(".json"))
-            flavor = SyntaxFlavor.JSON;
-        else if (f.getName().endsWith(".conf"))
-            flavor = SyntaxFlavor.CONF;
+    private static SyntaxFlavor flavorFromExtension(String name,
+            ConfigOrigin origin) {
+        if (name.endsWith(".json"))
+            return SyntaxFlavor.JSON;
+        else if (name.endsWith(".conf"))
+            return SyntaxFlavor.CONF;
         else
             throw new ConfigException.IO(origin, "Unknown filename extension");
-        return parse(flavor, f);
+    }
+
+    static AbstractConfigValue parse(File f) {
+        return parse(null, f);
     }
 
     static AbstractConfigValue parse(SyntaxFlavor flavor, File f) {
         ConfigOrigin origin = new SimpleConfigOrigin(f.getPath());
+        try {
+            return parse(flavor, origin, f.toURI().toURL());
+        } catch (MalformedURLException e) {
+            throw new ConfigException.IO(origin,
+                    "failed to create url from file path", e);
+        }
+    }
 
+    static AbstractConfigValue parse(URL url) {
+        return parse(null, url);
+    }
+
+    static AbstractConfigValue parse(SyntaxFlavor flavor, URL url) {
+        ConfigOrigin origin = new SimpleConfigOrigin(url.toExternalForm());
+        return parse(flavor, origin, url);
+    }
+
+    static AbstractConfigValue parse(SyntaxFlavor flavor, ConfigOrigin origin,
+            URL url) {
         AbstractConfigValue result = null;
         try {
-            InputStream stream = new BufferedInputStream(new FileInputStream(f));
-            result = parse(flavor, origin, stream);
-            stream.close();
+            InputStream stream = new BufferedInputStream(url.openStream());
+            try {
+                result = parse(
+                        flavor != null ? flavor : flavorFromExtension(
+                                url.getPath(), origin), origin, stream);
+            } finally {
+                stream.close();
+            }
         } catch (IOException e) {
-            throw new ConfigException.IO(origin, "failed to read file", e);
+            throw new ConfigException.IO(origin, "failed to read url", e);
         }
         return result;
     }

src/test/resources/test01.conf

@@ -0,0 +1,55 @@
+{
+    "ints" : {
+        "fortyTwo" : 42,
+        "fortyTwoAgain" : ${ints.fortyTwo}
+    },
+
+    "floats" : {
+        "fortyTwoPointOne" : 42.1,
+        "fortyTwoPointOneAgain" : ${floats.fortyTwoPointOne}
+    },
+
+    "strings" : {
+        "abcd" : "abcd",
+        "abcdAgain" : ${strings.a}${strings.b}${strings.c}${strings.d},
+        "a" : "a",
+        "b" : "b",
+        "c" : "c",
+        "d" : "d",
+        "concatenated" : null bar 42 baz true 3.14 hi,
+        "number" : "57"
+    },
+
+    "arrays" : {
+        "empty" : [],
+        "ofInt" : [1, 2, 3],
+        "ofString" : [ ${strings.a}, ${strings.b}, ${strings.c} ],
+        "ofDouble" : [3.14, 4.14, 5.14],
+        "ofNull" : [null, null, null],
+        "ofBoolean" : [true, false],
+        "ofArray" : [${arrays.ofString}, ${arrays.ofString}, ${arrays.ofString}],
+        "ofObject" : [${ints}, ${booleans}, ${strings}]
+    },
+
+    "booleans" : {
+        "true" : true,
+        "trueAgain" : ${booleans.true},
+        "false" : false,
+        "falseAgain" : ${booleans.false}
+    },
+
+    "nulls" : {
+        "null" : null,
+        "nullAgain" : null
+    },
+
+    "durations" : {
+        "second" : 1s,
+        "secondsList" : [1s,2seconds,3 s]
+    },
+
+    "memsizes" : {
+        "meg" : 1M,
+        "megsList" : [1M, 1024K]
+    }
+}

src/test/resources/test01.json

@@ -0,0 +1,4 @@
+{
+    "fromJson1" : 1,
+    "fromJsonA" : "A"
+}

src/test/resources/test01.properties

@@ -0,0 +1,4 @@
+# .properties file
+fromProps.abc=abc
+fromProps.one=1
+fromProps.bool=true

src/test/scala/com/typesafe/config/impl/ConfigSubstitutionTest.scala

@@ -7,10 +7,6 @@ import com.typesafe.config.ConfigException
 
 class ConfigSubstitutionTest extends TestUtils {
 
-    private def parseObject(s: String) = {
-        Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test string"), s).asInstanceOf[AbstractConfigObject]
-    }
-
     private def subst(ref: String, style: SubstitutionStyle = SubstitutionStyle.PATH) = {
         val pieces = java.util.Collections.singletonList[Object](new Substitution(ref, style))
         new ConfigSubstitution(fakeOrigin(), pieces)
@@ -22,12 +18,6 @@ class ConfigSubstitutionTest extends TestUtils {
         new ConfigSubstitution(fakeOrigin(), pieces.asJava)
     }
 
-    private def intValue(i: Int) = new ConfigInt(fakeOrigin(), i)
-    private def boolValue(b: Boolean) = new ConfigBoolean(fakeOrigin(), b)
-    private def nullValue() = new ConfigNull(fakeOrigin())
-    private def stringValue(s: String) = new ConfigString(fakeOrigin(), s)
-    private def doubleValue(d: Double) = new ConfigDouble(fakeOrigin(), d)
-
     private def resolveWithoutFallbacks(v: AbstractConfigObject) = {
         SubstitutionResolver.resolveWithoutFallbacks(v, v).asInstanceOf[AbstractConfigObject]
     }

src/test/scala/com/typesafe/config/impl/ConfigTest.scala

@@ -0,0 +1,236 @@
+package com.typesafe.config.impl
+
+import org.junit.Assert._
+import org.junit._
+import com.typesafe.config.ConfigValue
+import com.typesafe.config.Config
+import com.typesafe.config.ConfigObject
+import com.typesafe.config.ConfigException
+import java.util.concurrent.TimeUnit
+import scala.collection.JavaConverters._
+
+class ConfigTest extends TestUtils {
+
+    @Test
+    def mergeTrivial() {
+        val obj1 = parseObject("""{ "a" : 1 }""")
+        val obj2 = parseObject("""{ "b" : 2 }""")
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2).asJava, null)
+
+        assertEquals(1, merged.getInt("a"))
+        assertEquals(2, merged.getInt("b"))
+        assertEquals(2, merged.keySet().size)
+    }
+
+    @Test
+    def mergeEmpty() {
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List[AbstractConfigObject]().asJava, null)
+
+        assertEquals(0, merged.keySet().size)
+    }
+
+    @Test
+    def mergeOne() {
+        val obj1 = parseObject("""{ "a" : 1 }""")
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1).asJava, null)
+
+        assertEquals(1, merged.getInt("a"))
+        assertEquals(1, merged.keySet().size)
+    }
+
+    @Test
+    def mergeOverride() {
+        val obj1 = parseObject("""{ "a" : 1 }""")
+        val obj2 = parseObject("""{ "a" : 2 }""")
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2).asJava, null)
+
+        assertEquals(1, merged.getInt("a"))
+        assertEquals(1, merged.keySet().size)
+
+        val merged2 = AbstractConfigObject.merge(fakeOrigin(), List(obj2, obj1).asJava, null)
+
+        assertEquals(2, merged2.getInt("a"))
+        assertEquals(1, merged2.keySet().size)
+    }
+
+    @Test
+    def mergeN() {
+        val obj1 = parseObject("""{ "a" : 1 }""")
+        val obj2 = parseObject("""{ "b" : 2 }""")
+        val obj3 = parseObject("""{ "c" : 3 }""")
+        val obj4 = parseObject("""{ "d" : 4 }""")
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2, obj3, obj4).asJava, null)
+
+        assertEquals(1, merged.getInt("a"))
+        assertEquals(2, merged.getInt("b"))
+        assertEquals(3, merged.getInt("c"))
+        assertEquals(4, merged.getInt("d"))
+        assertEquals(4, merged.keySet().size)
+    }
+
+    @Test
+    def mergeOverrideN() {
+        val obj1 = parseObject("""{ "a" : 1 }""")
+        val obj2 = parseObject("""{ "a" : 2 }""")
+        val obj3 = parseObject("""{ "a" : 3 }""")
+        val obj4 = parseObject("""{ "a" : 4 }""")
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2, obj3, obj4).asJava, null)
+
+        assertEquals(1, merged.getInt("a"))
+        assertEquals(1, merged.keySet().size)
+
+        val merged2 = AbstractConfigObject.merge(fakeOrigin(), List(obj4, obj3, obj2, obj1).asJava, null)
+
+        assertEquals(4, merged2.getInt("a"))
+        assertEquals(1, merged2.keySet().size)
+    }
+
+    @Test
+    def mergeNested() {
+        val obj1 = parseObject("""{ "root" : { "a" : 1, "z" : 101 } }""")
+        val obj2 = parseObject("""{ "root" : { "b" : 2, "z" : 102 } }""")
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2).asJava, null)
+
+        assertEquals(1, merged.getInt("root.a"))
+        assertEquals(2, merged.getInt("root.b"))
+        assertEquals(101, merged.getInt("root.z"))
+        assertEquals(1, merged.keySet().size)
+        assertEquals(3, merged.getObject("root").keySet().size)
+    }
+
+    @Test
+    def mergeOverrideObjectAndPrimitive() {
+        val obj1 = parseObject("""{ "a" : 1 }""")
+        val obj2 = parseObject("""{ "a" : { "b" : 42 } }""")
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2).asJava, null)
+
+        assertEquals(1, merged.getInt("a"))
+        assertEquals(1, merged.keySet().size)
+
+        val merged2 = AbstractConfigObject.merge(fakeOrigin(), List(obj2, obj1).asJava, null)
+
+        assertEquals(42, merged2.getObject("a").getInt("b"))
+        assertEquals(42, merged2.getInt("a.b"))
+        assertEquals(1, merged2.keySet().size)
+        assertEquals(1, merged2.getObject("a").keySet().size)
+    }
+
+    @Test
+    def mergeObjectThenPrimitiveThenObject() {
+        val obj1 = parseObject("""{ "a" : { "b" : 42 } }""")
+        val obj2 = parseObject("""{ "a" : 2 }""")
+        val obj3 = parseObject("""{ "a" : { "b" : 43 } }""")
+
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2, obj3).asJava, null)
+
+        assertEquals(42, merged.getInt("a.b"))
+        assertEquals(1, merged.keySet().size)
+        assertEquals(1, merged.getObject("a").keySet().size())
+    }
+
+    @Test
+    def mergePrimitiveThenObjectThenPrimitive() {
+        val obj1 = parseObject("""{ "a" : 1 }""")
+        val obj2 = parseObject("""{ "a" : { "b" : 42 } }""")
+        val obj3 = parseObject("""{ "a" : 3 }""")
+
+        val merged = AbstractConfigObject.merge(fakeOrigin(), List(obj1, obj2, obj3).asJava, null)
+
+        assertEquals(1, merged.getInt("a"))
+        assertEquals(1, merged.keySet().size)
+    }
+
+    @Test
+    def test01() {
+        val conf = Config.load("test01")
+
+        // get all the primitive types
+        assertEquals(42, conf.getInt("ints.fortyTwo"))
+        assertEquals(42, conf.getInt("ints.fortyTwoAgain"))
+        assertEquals(42L, conf.getLong("ints.fortyTwoAgain"))
+        assertEquals(42.1, conf.getDouble("floats.fortyTwoPointOne"), 1e-6)
+        assertEquals(42.1, conf.getDouble("floats.fortyTwoPointOneAgain"), 1e-6)
+        assertEquals("abcd", conf.getString("strings.abcd"))
+        assertEquals("abcd", conf.getString("strings.abcdAgain"))
+        assertEquals("null bar 42 baz true 3.14 hi", conf.getString("strings.concatenated"))
+        assertEquals(true, conf.getBoolean("booleans.trueAgain"))
+        assertEquals(false, conf.getBoolean("booleans.falseAgain"))
+        // FIXME need to add a way to get a null
+        //assertEquals(null, conf.getAny("nulls.null"))
+
+        // get empty array as any type of array
+        assertEquals(Seq(), conf.getAnyList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getIntList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getLongList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getStringList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getLongList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getDoubleList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getObjectList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getBooleanList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getNumberList("arrays.empty").asScala)
+        assertEquals(Seq(), conf.getList("arrays.empty").asScala)
+
+        // get typed arrays
+        assertEquals(Seq(1, 2, 3), conf.getIntList("arrays.ofInt").asScala)
+        assertEquals(Seq(1L, 2L, 3L), conf.getLongList("arrays.ofInt").asScala)
+        assertEquals(Seq("a", "b", "c"), conf.getStringList("arrays.ofString").asScala)
+        assertEquals(Seq(3.14, 4.14, 5.14), conf.getDoubleList("arrays.ofDouble").asScala)
+        assertEquals(Seq(null, null, null), conf.getAnyList("arrays.ofNull").asScala)
+        assertEquals(Seq(true, false), conf.getBooleanList("arrays.ofBoolean").asScala)
+        val listOfLists = conf.getAnyList("arrays.ofArray").asScala map { _.asInstanceOf[java.util.List[_]].asScala }
+        assertEquals(Seq(Seq("a", "b", "c"), Seq("a", "b", "c"), Seq("a", "b", "c")), listOfLists)
+        assertEquals(3, conf.getObjectList("arrays.ofObject").asScala.length)
+
+        // plain getList should work
+        assertEquals(Seq(intValue(1), intValue(2), intValue(3)), conf.getList("arrays.ofInt").asScala)
+        assertEquals(Seq(stringValue("a"), stringValue("b"), stringValue("c")), conf.getList("arrays.ofString").asScala)
+
+        // should throw Missing if key doesn't exist
+        intercept[ConfigException.Missing] {
+            conf.getInt("doesnotexist")
+        }
+
+        // should throw Null if key is null
+        intercept[ConfigException.Null] {
+            conf.getInt("nulls.null")
+        }
+
+        // should throw WrongType if key is wrong type and not convertible
+        intercept[ConfigException.WrongType] {
+            conf.getInt("booleans.trueAgain")
+        }
+
+        // should convert numbers to string
+        assertEquals("42", conf.getString("ints.fortyTwo"))
+        assertEquals("42.1", conf.getString("floats.fortyTwoPointOne"))
+
+        // should convert string to number
+        assertEquals(57, conf.getInt("strings.number"))
+
+        // should get durations
+        def asNanos(secs: Int) = TimeUnit.SECONDS.toNanos(secs)
+        assertEquals(1000L, conf.getMilliseconds("durations.second"))
+        assertEquals(asNanos(1), conf.getNanoseconds("durations.second"))
+        assertEquals(Seq(1000L, 2000L, 3000L),
+            conf.getMillisecondsList("durations.secondsList").asScala)
+        assertEquals(Seq(asNanos(1), asNanos(2), asNanos(3)),
+            conf.getNanosecondsList("durations.secondsList").asScala)
+
+        // should get size in bytes
+        assertEquals(1024 * 1024L, conf.getMemorySize("memsizes.meg"))
+        assertEquals(Seq(1024 * 1024L, 1024 * 1024L),
+            conf.getMemorySizeList("memsizes.megsList").asScala)
+
+        // should have loaded stuff from .json
+        assertEquals(1, conf.getInt("fromJson1"))
+        assertEquals("A", conf.getString("fromJsonA"))
+
+        // should have loaded stuff from .properties
+        assertEquals("abc", conf.getString("fromProps.abc"))
+        assertEquals(1, conf.getInt("fromProps.one"))
+        assertEquals(true, conf.getBoolean("fromProps.bool"))
+
+        // toString() on conf objects doesn't throw (toString is just a debug string so not testing its result)
+        conf.toString()
+    }
+}

src/test/scala/com/typesafe/config/impl/TestUtils.scala

@@ -157,4 +157,15 @@ abstract trait TestUtils {
             }
         }
     }
+
+    protected def intValue(i: Int) = new ConfigInt(fakeOrigin(), i)
+    protected def boolValue(b: Boolean) = new ConfigBoolean(fakeOrigin(), b)
+    protected def nullValue() = new ConfigNull(fakeOrigin())
+    protected def stringValue(s: String) = new ConfigString(fakeOrigin(), s)
+    protected def doubleValue(d: Double) = new ConfigDouble(fakeOrigin(), d)
+
+    protected def parseObject(s: String) = {
+        Parser.parse(SyntaxFlavor.CONF, new SimpleConfigOrigin("test string"), s).asInstanceOf[AbstractConfigObject]
+    }
+
 }