Merge remote-tracking branch 'origin/master' into wip/havocp-resolve-fixes

Conflicts:
	config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java

HOCON.md

@@ -863,7 +863,7 @@ usual the comma may be omitted if there's a newline).
 
 If an unquoted `include` at the start of a key is followed by
 anything other than a single quoted string or the
-`url("")`/`file("")/`classpath("")` syntax, it is invalid and an
+`url("")`/`file("")`/`classpath("")` syntax, it is invalid and an
 error should be generated.
 
 There can be any amount of whitespace, including newlines, between
@@ -1209,9 +1209,9 @@ parsed as a number plus an optional unit string.
 The supported unit strings for duration are case sensitive and
 must be lowercase. Exactly these strings are supported:
 
- - `ns`, `nanosecond`, `nanoseconds`
+ - `ns`, `nano`, `nanos`, `nanosecond`, `nanoseconds`
- - `us`, `microsecond`, `microseconds`
+ - `us`, `micro`, `micros`, `microsecond`, `microseconds`
- - `ms`, `millisecond`, `milliseconds`
+ - `ms`, `milli`, `millis`, `millisecond`, `milliseconds`
  - `s`, `second`, `seconds`
  - `m`, `minute`, `minutes`
  - `h`, `hour`, `hours`
@@ -1274,6 +1274,13 @@ spec copies that. You can certainly find examples of mapping these
 to powers of ten, though. If you don't like ambiguity, don't use
 the single-letter abbreviations.
 
+Note: any value in zetta/zebi or yotta/yobi will overflow a 64-bit
+integer, and of course large-enough values in any of the units may
+overflow. Most real-world APIs and apps will not support byte
+counts that overflow a 64-bit integer. The huge units are provided
+just to be complete but probably aren't useful in practice. At
+least not in 2014.
+
 ### Config object merging and file merging
 
 It may be useful to offer a method to merge two objects. If such a

README.md

@@ -92,6 +92,10 @@ Maven Central.
         <version>1.2.1</version>
     </dependency>
 
+sbt dependency:
+
+    libraryDependencies += "com.typesafe" % "config" % "1.2.1"
+
 Link for direct download if you don't use a dependency manager:
 
  - http://central.maven.org/maven2/com/typesafe/config/
@@ -743,3 +747,7 @@ format.
 #### Ruby port
 
    * https://github.com/cprice404/ruby-hocon
+
+#### Python port
+
+   * pyhocon https://github.com/chimpler/pyhocon

config/build.sbt

@@ -16,7 +16,7 @@ crossPaths := false
 
 libraryDependencies += "net.liftweb" %% "lift-json" % "2.5" % "test"
 
-libraryDependencies += "com.novocode" % "junit-interface" % "0.10-M4" % "test"
+libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test"
 
 externalResolvers += "Scala Tools Snapshots" at "http://scala-tools.org/repo-snapshots/"
 

config/src/main/java/com/typesafe/config/ConfigFactory.java

@@ -591,10 +591,35 @@ public final class ConfigFactory {
         return parseURL(url, ConfigParseOptions.defaults());
     }
 
+    /**
+     * Parses a file into a Config instance. Does not call
+     * {@link Config#resolve} or merge the file with any other
+     * configuration; this method parses a single file and does
+     * nothing else. It does process "include" statements in the
+     * parsed file, and may end up doing other IO due to those
+     * statements.
+     *
+     * @param file
+     *       the file to parse
+     * @param options
+     *       parse options to control how the file is interpreted
+     * @return the parsed configuration
+     * @throws ConfigException on IO or parse errors
+     */
     public static Config parseFile(File file, ConfigParseOptions options) {
         return Parseable.newFile(file, options).parse().toConfig();
     }
 
+    /**
+     * Parses a file into a Config instance as with
+     * {@link #parseFile(File,ConfigParseOptions)} but always uses the
+     * default parse options.
+     *
+     * @param file
+     *       the file to parse
+     * @return the parsed configuration
+     * @throws ConfigException on IO or parse errors
+     */
     public static Config parseFile(File file) {
         return parseFile(file, ConfigParseOptions.defaults());
     }
@@ -637,6 +662,14 @@ public final class ConfigFactory {
         return ConfigImpl.parseFileAnySyntax(fileBasename, options).toConfig();
     }
 
+    /**
+     * Like {@link #parseFileAnySyntax(File,ConfigParseOptions)} but always uses
+     * default parse options.
+     *
+     * @param fileBasename
+     *            a filename with or without extension
+     * @return the parsed configuration
+     */
     public static Config parseFileAnySyntax(File fileBasename) {
         return parseFileAnySyntax(fileBasename, ConfigParseOptions.defaults());
     }

config/src/main/java/com/typesafe/config/impl/SimpleConfig.java

@@ -5,6 +5,8 @@ package com.typesafe.config.impl;
 
 import java.io.ObjectStreamException;
 import java.io.Serializable;
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -512,12 +514,12 @@ final class SimpleConfig implements Config, MergeableValue, Serializable {
             unitString = unitString + "s";
 
         // note that this is deliberately case-sensitive
-        if (unitString.equals("") || unitString.equals("ms")
+        if (unitString.equals("") || unitString.equals("ms") || unitString.equals("millis")
                 || unitString.equals("milliseconds")) {
             units = TimeUnit.MILLISECONDS;
-        } else if (unitString.equals("us") || unitString.equals("microseconds")) {
+        } else if (unitString.equals("us") || unitString.equals("micros") || unitString.equals("microseconds")) {
             units = TimeUnit.MICROSECONDS;
-        } else if (unitString.equals("ns") || unitString.equals("nanoseconds")) {
+        } else if (unitString.equals("ns") || unitString.equals("nanos") || unitString.equals("nanoseconds")) {
             units = TimeUnit.NANOSECONDS;
         } else if (unitString.equals("d") || unitString.equals("days")) {
             units = TimeUnit.DAYS;
@@ -575,19 +577,13 @@ final class SimpleConfig implements Config, MergeableValue, Serializable {
         final String prefix;
         final int powerOf;
         final int power;
-        final long bytes;
+        final BigInteger bytes;
 
         MemoryUnit(String prefix, int powerOf, int power) {
             this.prefix = prefix;
             this.powerOf = powerOf;
             this.power = power;
-            int i = power;
+            this.bytes = BigInteger.valueOf(powerOf).pow(power);
-            long bytes = 1;
-            while (i > 0) {
-                bytes *= powerOf;
-                --i;
-            }
-            this.bytes = bytes;
         }
 
         private static Map<String, MemoryUnit> makeUnitsMap() {
@@ -667,13 +663,20 @@ final class SimpleConfig implements Config, MergeableValue, Serializable {
         }
 
         try {
+            BigInteger result;
             // if the string is purely digits, parse as an integer to avoid
             // possible precision loss; otherwise as a double.
             if (numberString.matches("[0-9]+")) {
-                return Long.parseLong(numberString) * units.bytes;
+                result = units.bytes.multiply(new BigInteger(numberString));
             } else {
-                return (long) (Double.parseDouble(numberString) * units.bytes);
+                BigDecimal resultDecimal = (new BigDecimal(units.bytes)).multiply(new BigDecimal(numberString));
+                result = resultDecimal.toBigInteger();
             }
+            if (result.bitLength() < 64)
+                return result.longValue();
+            else
+                throw new ConfigException.BadValue(originForException, pathForException,
+                        "size-in-bytes value is out of range for a 64-bit long: '" + input + "'");
         } catch (NumberFormatException e) {
             throw new ConfigException.BadValue(originForException, pathForException,
                     "Could not parse size-in-bytes number '" + numberString + "'");

config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java

@@ -115,7 +115,11 @@ final class SimpleConfigList extends AbstractConfigValue implements ConfigList,
         }
 
         if (changed != null) {
-            return new SimpleConfigList(origin(), changed, newResolveStatus);
+            if (newResolveStatus != null) {
+                return new SimpleConfigList(origin(), changed, newResolveStatus);
+            } else {
+                return new SimpleConfigList(origin(), changed);
+            }
         } else {
             return this;
         }
@@ -151,7 +155,7 @@ final class SimpleConfigList extends AbstractConfigValue implements ConfigList,
         } else {
             try {
                 ResolveModifier modifier = new ResolveModifier(context, source.pushParent(this));
-                SimpleConfigList value = modifyMayThrow(modifier, ResolveStatus.RESOLVED);
+                SimpleConfigList value = modifyMayThrow(modifier, context.options().getAllowUnresolved() ? null : ResolveStatus.RESOLVED);
                 return ResolveResult.make(modifier.context, value);
             } catch (NotPossibleToResolve e) {
                 throw e;

config/src/main/java/com/typesafe/config/impl/Tokenizer.java

@@ -457,23 +457,22 @@ final class Tokenizer {
         private Token pullQuotedString() throws ProblemException {
             // the open quote has already been consumed
             StringBuilder sb = new StringBuilder();
-            int c = '\0'; // value doesn't get used
+            while (true) {
-            do {
+                int c = nextCharRaw();
-                c = nextCharRaw();
                 if (c == -1)
                     throw problem("End of input but string quote was still open");
 
                 if (c == '\\') {
                     pullEscapeSequence(sb);
                 } else if (c == '"') {
-                    // end the loop, done!
+                    break;
                 } else if (Character.isISOControl(c)) {
                     throw problem(asString(c), "JSON does not allow unescaped " + asString(c)
                             + " in quoted strings, use a backslash escape");
                 } else {
                     sb.appendCodePoint(c);
                 }
-            } while (c != '"');
+            }
 
             // maybe switch to triple-quoted string, sort of hacky...
             if (sb.length() == 0) {

config/src/test/resources/test01.conf

@@ -54,7 +54,9 @@
         "second" : 1s,
         "secondsList" : [1s,2seconds,3 s, 4000],
         "secondAsNumber" : 1000,
-        "halfSecond" : 0.5s
+        "halfSecond" : 0.5s,
+        "millis" : 1 milli,
+        "micros" : 2000 micros
     },
 
     "memsizes" : {

config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala

@@ -769,6 +769,8 @@ class ConfigTest extends TestUtils {
             assertEquals(Seq(1, 2, 3, 4) map s2unit,
                 conf.getDurationList("durations.secondsList", unit).asScala)
             assertEquals(ms2unit(500L), conf.getDuration("durations.halfSecond", unit))
+            assertEquals(ms2unit(1L), conf.getDuration("durations.millis", unit))
+            assertEquals(ms2unit(2L), conf.getDuration("durations.micros", unit))
         }
 
         assertDurationAsTimeUnit(NANOSECONDS)
@@ -1118,6 +1120,15 @@ class ConfigTest extends TestUtils {
         assertTrue("after resolution, config is now resolved", resolved2.isResolved)
     }
 
+    @Test
+    def allowUnresolvedDoesAllowUnresolvedArrayElements() {
+        val values = ConfigFactory.parseString("unknown = [someVal], known = 42")
+        val unresolved = ConfigFactory.parseString("concat = [${unknown}[]], sibling = [${unknown}, ${known}]")
+        unresolved.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+        unresolved.withFallback(values).resolve()
+        unresolved.resolveWith(values)
+    }
+
     @Test
     def allowUnresolvedDoesAllowUnresolved() {
         val values = ConfigFactory.parseString("{ foo = 1, bar = 2, m = 3, n = 4}")

config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala

@@ -50,16 +50,19 @@ class UnitParserTest extends TestUtils {
 
         val conf = parseConfig("foo = 1d")
         assertEquals("could get 1d from conf as days",
-                     1L, conf.getDuration("foo", TimeUnit.DAYS))
+            1L, conf.getDuration("foo", TimeUnit.DAYS))
         assertEquals("could get 1d from conf as nanos",
-                     dayInNanos, conf.getNanoseconds("foo"))
+            dayInNanos, conf.getNanoseconds("foo"))
         assertEquals("could get 1d from conf as millis",
-                     TimeUnit.DAYS.toMillis(1), conf.getMilliseconds("foo"))
+            TimeUnit.DAYS.toMillis(1), conf.getMilliseconds("foo"))
     }
 
     @Test
     def parseMemorySizeInBytes(): Unit = {
-        def parseMem(s: String) = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
+        def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
+
+        assertEquals(Long.MaxValue, parseMem(s"${Long.MaxValue} bytes"))
+        assertEquals(Long.MinValue, parseMem(s"${Long.MinValue} bytes"))
 
         val oneMebis = List("1048576", "1048576b", "1048576bytes", "1048576byte",
             "1048576  b", "1048576  bytes",
@@ -88,7 +91,7 @@ class UnitParserTest extends TestUtils {
         }
 
         var result = 1024L * 1024 * 1024
-        for (unit <- Seq("tebi", "pebi", "exbi", "zebi", "yobi")) {
+        for (unit <- Seq("tebi", "pebi", "exbi")) {
             val first = unit.substring(0, 1).toUpperCase()
             result = result * 1024;
             assertEquals(result, parseMem("1" + first))
@@ -99,7 +102,7 @@ class UnitParserTest extends TestUtils {
         }
 
         result = 1000L * 1000 * 1000
-        for (unit <- Seq("tera", "peta", "exa", "zetta", "yotta")) {
+        for (unit <- Seq("tera", "peta", "exa")) {
             val first = unit.substring(0, 1).toUpperCase()
             result = result * 1000;
             assertEquals(result, parseMem("1" + first + "B"))
@@ -119,4 +122,45 @@ class UnitParserTest extends TestUtils {
         }
         assertTrue(e2.getMessage().contains("size-in-bytes number"))
     }
+
+    // later on we'll want to check this with BigInteger version of getBytes
+    @Test
+    def parseHugeMemorySizes(): Unit = {
+        def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
+        def assertOutOfRange(s: String) = {
+            val fail = intercept[ConfigException.BadValue] {
+                parseMem(s)
+            }
+            assertTrue("number was too big", fail.getMessage.contains("out of range"))
+        }
+
+        import java.math.BigInteger
+        assertOutOfRange(s"${BigInteger.valueOf(Long.MaxValue).add(BigInteger.valueOf(1)).toString} bytes")
+        assertOutOfRange(s"${BigInteger.valueOf(Long.MinValue).subtract(BigInteger.valueOf(1)).toString} bytes")
+
+        var result = 1024L * 1024 * 1024
+        for (unit <- Seq("zebi", "yobi")) {
+            val first = unit.substring(0, 1).toUpperCase()
+            assertOutOfRange("1" + first)
+            assertOutOfRange("1" + first + "i")
+            assertOutOfRange("1" + first + "iB")
+            assertOutOfRange("1" + unit + "byte")
+            assertOutOfRange("1" + unit + "bytes")
+            assertOutOfRange("1.1" + first)
+            assertOutOfRange("-1" + first)
+        }
+
+        result = 1000L * 1000 * 1000
+        for (unit <- Seq("zetta", "yotta")) {
+            val first = unit.substring(0, 1).toUpperCase()
+            assertOutOfRange("1" + first + "B")
+            assertOutOfRange("1" + unit + "byte")
+            assertOutOfRange("1" + unit + "bytes")
+            assertOutOfRange("1.1" + first + "B")
+            assertOutOfRange("-1" + first + "B")
+        }
+
+        assertOutOfRange("1000 exabytes")
+        assertOutOfRange("10000000 petabytes")
+    }
 }

project/build.properties

@@ -1 +1 @@
-sbt.version=0.13.1
+sbt.version=0.13.7