From 6311ef8d4deeb0daee2ba8b20a6f61472c63a970 Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Mon, 23 Jun 2014 12:13:40 -0400 Subject: [PATCH 01/15] Throw an exception if size-in-bytes values are out of Long range Fixes #170 --- HOCON.md | 7 +++ .../typesafe/config/impl/SimpleConfig.java | 31 +++++++++---- .../typesafe/config/impl/UnitParserTest.scala | 46 +++++++++++++++++-- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/HOCON.md b/HOCON.md index d439fed6..2f17cedf 100644 --- a/HOCON.md +++ b/HOCON.md @@ -1265,6 +1265,13 @@ spec copies that. You can certainly find examples of mapping these to powers of ten, though. If you don't like ambiguity, don't use the single-letter abbreviations. +Note: any value in zetta/zebi or yotta/yobi will overflow a 64-bit +integer, and of course large-enough values in any of the units may +overflow. Most real-world APIs and apps will not support byte +counts that overflow a 64-bit integer. The huge units are provided +just to be complete but probably aren't useful in practice. At +least not in 2014. + ### Config object merging and file merging It may be useful to offer a method to merge two objects. If such a diff --git a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java index 277c2d04..7cea76df 100644 --- a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java +++ b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java @@ -5,6 +5,8 @@ package com.typesafe.config.impl; import java.io.ObjectStreamException; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; @@ -575,19 +577,13 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { final String prefix; final int powerOf; final int power; - final long bytes; + final BigInteger bytes; MemoryUnit(String prefix, int powerOf, int power) { this.prefix = prefix; this.powerOf = powerOf; this.power = power; - int i = power; - long bytes = 1; - while (i > 0) { - bytes *= powerOf; - --i; - } - this.bytes = bytes; + this.bytes = BigInteger.valueOf(powerOf).pow(power); } private static Map makeUnitsMap() { @@ -628,6 +624,12 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { } } + private static boolean isValidLong(BigInteger v) { + BigInteger max = BigInteger.valueOf(Long.MAX_VALUE); + BigInteger min = BigInteger.valueOf(Long.MIN_VALUE); + return max.compareTo(v) >= 0 && min.compareTo(v) <= 0; + } + /** * Parses a size-in-bytes string. If no units are specified in the string, * it is assumed to be in bytes. The returned value is in bytes. The purpose @@ -667,13 +669,22 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { } try { + BigInteger result; // if the string is purely digits, parse as an integer to avoid // possible precision loss; otherwise as a double. if (numberString.matches("[0-9]+")) { - return Long.parseLong(numberString) * units.bytes; + Long v = Long.parseLong(numberString); + result = units.bytes.multiply(BigInteger.valueOf(v)); } else { - return (long) (Double.parseDouble(numberString) * units.bytes); + Double v = Double.parseDouble(numberString); + BigDecimal resultDecimal = (new BigDecimal(units.bytes)).multiply(BigDecimal.valueOf(v)); + result = resultDecimal.toBigInteger(); } + if (isValidLong(result)) + return result.longValue(); + else + throw new ConfigException.BadValue(originForException, pathForException, + "size-in-bytes value is out of range for a 64-bit long: '" + input + "'"); } catch (NumberFormatException e) { throw new ConfigException.BadValue(originForException, pathForException, "Could not parse size-in-bytes number '" + numberString + "'"); diff --git a/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala b/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala index 4a6a324b..3106b55a 100644 --- a/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala @@ -50,11 +50,11 @@ class UnitParserTest extends TestUtils { val conf = parseConfig("foo = 1d") assertEquals("could get 1d from conf as days", - 1L, conf.getDuration("foo", TimeUnit.DAYS)) + 1L, conf.getDuration("foo", TimeUnit.DAYS)) assertEquals("could get 1d from conf as nanos", - dayInNanos, conf.getNanoseconds("foo")) + dayInNanos, conf.getNanoseconds("foo")) assertEquals("could get 1d from conf as millis", - TimeUnit.DAYS.toMillis(1), conf.getMilliseconds("foo")) + TimeUnit.DAYS.toMillis(1), conf.getMilliseconds("foo")) } @Test @@ -88,7 +88,7 @@ class UnitParserTest extends TestUtils { } var result = 1024L * 1024 * 1024 - for (unit <- Seq("tebi", "pebi", "exbi", "zebi", "yobi")) { + for (unit <- Seq("tebi", "pebi", "exbi")) { val first = unit.substring(0, 1).toUpperCase() result = result * 1024; assertEquals(result, parseMem("1" + first)) @@ -99,7 +99,7 @@ class UnitParserTest extends TestUtils { } result = 1000L * 1000 * 1000 - for (unit <- Seq("tera", "peta", "exa", "zetta", "yotta")) { + for (unit <- Seq("tera", "peta", "exa")) { val first = unit.substring(0, 1).toUpperCase() result = result * 1000; assertEquals(result, parseMem("1" + first + "B")) @@ -119,4 +119,40 @@ class UnitParserTest extends TestUtils { } assertTrue(e2.getMessage().contains("size-in-bytes number")) } + + // later on we'll want to check this with BigInteger version of getBytes + @Test + def parseHugeMemorySizes(): Unit = { + def parseMem(s: String) = SimpleConfig.parseBytes(s, fakeOrigin(), "test") + def assertOutOfRange(s: String) = { + val fail = intercept[ConfigException.BadValue] { + parseMem(s) + } + assertTrue("number was too big", fail.getMessage.contains("out of range")) + } + var result = 1024L * 1024 * 1024 + for (unit <- Seq("zebi", "yobi")) { + val first = unit.substring(0, 1).toUpperCase() + assertOutOfRange("1" + first) + assertOutOfRange("1" + first + "i") + assertOutOfRange("1" + first + "iB") + assertOutOfRange("1" + unit + "byte") + assertOutOfRange("1" + unit + "bytes") + assertOutOfRange("1.1" + first) + assertOutOfRange("-1" + first) + } + + result = 1000L * 1000 * 1000 + for (unit <- Seq("zetta", "yotta")) { + val first = unit.substring(0, 1).toUpperCase() + assertOutOfRange("1" + first + "B") + assertOutOfRange("1" + unit + "byte") + assertOutOfRange("1" + unit + "bytes") + assertOutOfRange("1.1" + first + "B") + assertOutOfRange("-1" + first + "B") + } + + assertOutOfRange("1000 exabytes") + assertOutOfRange("10000000 petabytes") + } } From 02867c8aec9d69b392750601820513899e9a5e31 Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Wed, 25 Jun 2014 16:52:14 -0400 Subject: [PATCH 02/15] Improve size-in-bytes parser's use of BigInteger/BigDecimal - use bitLength to see if we overflow Long - use BigInteger and BigDecimal to parse, instead of parseLong/parseDouble --- .../com/typesafe/config/impl/SimpleConfig.java | 14 +++----------- .../com/typesafe/config/impl/UnitParserTest.scala | 12 ++++++++++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java index 7cea76df..9f461e39 100644 --- a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java +++ b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java @@ -624,12 +624,6 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { } } - private static boolean isValidLong(BigInteger v) { - BigInteger max = BigInteger.valueOf(Long.MAX_VALUE); - BigInteger min = BigInteger.valueOf(Long.MIN_VALUE); - return max.compareTo(v) >= 0 && min.compareTo(v) <= 0; - } - /** * Parses a size-in-bytes string. If no units are specified in the string, * it is assumed to be in bytes. The returned value is in bytes. The purpose @@ -673,14 +667,12 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { // if the string is purely digits, parse as an integer to avoid // possible precision loss; otherwise as a double. if (numberString.matches("[0-9]+")) { - Long v = Long.parseLong(numberString); - result = units.bytes.multiply(BigInteger.valueOf(v)); + result = units.bytes.multiply(new BigInteger(numberString)); } else { - Double v = Double.parseDouble(numberString); - BigDecimal resultDecimal = (new BigDecimal(units.bytes)).multiply(BigDecimal.valueOf(v)); + BigDecimal resultDecimal = (new BigDecimal(units.bytes)).multiply(new BigDecimal(numberString)); result = resultDecimal.toBigInteger(); } - if (isValidLong(result)) + if (result.bitLength() < 64) return result.longValue(); else throw new ConfigException.BadValue(originForException, pathForException, diff --git a/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala b/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala index 3106b55a..12a1ce52 100644 --- a/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala @@ -59,7 +59,10 @@ class UnitParserTest extends TestUtils { @Test def parseMemorySizeInBytes(): Unit = { - def parseMem(s: String) = SimpleConfig.parseBytes(s, fakeOrigin(), "test") + def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test") + + assertEquals(Long.MaxValue, parseMem(s"${Long.MaxValue} bytes")) + assertEquals(Long.MinValue, parseMem(s"${Long.MinValue} bytes")) val oneMebis = List("1048576", "1048576b", "1048576bytes", "1048576byte", "1048576 b", "1048576 bytes", @@ -123,13 +126,18 @@ class UnitParserTest extends TestUtils { // later on we'll want to check this with BigInteger version of getBytes @Test def parseHugeMemorySizes(): Unit = { - def parseMem(s: String) = SimpleConfig.parseBytes(s, fakeOrigin(), "test") + def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test") def assertOutOfRange(s: String) = { val fail = intercept[ConfigException.BadValue] { parseMem(s) } assertTrue("number was too big", fail.getMessage.contains("out of range")) } + + import java.math.BigInteger + assertOutOfRange(s"${BigInteger.valueOf(Long.MaxValue).add(BigInteger.valueOf(1)).toString} bytes") + assertOutOfRange(s"${BigInteger.valueOf(Long.MinValue).subtract(BigInteger.valueOf(1)).toString} bytes") + var result = 1024L * 1024 * 1024 for (unit <- Seq("zebi", "yobi")) { val first = unit.substring(0, 1).toUpperCase() From 60c661d683b4ecfb0e4f0f7dacfe77815197ac51 Mon Sep 17 00:00:00 2001 From: Alex Wei Date: Fri, 8 Aug 2014 09:48:56 +1000 Subject: [PATCH 03/15] Added missing duration unit short names that are supported in scala.concurrent.duration.DurationConversions. --- .../main/java/com/typesafe/config/impl/SimpleConfig.java | 6 +++--- config/src/test/resources/test01.conf | 4 +++- .../test/scala/com/typesafe/config/impl/ConfigTest.scala | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java index 277c2d04..b202c90c 100644 --- a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java +++ b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java @@ -512,12 +512,12 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { unitString = unitString + "s"; // note that this is deliberately case-sensitive - if (unitString.equals("") || unitString.equals("ms") + if (unitString.equals("") || unitString.equals("ms") || unitString.equals("millis") || unitString.equals("milliseconds")) { units = TimeUnit.MILLISECONDS; - } else if (unitString.equals("us") || unitString.equals("microseconds")) { + } else if (unitString.equals("us") || unitString.equals("micros") || unitString.equals("microseconds")) { units = TimeUnit.MICROSECONDS; - } else if (unitString.equals("ns") || unitString.equals("nanoseconds")) { + } else if (unitString.equals("ns") || unitString.equals("nanos") || unitString.equals("nanoseconds")) { units = TimeUnit.NANOSECONDS; } else if (unitString.equals("d") || unitString.equals("days")) { units = TimeUnit.DAYS; diff --git a/config/src/test/resources/test01.conf b/config/src/test/resources/test01.conf index eae7f69e..586d23f3 100644 --- a/config/src/test/resources/test01.conf +++ b/config/src/test/resources/test01.conf @@ -54,7 +54,9 @@ "second" : 1s, "secondsList" : [1s,2seconds,3 s, 4000], "secondAsNumber" : 1000, - "halfSecond" : 0.5s + "halfSecond" : 0.5s, + "millis" : 1 milli, + "micros" : 2000 micros }, "memsizes" : { diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index a68f3897..7ca5fa66 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -769,6 +769,8 @@ class ConfigTest extends TestUtils { assertEquals(Seq(1, 2, 3, 4) map s2unit, conf.getDurationList("durations.secondsList", unit).asScala) assertEquals(ms2unit(500L), conf.getDuration("durations.halfSecond", unit)) + assertEquals(ms2unit(1L), conf.getDuration("durations.millis", unit)) + assertEquals(ms2unit(2L), conf.getDuration("durations.micros", unit)) } assertDurationAsTimeUnit(NANOSECONDS) From ebb526304ecebb561ba5c5f4ff4ce369b303e6e0 Mon Sep 17 00:00:00 2001 From: Alex Wei Date: Fri, 8 Aug 2014 09:57:42 +1000 Subject: [PATCH 04/15] Added more duration unit short names for hour, minute and second. --- .../main/java/com/typesafe/config/impl/SimpleConfig.java | 6 +++--- config/src/test/resources/test01.conf | 5 ++++- .../test/scala/com/typesafe/config/impl/ConfigTest.scala | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java index b202c90c..b3fe1135 100644 --- a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java +++ b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java @@ -521,11 +521,11 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { units = TimeUnit.NANOSECONDS; } else if (unitString.equals("d") || unitString.equals("days")) { units = TimeUnit.DAYS; - } else if (unitString.equals("h") || unitString.equals("hours")) { + } else if (unitString.equals("h") || unitString.equals("hours") || unitString.equals("hr") || unitString.equals("hrs")) { units = TimeUnit.HOURS; - } else if (unitString.equals("s") || unitString.equals("seconds")) { + } else if (unitString.equals("s") || unitString.equals("seconds") || unitString.equals("secs")) { units = TimeUnit.SECONDS; - } else if (unitString.equals("m") || unitString.equals("minutes")) { + } else if (unitString.equals("m") || unitString.equals("minutes") || unitString.equals("mins")) { units = TimeUnit.MINUTES; } else { throw new ConfigException.BadValue(originForException, diff --git a/config/src/test/resources/test01.conf b/config/src/test/resources/test01.conf index 586d23f3..2e56d24b 100644 --- a/config/src/test/resources/test01.conf +++ b/config/src/test/resources/test01.conf @@ -56,7 +56,10 @@ "secondAsNumber" : 1000, "halfSecond" : 0.5s, "millis" : 1 milli, - "micros" : 2000 micros + "micros" : 2000 micros, + "secs" : 2 sec, + "mins" : 1 min, + "hrs" : 1 hr }, "memsizes" : { diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index 7ca5fa66..849dd932 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -771,6 +771,9 @@ class ConfigTest extends TestUtils { assertEquals(ms2unit(500L), conf.getDuration("durations.halfSecond", unit)) assertEquals(ms2unit(1L), conf.getDuration("durations.millis", unit)) assertEquals(ms2unit(2L), conf.getDuration("durations.micros", unit)) + assertEquals(ms2unit(2000L), conf.getDuration("durations.secs", unit)) + assertEquals(ms2unit(60000L), conf.getDuration("durations.mins", unit)) + assertEquals(ms2unit(3600000L), conf.getDuration("durations.hrs", unit)) } assertDurationAsTimeUnit(NANOSECONDS) From c8cab8c79ef21507986fdc4118624e50e2b8ad15 Mon Sep 17 00:00:00 2001 From: Alex Wei Date: Mon, 11 Aug 2014 18:31:14 +1000 Subject: [PATCH 05/15] Revert "Added more duration unit short names for hour, minute and second." This reverts commit ebb526304ecebb561ba5c5f4ff4ce369b303e6e0. --- .../main/java/com/typesafe/config/impl/SimpleConfig.java | 6 +++--- config/src/test/resources/test01.conf | 5 +---- .../test/scala/com/typesafe/config/impl/ConfigTest.scala | 3 --- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java index b3fe1135..b202c90c 100644 --- a/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java +++ b/config/src/main/java/com/typesafe/config/impl/SimpleConfig.java @@ -521,11 +521,11 @@ final class SimpleConfig implements Config, MergeableValue, Serializable { units = TimeUnit.NANOSECONDS; } else if (unitString.equals("d") || unitString.equals("days")) { units = TimeUnit.DAYS; - } else if (unitString.equals("h") || unitString.equals("hours") || unitString.equals("hr") || unitString.equals("hrs")) { + } else if (unitString.equals("h") || unitString.equals("hours")) { units = TimeUnit.HOURS; - } else if (unitString.equals("s") || unitString.equals("seconds") || unitString.equals("secs")) { + } else if (unitString.equals("s") || unitString.equals("seconds")) { units = TimeUnit.SECONDS; - } else if (unitString.equals("m") || unitString.equals("minutes") || unitString.equals("mins")) { + } else if (unitString.equals("m") || unitString.equals("minutes")) { units = TimeUnit.MINUTES; } else { throw new ConfigException.BadValue(originForException, diff --git a/config/src/test/resources/test01.conf b/config/src/test/resources/test01.conf index 2e56d24b..586d23f3 100644 --- a/config/src/test/resources/test01.conf +++ b/config/src/test/resources/test01.conf @@ -56,10 +56,7 @@ "secondAsNumber" : 1000, "halfSecond" : 0.5s, "millis" : 1 milli, - "micros" : 2000 micros, - "secs" : 2 sec, - "mins" : 1 min, - "hrs" : 1 hr + "micros" : 2000 micros }, "memsizes" : { diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index 849dd932..7ca5fa66 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -771,9 +771,6 @@ class ConfigTest extends TestUtils { assertEquals(ms2unit(500L), conf.getDuration("durations.halfSecond", unit)) assertEquals(ms2unit(1L), conf.getDuration("durations.millis", unit)) assertEquals(ms2unit(2L), conf.getDuration("durations.micros", unit)) - assertEquals(ms2unit(2000L), conf.getDuration("durations.secs", unit)) - assertEquals(ms2unit(60000L), conf.getDuration("durations.mins", unit)) - assertEquals(ms2unit(3600000L), conf.getDuration("durations.hrs", unit)) } assertDurationAsTimeUnit(NANOSECONDS) From b76b406f2b0efce8f06d3f6a1c6eff08fc207715 Mon Sep 17 00:00:00 2001 From: Alex Wei Date: Mon, 11 Aug 2014 18:36:14 +1000 Subject: [PATCH 06/15] Patch spec with duration units from Scala that were previously missing. --- HOCON.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HOCON.md b/HOCON.md index d439fed6..ff5425e9 100644 --- a/HOCON.md +++ b/HOCON.md @@ -1200,9 +1200,9 @@ parsed as a number plus an optional unit string. The supported unit strings for duration are case sensitive and must be lowercase. Exactly these strings are supported: - - `ns`, `nanosecond`, `nanoseconds` - - `us`, `microsecond`, `microseconds` - - `ms`, `millisecond`, `milliseconds` + - `ns`, `nano`, `nanos`, `nanosecond`, `nanoseconds` + - `us`, `micro`, `micros`, `microsecond`, `microseconds` + - `ms`, `milli`, `millis`, `millisecond`, `milliseconds` - `s`, `second`, `seconds` - `m`, `minute`, `minutes` - `h`, `hour`, `hours` From c1d5c1188719a7624430b76fb6b0ade38f064a5f Mon Sep 17 00:00:00 2001 From: Kornel Kielczewski Date: Tue, 16 Sep 2014 09:55:51 +0200 Subject: [PATCH 07/15] Added sbt library dependencies to README.md - Lots of people use sbt, why keep a maven sample and forget about sbt? :) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index c2775604..9806695c 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,10 @@ Maven Central. 1.2.1 +sbt dependency: + + libraryDependencies += "com.typesafe" % "config" % "1.2.1" + Obsolete releases are here, but you probably don't want these: - http://repo.typesafe.com/typesafe/releases/com/typesafe/config/config/ From b83089b91d29b3054cebd77ee5b8a09fbe13bb87 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Wed, 5 Nov 2014 14:35:42 -0500 Subject: [PATCH 08/15] Avoid awkward do-while in Tokenizer.pullQuotedString Not a big deal, but I think this reads more clearly now. --- .../main/java/com/typesafe/config/impl/Tokenizer.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/Tokenizer.java b/config/src/main/java/com/typesafe/config/impl/Tokenizer.java index 0da23070..03460ed7 100644 --- a/config/src/main/java/com/typesafe/config/impl/Tokenizer.java +++ b/config/src/main/java/com/typesafe/config/impl/Tokenizer.java @@ -457,23 +457,22 @@ final class Tokenizer { private Token pullQuotedString() throws ProblemException { // the open quote has already been consumed StringBuilder sb = new StringBuilder(); - int c = '\0'; // value doesn't get used - do { - c = nextCharRaw(); + while (true) { + int c = nextCharRaw(); if (c == -1) throw problem("End of input but string quote was still open"); if (c == '\\') { pullEscapeSequence(sb); } else if (c == '"') { - // end the loop, done! + break; } else if (Character.isISOControl(c)) { throw problem(asString(c), "JSON does not allow unescaped " + asString(c) + " in quoted strings, use a backslash escape"); } else { sb.appendCodePoint(c); } - } while (c != '"'); + } // maybe switch to triple-quoted string, sort of hacky... if (sb.length() == 0) { From f3ec3aa9108df03e1cf147c6775fb5e0cb06e2cf Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Tue, 11 Nov 2014 14:49:47 -0500 Subject: [PATCH 09/15] Missing backtick in HOCON.md Reported by @takc923 --- HOCON.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOCON.md b/HOCON.md index d439fed6..595e1b24 100644 --- a/HOCON.md +++ b/HOCON.md @@ -854,7 +854,7 @@ usual the comma may be omitted if there's a newline). If an unquoted `include` at the start of a key is followed by anything other than a single quoted string or the -`url("")`/`file("")/`classpath("")` syntax, it is invalid and an +`url("")`/`file("")`/`classpath("")` syntax, it is invalid and an error should be generated. There can be any amount of whitespace, including newlines, between From 54e780d30854d23c94cdef75d0d81c9ae18e2cc0 Mon Sep 17 00:00:00 2001 From: Ben McCann Date: Mon, 1 Dec 2014 17:09:46 -0800 Subject: [PATCH 10/15] Upgrade to SBT 0.13.7 --- config/build.sbt | 2 +- project/build.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/build.sbt b/config/build.sbt index d98130a8..511a4fe1 100644 --- a/config/build.sbt +++ b/config/build.sbt @@ -16,7 +16,7 @@ crossPaths := false libraryDependencies += "net.liftweb" %% "lift-json" % "2.5" % "test" -libraryDependencies += "com.novocode" % "junit-interface" % "0.10-M4" % "test" +libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test" externalResolvers += "Scala Tools Snapshots" at "http://scala-tools.org/repo-snapshots/" diff --git a/project/build.properties b/project/build.properties index 37b489cb..748703f7 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.1 +sbt.version=0.13.7 From 5447501df0a8146da08a9283957502f275273a70 Mon Sep 17 00:00:00 2001 From: Francois Dang Ngoc Date: Sun, 7 Dec 2014 22:52:32 -0500 Subject: [PATCH 11/15] updated README.md to add link to python port (pyhocon) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3725cc5d..58e9b973 100644 --- a/README.md +++ b/README.md @@ -743,3 +743,7 @@ format. #### Ruby port * https://github.com/cprice404/ruby-hocon + +#### Python port + + * pyhocon https://github.com/chimpler/pyhocon From 5c464b3b5ac04d1a7b803434e42196327426a566 Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Thu, 18 Dec 2014 15:13:03 -0500 Subject: [PATCH 12/15] Document ConfigFactory.parseFile --- .../com/typesafe/config/ConfigFactory.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/config/src/main/java/com/typesafe/config/ConfigFactory.java b/config/src/main/java/com/typesafe/config/ConfigFactory.java index e59444bf..8ae6bf87 100644 --- a/config/src/main/java/com/typesafe/config/ConfigFactory.java +++ b/config/src/main/java/com/typesafe/config/ConfigFactory.java @@ -591,10 +591,35 @@ public final class ConfigFactory { return parseURL(url, ConfigParseOptions.defaults()); } + /** + * Parses a file into a Config instance. Does not call + * {@link Config#resolve} or merge the file with any other + * configuration; this method parses a single file and does + * nothing else. It does process "include" statements in the + * parsed file, and may end up doing other IO due to those + * statements. + * + * @param file + * the file to parse + * @param options + * parse options to control how the file is interpreted + * @return the parsed configuration + * @throws ConfigException on IO or parse errors + */ public static Config parseFile(File file, ConfigParseOptions options) { return Parseable.newFile(file, options).parse().toConfig(); } + /** + * Parses a file into a Config instance as with + * {@link #parseFile(File,ConfigParseOptions)} but always uses the + * default parse options. + * + * @param file + * the file to parse + * @return the parsed configuration + * @throws ConfigException on IO or parse errors + */ public static Config parseFile(File file) { return parseFile(file, ConfigParseOptions.defaults()); } @@ -637,6 +662,14 @@ public final class ConfigFactory { return ConfigImpl.parseFileAnySyntax(fileBasename, options).toConfig(); } + /** + * Like {@link #parseFileAnySyntax(File,ConfigParseOptions)} but always uses + * default parse options. + * + * @param fileBasename + * a filename with or without extension + * @return the parsed configuration + */ public static Config parseFileAnySyntax(File fileBasename) { return parseFileAnySyntax(fileBasename, ConfigParseOptions.defaults()); } From 3c6488fbc2f4875d9c7088bd3dcb2af964f1b064 Mon Sep 17 00:00:00 2001 From: ian Date: Tue, 22 Jul 2014 19:43:33 -0400 Subject: [PATCH 13/15] Fix 'allow unresolved' for at least two list cases lists did not respond well to cases where only some of their elements were resolved or even when none of them were resolved but one of their object identities happened to change. Previously this would throw a nasty 'bug or broken' exception so it seems unlikely to have unintended side effects. --- .../java/com/typesafe/config/impl/SimpleConfigList.java | 8 ++++++-- .../test/scala/com/typesafe/config/impl/ConfigTest.scala | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java b/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java index 5ec13f29..ec5a4b98 100644 --- a/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java +++ b/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java @@ -98,7 +98,11 @@ final class SimpleConfigList extends AbstractConfigValue implements ConfigList, } if (changed != null) { - return new SimpleConfigList(origin(), changed, newResolveStatus); + if (newResolveStatus != null) { + return new SimpleConfigList(origin(), changed, newResolveStatus); + } else { + return new SimpleConfigList(origin(), changed); + } } else { return this; } @@ -122,7 +126,7 @@ final class SimpleConfigList extends AbstractConfigValue implements ConfigList, return context.resolve(v); } - }, ResolveStatus.RESOLVED); + }, null /* don't force resolve status -- could be allowing unresolved */); } catch (NotPossibleToResolve e) { throw e; } catch (RuntimeException e) { diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index a68f3897..568c19a0 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -1120,9 +1120,10 @@ class ConfigTest extends TestUtils { @Test def allowUnresolvedDoesAllowUnresolved() { - val values = ConfigFactory.parseString("{ foo = 1, bar = 2, m = 3, n = 4}") + val values = ConfigFactory.parseString("{ foo = 1, bar = 2, m = 3, n = 4, unknown = [someVal]}") assertTrue("config with no substitutions starts as resolved", values.isResolved) - val unresolved = ConfigFactory.parseString("a = ${foo}, b = ${bar}, c { x = ${m}, y = ${n}, z = foo${m}bar }, alwaysResolveable=${alwaysValue}, alwaysValue=42") + val unresolved = ConfigFactory.parseString( + "l = [${unknown}[]], l2 = [${unknown}, ${alwaysValue}], a = ${foo}, b = ${bar}, c { x = ${m}, y = ${n}, z = foo${m}bar }, alwaysResolveable=${alwaysValue}, alwaysValue=42") assertFalse("config with substitutions starts as not resolved", unresolved.isResolved) // resolve() by default throws with unresolveable substs From ed19f4c09e8577fae5dbd8ce4446abe183b2d428 Mon Sep 17 00:00:00 2001 From: ian Date: Wed, 24 Dec 2014 16:10:20 -0500 Subject: [PATCH 14/15] isolate test for allowing unresolved list elements --- .../com/typesafe/config/impl/ConfigTest.scala | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala index 568c19a0..b6368299 100644 --- a/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala +++ b/config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala @@ -1118,12 +1118,20 @@ class ConfigTest extends TestUtils { assertTrue("after resolution, config is now resolved", resolved2.isResolved) } + @Test + def allowUnresolvedDoesAllowUnresolvedArrayElements() { + val values = ConfigFactory.parseString("unknown = [someVal], known = 42") + val unresolved = ConfigFactory.parseString("concat = [${unknown}[]], sibling = [${unknown}, ${known}]") + unresolved.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true)) + unresolved.withFallback(values).resolve() + unresolved.resolveWith(values) + } + @Test def allowUnresolvedDoesAllowUnresolved() { - val values = ConfigFactory.parseString("{ foo = 1, bar = 2, m = 3, n = 4, unknown = [someVal]}") + val values = ConfigFactory.parseString("{ foo = 1, bar = 2, m = 3, n = 4}") assertTrue("config with no substitutions starts as resolved", values.isResolved) - val unresolved = ConfigFactory.parseString( - "l = [${unknown}[]], l2 = [${unknown}, ${alwaysValue}], a = ${foo}, b = ${bar}, c { x = ${m}, y = ${n}, z = foo${m}bar }, alwaysResolveable=${alwaysValue}, alwaysValue=42") + val unresolved = ConfigFactory.parseString("a = ${foo}, b = ${bar}, c { x = ${m}, y = ${n}, z = foo${m}bar }, alwaysResolveable=${alwaysValue}, alwaysValue=42") assertFalse("config with substitutions starts as not resolved", unresolved.isResolved) // resolve() by default throws with unresolveable substs From 94dda5f9cf6bd6d62ff62af236ebbecc4b17923f Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Sat, 27 Dec 2014 16:41:47 -0500 Subject: [PATCH 15/15] If not allowing unresolved, optimize resolve status computation for List --- .../main/java/com/typesafe/config/impl/SimpleConfigList.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java b/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java index ec5a4b98..2db8396e 100644 --- a/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java +++ b/config/src/main/java/com/typesafe/config/impl/SimpleConfigList.java @@ -125,8 +125,7 @@ final class SimpleConfigList extends AbstractConfigValue implements ConfigList, throws NotPossibleToResolve { return context.resolve(v); } - - }, null /* don't force resolve status -- could be allowing unresolved */); + }, context.options().getAllowUnresolved() ? null : ResolveStatus.RESOLVED); } catch (NotPossibleToResolve e) { throw e; } catch (RuntimeException e) {