From fda3f7eead5c8da370e23d85183bd9c2f9ef9f5d Mon Sep 17 00:00:00 2001 From: Havoc Pennington Date: Mon, 7 Nov 2011 19:16:26 -0500 Subject: [PATCH] move tokenizer tests into their own file --- .../com/typesafe/config/impl/JsonTest.scala | 132 ---------------- .../typesafe/config/impl/TokenizerTest.scala | 148 ++++++++++++++++++ 2 files changed, 148 insertions(+), 132 deletions(-) create mode 100644 src/test/scala/com/typesafe/config/impl/TokenizerTest.scala diff --git a/src/test/scala/com/typesafe/config/impl/JsonTest.scala b/src/test/scala/com/typesafe/config/impl/JsonTest.scala index b24a9cf0..b7de4771 100644 --- a/src/test/scala/com/typesafe/config/impl/JsonTest.scala +++ b/src/test/scala/com/typesafe/config/impl/JsonTest.scala @@ -14,142 +14,10 @@ class JsonTest extends TestUtils { def setup() { } - def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = { - Tokenizer.tokenize(origin, input) - } - - def tokenize(input: Reader): java.util.Iterator[Token] = { - tokenize(new SimpleConfigOrigin("anonymous Reader"), input) - } - - def tokenize(s: String): java.util.Iterator[Token] = { - val reader = new StringReader(s) - val result = tokenize(reader) - // reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken - result - } - - def tokenizeAsList(s: String) = { - import scala.collection.JavaConverters._ - tokenize(s).asScala.toList - } - def parse(s: String): ConfigValue = { Parser.parse(SyntaxFlavor.JSON, new SimpleConfigOrigin("test string"), s) } - @Test - def tokenizeEmptyString() { - assertEquals(List(Tokens.START, Tokens.END), - tokenizeAsList("")) - } - - @Test - def tokenizeAllTypesNoSpaces() { - // all token types with no spaces (not sure JSON spec wants this to work, - // but spec is unclear to me when spaces are required, and banning them - // is actually extra work) - val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, - Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"), - Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), - Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) - assertEquals(expected, tokenizeAsList(""",:}{]["foo"42true3.14falsenull""" + "\n")) - } - - @Test - def tokenizeAllTypesWithSingleSpaces() { - // all token types with no spaces (not sure JSON spec wants this to work, - // but spec is unclear to me when spaces are required, and banning them - // is actually extra work) - val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, - Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"), - Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), - Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) - assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n ")) - } - - @Test - def tokenizeAllTypesWithMultipleSpaces() { - // all token types with no spaces (not sure JSON spec wants this to work, - // but spec is unclear to me when spaces are required, and banning them - // is actually extra work) - val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, - Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"), - Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), - Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) - assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n ")) - } - - @Test - def tokenizerUnescapeStrings(): Unit = { - case class UnescapeTest(escaped: String, result: ConfigString) - implicit def pair2unescapetest(pair: (String, String)): UnescapeTest = UnescapeTest(pair._1, new ConfigString(fakeOrigin(), pair._2)) - - // getting the actual 6 chars we want in a string is a little pesky. - // \u005C is backslash. Just prove we're doing it right here. - assertEquals(6, "\\u0046".length) - assertEquals('4', "\\u0046"(4)) - assertEquals('6', "\\u0046"(5)) - - val tests = List[UnescapeTest]((""" "" """, ""), - (" \"\0\" ", "\0"), // nul byte - (""" "\"\\\/\b\f\n\r\t" """, "\"\\/\b\f\n\r\t"), - ("\"\\u0046\"", "F"), - ("\"\\u0046\\u0046\"", "FF")) - - for (t <- tests) { - describeFailure(t.toString) { - assertEquals(List(Tokens.START, Tokens.newValue(t.result), Tokens.END), - tokenizeAsList(t.escaped)) - } - } - } - - @Test - def tokenizerThrowsOnInvalidStrings(): Unit = { - val invalidTests = List(""" "\" """, // nothing after a backslash - """ "\q" """, // there is no \q escape sequence - "\"\\u123\"", // too short - "\"\\u12\"", // too short - "\"\\u1\"", // too short - "\"\\u\"", // too short - "\"", // just a single quote - """ "abcdefg""" // no end quote - ) - - for (t <- invalidTests) { - describeFailure(t) { - intercept[ConfigException] { - tokenizeAsList(t) - } - } - } - } - - @Test - def tokenizerParseNumbers(): Unit = { - abstract class NumberTest(val s: String, val result: Token) - case class LongTest(override val s: String, override val result: Token) extends NumberTest(s, result) - case class DoubleTest(override val s: String, override val result: Token) extends NumberTest(s, result) - implicit def pair2inttest(pair: (String, Int)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2)) - implicit def pair2longtest(pair: (String, Long)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2)) - implicit def pair2doubletest(pair: (String, Double)) = DoubleTest(pair._1, Tokens.newDouble(fakeOrigin(), pair._2)) - - val tests = List[NumberTest](("1", 1), - ("1.2", 1.2), - ("1e6", 1e6), - ("1e-6", 1e-6), - ("-1", -1), - ("-1.2", -1.2)) - - for (t <- tests) { - describeFailure(t.toString()) { - assertEquals(List(Tokens.START, t.result, Tokens.END), - tokenizeAsList(t.s)) - } - } - } - private[this] def toLift(value: ConfigValue): lift.JValue = { import scala.collection.JavaConverters._ diff --git a/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala b/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala new file mode 100644 index 00000000..166e30ff --- /dev/null +++ b/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala @@ -0,0 +1,148 @@ +package com.typesafe.config.impl + +import org.junit.Assert._ +import org.junit._ +import net.liftweb.{ json => lift } +import java.io.Reader +import java.io.StringReader +import com.typesafe.config._ +import java.util.HashMap + +class TokenizerTest extends TestUtils { + + @org.junit.Before + def setup() { + } + + def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = { + Tokenizer.tokenize(origin, input) + } + + def tokenize(input: Reader): java.util.Iterator[Token] = { + tokenize(new SimpleConfigOrigin("anonymous Reader"), input) + } + + def tokenize(s: String): java.util.Iterator[Token] = { + val reader = new StringReader(s) + val result = tokenize(reader) + // reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken + result + } + + def tokenizeAsList(s: String) = { + import scala.collection.JavaConverters._ + tokenize(s).asScala.toList + } + + @Test + def tokenizeEmptyString() { + assertEquals(List(Tokens.START, Tokens.END), + tokenizeAsList("")) + } + + @Test + def tokenizeAllTypesNoSpaces() { + // all token types with no spaces (not sure JSON spec wants this to work, + // but spec is unclear to me when spaces are required, and banning them + // is actually extra work) + val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, + Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"), + Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), + Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) + assertEquals(expected, tokenizeAsList(""",:}{]["foo"42true3.14falsenull""" + "\n")) + } + + @Test + def tokenizeAllTypesWithSingleSpaces() { + // all token types with no spaces (not sure JSON spec wants this to work, + // but spec is unclear to me when spaces are required, and banning them + // is actually extra work) + val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, + Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"), + Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), + Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) + assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n ")) + } + + @Test + def tokenizeAllTypesWithMultipleSpaces() { + // all token types with no spaces (not sure JSON spec wants this to work, + // but spec is unclear to me when spaces are required, and banning them + // is actually extra work) + val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY, + Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"), + Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14), + Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END) + assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n ")) + } + + @Test + def tokenizerUnescapeStrings(): Unit = { + case class UnescapeTest(escaped: String, result: ConfigString) + implicit def pair2unescapetest(pair: (String, String)): UnescapeTest = UnescapeTest(pair._1, new ConfigString(fakeOrigin(), pair._2)) + + // getting the actual 6 chars we want in a string is a little pesky. + // \u005C is backslash. Just prove we're doing it right here. + assertEquals(6, "\\u0046".length) + assertEquals('4', "\\u0046"(4)) + assertEquals('6', "\\u0046"(5)) + + val tests = List[UnescapeTest]((""" "" """, ""), + (" \"\0\" ", "\0"), // nul byte + (""" "\"\\\/\b\f\n\r\t" """, "\"\\/\b\f\n\r\t"), + ("\"\\u0046\"", "F"), + ("\"\\u0046\\u0046\"", "FF")) + + for (t <- tests) { + describeFailure(t.toString) { + assertEquals(List(Tokens.START, Tokens.newValue(t.result), Tokens.END), + tokenizeAsList(t.escaped)) + } + } + } + + @Test + def tokenizerThrowsOnInvalidStrings(): Unit = { + val invalidTests = List(""" "\" """, // nothing after a backslash + """ "\q" """, // there is no \q escape sequence + "\"\\u123\"", // too short + "\"\\u12\"", // too short + "\"\\u1\"", // too short + "\"\\u\"", // too short + "\"", // just a single quote + """ "abcdefg""" // no end quote + ) + + for (t <- invalidTests) { + describeFailure(t) { + intercept[ConfigException] { + tokenizeAsList(t) + } + } + } + } + + @Test + def tokenizerParseNumbers(): Unit = { + abstract class NumberTest(val s: String, val result: Token) + case class LongTest(override val s: String, override val result: Token) extends NumberTest(s, result) + case class DoubleTest(override val s: String, override val result: Token) extends NumberTest(s, result) + implicit def pair2inttest(pair: (String, Int)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2)) + implicit def pair2longtest(pair: (String, Long)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2)) + implicit def pair2doubletest(pair: (String, Double)) = DoubleTest(pair._1, Tokens.newDouble(fakeOrigin(), pair._2)) + + val tests = List[NumberTest](("1", 1), + ("1.2", 1.2), + ("1e6", 1e6), + ("1e-6", 1e-6), + ("-1", -1), + ("-1.2", -1.2)) + + for (t <- tests) { + describeFailure(t.toString()) { + assertEquals(List(Tokens.START, t.result, Tokens.END), + tokenizeAsList(t.s)) + } + } + } +}