mirror of
https://github.com/lightbend/config.git
synced 2025-02-21 00:30:32 +08:00
move tokenizer tests into their own file
This commit is contained in:
Havoc Pennington
parent
ff5bf095c7
commit
fda3f7eead
@ -14,142 +14,10 @@ class JsonTest extends TestUtils {
|
||||
def setup() {
|
||||
}
|
||||
|
||||
def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = {
|
||||
Tokenizer.tokenize(origin, input)
|
||||
}
|
||||
|
||||
def tokenize(input: Reader): java.util.Iterator[Token] = {
|
||||
tokenize(new SimpleConfigOrigin("anonymous Reader"), input)
|
||||
}
|
||||
|
||||
def tokenize(s: String): java.util.Iterator[Token] = {
|
||||
val reader = new StringReader(s)
|
||||
val result = tokenize(reader)
|
||||
// reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken
|
||||
result
|
||||
}
|
||||
|
||||
def tokenizeAsList(s: String) = {
|
||||
import scala.collection.JavaConverters._
|
||||
tokenize(s).asScala.toList
|
||||
}
|
||||
|
||||
def parse(s: String): ConfigValue = {
|
||||
Parser.parse(SyntaxFlavor.JSON, new SimpleConfigOrigin("test string"), s)
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeEmptyString() {
|
||||
assertEquals(List(Tokens.START, Tokens.END),
|
||||
tokenizeAsList(""))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeAllTypesNoSpaces() {
|
||||
// all token types with no spaces (not sure JSON spec wants this to work,
|
||||
// but spec is unclear to me when spaces are required, and banning them
|
||||
// is actually extra work)
|
||||
val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
|
||||
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"),
|
||||
Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14),
|
||||
Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END)
|
||||
assertEquals(expected, tokenizeAsList(""",:}{]["foo"42true3.14falsenull""" + "\n"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeAllTypesWithSingleSpaces() {
|
||||
// all token types with no spaces (not sure JSON spec wants this to work,
|
||||
// but spec is unclear to me when spaces are required, and banning them
|
||||
// is actually extra work)
|
||||
val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
|
||||
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"),
|
||||
Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14),
|
||||
Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END)
|
||||
assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n "))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeAllTypesWithMultipleSpaces() {
|
||||
// all token types with no spaces (not sure JSON spec wants this to work,
|
||||
// but spec is unclear to me when spaces are required, and banning them
|
||||
// is actually extra work)
|
||||
val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
|
||||
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"),
|
||||
Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14),
|
||||
Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END)
|
||||
assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n "))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizerUnescapeStrings(): Unit = {
|
||||
case class UnescapeTest(escaped: String, result: ConfigString)
|
||||
implicit def pair2unescapetest(pair: (String, String)): UnescapeTest = UnescapeTest(pair._1, new ConfigString(fakeOrigin(), pair._2))
|
||||
|
||||
// getting the actual 6 chars we want in a string is a little pesky.
|
||||
// \u005C is backslash. Just prove we're doing it right here.
|
||||
assertEquals(6, "\\u0046".length)
|
||||
assertEquals('4', "\\u0046"(4))
|
||||
assertEquals('6', "\\u0046"(5))
|
||||
|
||||
val tests = List[UnescapeTest]((""" "" """, ""),
|
||||
(" \"\0\" ", "\0"), // nul byte
|
||||
(""" "\"\\\/\b\f\n\r\t" """, "\"\\/\b\f\n\r\t"),
|
||||
("\"\\u0046\"", "F"),
|
||||
("\"\\u0046\\u0046\"", "FF"))
|
||||
|
||||
for (t <- tests) {
|
||||
describeFailure(t.toString) {
|
||||
assertEquals(List(Tokens.START, Tokens.newValue(t.result), Tokens.END),
|
||||
tokenizeAsList(t.escaped))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizerThrowsOnInvalidStrings(): Unit = {
|
||||
val invalidTests = List(""" "\" """, // nothing after a backslash
|
||||
""" "\q" """, // there is no \q escape sequence
|
||||
"\"\\u123\"", // too short
|
||||
"\"\\u12\"", // too short
|
||||
"\"\\u1\"", // too short
|
||||
"\"\\u\"", // too short
|
||||
"\"", // just a single quote
|
||||
""" "abcdefg""" // no end quote
|
||||
)
|
||||
|
||||
for (t <- invalidTests) {
|
||||
describeFailure(t) {
|
||||
intercept[ConfigException] {
|
||||
tokenizeAsList(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizerParseNumbers(): Unit = {
|
||||
abstract class NumberTest(val s: String, val result: Token)
|
||||
case class LongTest(override val s: String, override val result: Token) extends NumberTest(s, result)
|
||||
case class DoubleTest(override val s: String, override val result: Token) extends NumberTest(s, result)
|
||||
implicit def pair2inttest(pair: (String, Int)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2))
|
||||
implicit def pair2longtest(pair: (String, Long)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2))
|
||||
implicit def pair2doubletest(pair: (String, Double)) = DoubleTest(pair._1, Tokens.newDouble(fakeOrigin(), pair._2))
|
||||
|
||||
val tests = List[NumberTest](("1", 1),
|
||||
("1.2", 1.2),
|
||||
("1e6", 1e6),
|
||||
("1e-6", 1e-6),
|
||||
("-1", -1),
|
||||
("-1.2", -1.2))
|
||||
|
||||
for (t <- tests) {
|
||||
describeFailure(t.toString()) {
|
||||
assertEquals(List(Tokens.START, t.result, Tokens.END),
|
||||
tokenizeAsList(t.s))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private[this] def toLift(value: ConfigValue): lift.JValue = {
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
|
||||
148
src/test/scala/com/typesafe/config/impl/TokenizerTest.scala
Normal file
148
src/test/scala/com/typesafe/config/impl/TokenizerTest.scala
Normal file
@ -0,0 +1,148 @@
|
||||
package com.typesafe.config.impl
|
||||
|
||||
import org.junit.Assert._
|
||||
import org.junit._
|
||||
import net.liftweb.{ json => lift }
|
||||
import java.io.Reader
|
||||
import java.io.StringReader
|
||||
import com.typesafe.config._
|
||||
import java.util.HashMap
|
||||
|
||||
class TokenizerTest extends TestUtils {
|
||||
|
||||
@org.junit.Before
|
||||
def setup() {
|
||||
}
|
||||
|
||||
def tokenize(origin: ConfigOrigin, input: Reader): java.util.Iterator[Token] = {
|
||||
Tokenizer.tokenize(origin, input)
|
||||
}
|
||||
|
||||
def tokenize(input: Reader): java.util.Iterator[Token] = {
|
||||
tokenize(new SimpleConfigOrigin("anonymous Reader"), input)
|
||||
}
|
||||
|
||||
def tokenize(s: String): java.util.Iterator[Token] = {
|
||||
val reader = new StringReader(s)
|
||||
val result = tokenize(reader)
|
||||
// reader.close() // can't close until the iterator is traversed, so this tokenize() flavor is inherently broken
|
||||
result
|
||||
}
|
||||
|
||||
def tokenizeAsList(s: String) = {
|
||||
import scala.collection.JavaConverters._
|
||||
tokenize(s).asScala.toList
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeEmptyString() {
|
||||
assertEquals(List(Tokens.START, Tokens.END),
|
||||
tokenizeAsList(""))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeAllTypesNoSpaces() {
|
||||
// all token types with no spaces (not sure JSON spec wants this to work,
|
||||
// but spec is unclear to me when spaces are required, and banning them
|
||||
// is actually extra work)
|
||||
val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
|
||||
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"),
|
||||
Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14),
|
||||
Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END)
|
||||
assertEquals(expected, tokenizeAsList(""",:}{]["foo"42true3.14falsenull""" + "\n"))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeAllTypesWithSingleSpaces() {
|
||||
// all token types with no spaces (not sure JSON spec wants this to work,
|
||||
// but spec is unclear to me when spaces are required, and banning them
|
||||
// is actually extra work)
|
||||
val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
|
||||
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"),
|
||||
Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14),
|
||||
Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END)
|
||||
assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n "))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizeAllTypesWithMultipleSpaces() {
|
||||
// all token types with no spaces (not sure JSON spec wants this to work,
|
||||
// but spec is unclear to me when spaces are required, and banning them
|
||||
// is actually extra work)
|
||||
val expected = List(Tokens.START, Tokens.COMMA, Tokens.COLON, Tokens.CLOSE_CURLY,
|
||||
Tokens.OPEN_CURLY, Tokens.CLOSE_SQUARE, Tokens.OPEN_SQUARE, Tokens.newString(fakeOrigin(), "foo"),
|
||||
Tokens.newLong(fakeOrigin(), 42), Tokens.newBoolean(fakeOrigin(), true), Tokens.newDouble(fakeOrigin(), 3.14),
|
||||
Tokens.newBoolean(fakeOrigin(), false), Tokens.newNull(fakeOrigin()), Tokens.newLine(0), Tokens.END)
|
||||
assertEquals(expected, tokenizeAsList(""" , : } { ] [ "foo" 42 true 3.14 false null """ + "\n "))
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizerUnescapeStrings(): Unit = {
|
||||
case class UnescapeTest(escaped: String, result: ConfigString)
|
||||
implicit def pair2unescapetest(pair: (String, String)): UnescapeTest = UnescapeTest(pair._1, new ConfigString(fakeOrigin(), pair._2))
|
||||
|
||||
// getting the actual 6 chars we want in a string is a little pesky.
|
||||
// \u005C is backslash. Just prove we're doing it right here.
|
||||
assertEquals(6, "\\u0046".length)
|
||||
assertEquals('4', "\\u0046"(4))
|
||||
assertEquals('6', "\\u0046"(5))
|
||||
|
||||
val tests = List[UnescapeTest]((""" "" """, ""),
|
||||
(" \"\0\" ", "\0"), // nul byte
|
||||
(""" "\"\\\/\b\f\n\r\t" """, "\"\\/\b\f\n\r\t"),
|
||||
("\"\\u0046\"", "F"),
|
||||
("\"\\u0046\\u0046\"", "FF"))
|
||||
|
||||
for (t <- tests) {
|
||||
describeFailure(t.toString) {
|
||||
assertEquals(List(Tokens.START, Tokens.newValue(t.result), Tokens.END),
|
||||
tokenizeAsList(t.escaped))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizerThrowsOnInvalidStrings(): Unit = {
|
||||
val invalidTests = List(""" "\" """, // nothing after a backslash
|
||||
""" "\q" """, // there is no \q escape sequence
|
||||
"\"\\u123\"", // too short
|
||||
"\"\\u12\"", // too short
|
||||
"\"\\u1\"", // too short
|
||||
"\"\\u\"", // too short
|
||||
"\"", // just a single quote
|
||||
""" "abcdefg""" // no end quote
|
||||
)
|
||||
|
||||
for (t <- invalidTests) {
|
||||
describeFailure(t) {
|
||||
intercept[ConfigException] {
|
||||
tokenizeAsList(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
def tokenizerParseNumbers(): Unit = {
|
||||
abstract class NumberTest(val s: String, val result: Token)
|
||||
case class LongTest(override val s: String, override val result: Token) extends NumberTest(s, result)
|
||||
case class DoubleTest(override val s: String, override val result: Token) extends NumberTest(s, result)
|
||||
implicit def pair2inttest(pair: (String, Int)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2))
|
||||
implicit def pair2longtest(pair: (String, Long)) = LongTest(pair._1, Tokens.newLong(fakeOrigin(), pair._2))
|
||||
implicit def pair2doubletest(pair: (String, Double)) = DoubleTest(pair._1, Tokens.newDouble(fakeOrigin(), pair._2))
|
||||
|
||||
val tests = List[NumberTest](("1", 1),
|
||||
("1.2", 1.2),
|
||||
("1e6", 1e6),
|
||||
("1e-6", 1e-6),
|
||||
("-1", -1),
|
||||
("-1.2", -1.2))
|
||||
|
||||
for (t <- tests) {
|
||||
describeFailure(t.toString()) {
|
||||
assertEquals(List(Tokens.START, t.result, Tokens.END),
|
||||
tokenizeAsList(t.s))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user