github-repos/config
1
0
Fork 0
mirror of https://github.com/lightbend/config.git synced 2025-01-15 23:01:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add lossless comment tokens

Browse Source 
Preserve the original text of comments in comment tokens so they
can be reproduced by the TokenIterator's render() method.
This commit is contained in:
Preben Ingvaldsen 2015-03-02 11:54:43 -08:00
parent 574f810a0f
commit 2b1c7ef13d
4 changed files with 25 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/src/main/java/com/typesafe/config/impl/Tokenizer.java
View File

@ -278,10 +278,12 @@ final class Tokenizer {
// ONE char has always been consumed, either the # or the first /, but
// not both slashes
private Token pullComment(int firstChar) {
boolean doubleSlash = false;
if (firstChar == '/') {
int discard = nextCharRaw();
if (discard != '/')
throw new ConfigException.BugOrBroken("called pullComment but // not seen");
doubleSlash = true;
}
StringBuilder sb = new StringBuilder();
@ -289,7 +291,7 @@ final class Tokenizer {
int c = nextCharRaw();
if (c == -1 || c == '\n') {
putBack(c);
return Tokens.newComment(lineOrigin, sb.toString());
return Tokens.newComment(lineOrigin, sb.toString(), doubleSlash);
} else {
sb.appendCodePoint(c);
}

8
config/src/main/java/com/typesafe/config/impl/Tokens.java
View File

@ -198,8 +198,8 @@ final class Tokens {
static private class Comment extends Token {
final private String text;
Comment(ConfigOrigin origin, String text) {
super(TokenType.COMMENT, origin);
Comment(ConfigOrigin origin, String text, boolean doubleSlash) {
super(TokenType.COMMENT, origin, (doubleSlash? "//" : "#") + text);
this.text = text;
}
@ -409,8 +409,8 @@ final class Tokens {
return new Problem(origin, what, message, suggestQuotes, cause);
}
static Token newComment(ConfigOrigin origin, String text) {
return new Comment(origin, text);
static Token newComment(ConfigOrigin origin, String text, boolean doubleSlash) {
return new Comment(origin, text, doubleSlash);
}
static Token newUnquotedText(ConfigOrigin origin, String s) {

2
config/src/test/scala/com/typesafe/config/impl/TestUtils.scala
View File

@ -616,7 +616,7 @@ abstract trait TestUtils {
def tokenInt(i: Int) = Tokens.newInt(fakeOrigin(), i, null)
def tokenLong(l: Long) = Tokens.newLong(fakeOrigin(), l, null)
def tokenLine(line: Int) = Tokens.newLine(fakeOrigin.withLineNumber(line))
def tokenComment(text: String) = Tokens.newComment(fakeOrigin(), text)
def tokenComment(text: String, doubleSlash : Boolean) = Tokens.newComment(fakeOrigin(), text, doubleSlash)
def tokenWhitespace(text: String) = Tokens.newIgnoredWhitespace(fakeOrigin(), text)
private def tokenMaybeOptionalSubstitution(optional: Boolean, expression: Token*) = {

29
config/src/test/scala/com/typesafe/config/impl/TokenizerTest.scala
View File

@ -134,7 +134,7 @@ class TokenizerTest extends TestUtils {
tokenizerTest(List(tokenUnquoted("a/b/c/")), "a/b/c/")
tokenizerTest(List(tokenUnquoted("/")), "/")
tokenizerTest(List(tokenUnquoted("/"), tokenUnquoted(" "), tokenUnquoted("/")), "/ /")
//tokenizerTest(List(tokenComment("")), "//")
tokenizerTest(List(tokenComment("", true)), "//")
}
@Test
@ -283,17 +283,22 @@ class TokenizerTest extends TestUtils {
@Test
def commentsHandledInVariousContexts() {
// tokenizerTest(List(tokenString("//bar")), "\"//bar\"")
// tokenizerTest(List(tokenString("#bar")), "\"#bar\"")
// tokenizerTest(List(tokenUnquoted("bar"), tokenComment("comment")), "bar//comment")
// tokenizerTest(List(tokenUnquoted("bar"), tokenComment("comment")), "bar#comment")
// tokenizerTest(List(tokenInt(10), tokenComment("comment")), "10//comment")
// tokenizerTest(List(tokenInt(10), tokenComment("comment")), "10#comment")
// tokenizerTest(List(tokenDouble(3.14), tokenComment("comment")), "3.14//comment")
// tokenizerTest(List(tokenDouble(3.14), tokenComment("comment")), "3.14#comment")
// // be sure we keep the newline
// tokenizerTest(List(tokenInt(10), tokenComment("comment"), tokenLine(1), tokenInt(12)), "10//comment\n12")
// tokenizerTest(List(tokenInt(10), tokenComment("comment"), tokenLine(1), tokenInt(12)), "10#comment\n12")
tokenizerTest(List(tokenString("//bar")), "\"//bar\"")
tokenizerTest(List(tokenString("#bar")), "\"#bar\"")
tokenizerTest(List(tokenUnquoted("bar"), tokenComment("comment", true)), "bar//comment")
tokenizerTest(List(tokenUnquoted("bar"), tokenComment("comment", false)), "bar#comment")
tokenizerTest(List(tokenInt(10), tokenComment("comment", true)), "10//comment")
tokenizerTest(List(tokenInt(10), tokenComment("comment", false)), "10#comment")
tokenizerTest(List(tokenDouble(3.14), tokenComment("comment", true)), "3.14//comment")
tokenizerTest(List(tokenDouble(3.14), tokenComment("comment", false)), "3.14#comment")
// be sure we keep the newline
tokenizerTest(List(tokenInt(10), tokenComment("comment", true), tokenLine(1), tokenInt(12)), "10//comment\n12")
tokenizerTest(List(tokenInt(10), tokenComment("comment", false), tokenLine(1), tokenInt(12)), "10#comment\n12")
// be sure we handle multi-line comments
tokenizerTest(List(tokenComment("comment", true), tokenLine(1), tokenComment("comment2", true)),
"//comment\n//comment2")
tokenizerTest(List(tokenComment("comment", false), tokenLine(1), tokenComment("comment2", false)),
"#comment\n#comment2")
}
@Test