Change way indent and dedent special tokens are produced by TokenizedData

* Move dedent to the end of the previous line
* Leave indent to the first character of the current line
* Dedent is called as many times as indent

1 files changed, 36 insertions, 2 deletions

diff --git a/test/core/parser/utils/TokenizedDataTest.cpp b/test/core/parser/utils/TokenizedDataTest.cpp
index 8488459..31346bd 100644
--- a/test/core/parser/utils/TokenizedDataTest.cpp
+++ b/test/core/parser/utils/TokenizedDataTest.cpp
@@ -355,11 +355,43 @@ TEST(TokenizedData, specialTokenIndent)
 	            10, 10);
 	assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, 10, 37);
 	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
-	            38, 38);
+	            37, 37);
+	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
+	            37, 37);
 	assertText(reader, "test5", tokens, WhitespaceMode::COLLAPSE, 38, 43);
 	assertEnd(reader);
 }
 
+TEST(TokenizedData, specialTokenIndent2)
+{
+	TokenizedData data;
+	data.append("a\n\tb\n\t\tc\n\t\t\td\n\te\nf\n");
+	//           0 1 23 4 5 67 8 9 0 12 3 45 67 8
+	//           0                 1
+	const TokenSet tokens{Tokens::Indent, Tokens::Dedent};
+
+	TokenizedDataReader reader = data.reader();
+	assertText(reader, "a", tokens, WhitespaceMode::COLLAPSE, 0, 1);
+	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
+	            3, 3);
+	assertText(reader, "b", tokens, WhitespaceMode::COLLAPSE, 3, 4);
+	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
+	            7, 7);
+	assertText(reader, "c", tokens, WhitespaceMode::COLLAPSE, 7, 8);
+	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
+	            12, 12);
+	assertText(reader, "d", tokens, WhitespaceMode::COLLAPSE, 12, 13);
+	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
+	            13, 13);
+	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
+	            13, 13);
+	assertText(reader, "e", tokens, WhitespaceMode::COLLAPSE, 15, 16);
+	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
+	            16, 16);
+	assertText(reader, "f", tokens, WhitespaceMode::COLLAPSE, 17, 18);
+	assertEnd(reader);
+}
+
 TEST(TokenizedData, specialTokenIndentOverlap)
 {
 	TokenizedData data;
@@ -378,7 +410,9 @@ TEST(TokenizedData, specialTokenIndentOverlap)
 	            10, 10);
 	assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, 10, 37);
 	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
-	            38, 38);
+	            37, 37);
+	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
+	            37, 37);
 	assertText(reader, "test5", tokens, WhitespaceMode::COLLAPSE, 38, 43);
 	assertEnd(reader);
 }