Implement endAtWhitespace flag which tells TokenizedDataReader to stop reading data after the first whitespace character

2 files changed, 59 insertions, 25 deletions

diff --git a/test/core/parser/utils/TokenizedDataTest.cpp b/test/core/parser/utils/TokenizedDataTest.cpp
index 31346bd..e468a50 100644
--- a/test/core/parser/utils/TokenizedDataTest.cpp
+++ b/test/core/parser/utils/TokenizedDataTest.cpp
@@ -348,12 +348,13 @@ TEST(TokenizedData, specialTokenIndent)
 	const TokenSet tokens{Tokens::Indent, Tokens::Dedent};
 
 	TokenizedDataReader reader = data.reader();
-	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
-	            4, 4);
+	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 4,
+	            4);
 	assertText(reader, "test", tokens, WhitespaceMode::COLLAPSE, 4, 8);
 	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
 	            10, 10);
-	assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, 10, 37);
+	assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE,
+	           10, 37);
 	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
 	            37, 37);
 	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
@@ -372,11 +373,11 @@ TEST(TokenizedData, specialTokenIndent2)
 
 	TokenizedDataReader reader = data.reader();
 	assertText(reader, "a", tokens, WhitespaceMode::COLLAPSE, 0, 1);
-	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
-	            3, 3);
+	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 3,
+	            3);
 	assertText(reader, "b", tokens, WhitespaceMode::COLLAPSE, 3, 4);
-	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
-	            7, 7);
+	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 7,
+	            7);
 	assertText(reader, "c", tokens, WhitespaceMode::COLLAPSE, 7, 8);
 	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
 	            12, 12);
@@ -403,12 +404,13 @@ TEST(TokenizedData, specialTokenIndentOverlap)
 	data.mark(5, 4, 4);
 
 	TokenizedDataReader reader = data.reader();
-	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
-	            4, 4);
+	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 4,
+	            4);
 	assertToken(reader, 5, "test", tokens, WhitespaceMode::COLLAPSE, 4, 8);
 	assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE,
 	            10, 10);
-	assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, 10, 37);
+	assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE,
+	           10, 37);
 	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
 	            37, 37);
 	assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE,
@@ -417,5 +419,26 @@ TEST(TokenizedData, specialTokenIndentOverlap)
 	assertEnd(reader);
 }
 
+TEST(TokenizedData, endAtWhitespace)
+{
+	TokenizedData data;
+	data.append("    a    b c   d");
+	//           0123456789012345
+	//           0         1
+	const TokenSet tokens{};
+
+	data.mark(5, 4, 4);
+
+	TokenizedDataReader reader = data.reader();
+	assertTextEndAtWhitespace(reader, "    a", tokens, WhitespaceMode::PRESERVE,
+	                          0, 5);
+	assertTextEndAtWhitespace(reader, "    b", tokens, WhitespaceMode::PRESERVE,
+	                          5, 10);
+	assertTextEndAtWhitespace(reader, " c", tokens, WhitespaceMode::PRESERVE,
+	                          10, 12);
+	assertTextEndAtWhitespace(reader, "   d", tokens, WhitespaceMode::PRESERVE,
+	                          12, 16);
+	assertEnd(reader);
+}
 }
 
diff --git a/test/core/parser/utils/TokenizedDataTestUtils.hpp b/test/core/parser/utils/TokenizedDataTestUtils.hpp
index c384f9d..30f72ae 100644
--- a/test/core/parser/utils/TokenizedDataTestUtils.hpp
+++ b/test/core/parser/utils/TokenizedDataTestUtils.hpp
@@ -21,15 +21,17 @@
 
 namespace ousia {
 
-static void assertToken(TokenizedDataReader &reader, TokenId id,
-                 const std::string &text, const TokenSet &tokens = TokenSet{},
-                 WhitespaceMode mode = WhitespaceMode::TRIM,
-                 SourceOffset start = InvalidSourceOffset,
-                 SourceOffset end = InvalidSourceOffset,
-                 SourceId sourceId = InvalidSourceId)
+inline void assertToken(TokenizedDataReader &reader, TokenId id,
+                        const std::string &text,
+                        const TokenSet &tokens = TokenSet{},
+                        WhitespaceMode mode = WhitespaceMode::TRIM,
+                        SourceOffset start = InvalidSourceOffset,
+                        SourceOffset end = InvalidSourceOffset,
+                        SourceId sourceId = InvalidSourceId,
+                        bool endAtWhitespace = false)
 {
 	Token token;
-	ASSERT_TRUE(reader.read(token, tokens, mode));
+	ASSERT_TRUE(reader.read(token, tokens, mode, endAtWhitespace));
 	EXPECT_EQ(id, token.id);
 	EXPECT_EQ(text, token.content);
 	if (start != InvalidSourceOffset) {
@@ -41,23 +43,32 @@ static void assertToken(TokenizedDataReader &reader, TokenId id,
 	EXPECT_EQ(sourceId, token.getLocation().getSourceId());
 }
 
-static void assertText(TokenizedDataReader &reader, const std::string &text,
-                const TokenSet &tokens = TokenSet{},
-                WhitespaceMode mode = WhitespaceMode::TRIM,
-                SourceOffset start = InvalidSourceOffset,
-                SourceOffset end = InvalidSourceOffset,
-                SourceId id = InvalidSourceId)
+inline void assertText(TokenizedDataReader &reader, const std::string &text,
+                       const TokenSet &tokens = TokenSet{},
+                       WhitespaceMode mode = WhitespaceMode::TRIM,
+                       SourceOffset start = InvalidSourceOffset,
+                       SourceOffset end = InvalidSourceOffset,
+                       SourceId id = InvalidSourceId)
 {
 	assertToken(reader, Tokens::Data, text, tokens, mode, start, end, id);
 }
 
-static void assertEnd(TokenizedDataReader &reader)
+inline void assertTextEndAtWhitespace(
+    TokenizedDataReader &reader, const std::string &text,
+    const TokenSet &tokens = TokenSet{},
+    WhitespaceMode mode = WhitespaceMode::TRIM,
+    SourceOffset start = InvalidSourceOffset,
+    SourceOffset end = InvalidSourceOffset, SourceId id = InvalidSourceId)
+{
+	assertToken(reader, Tokens::Data, text, tokens, mode, start, end, id, true);
+}
+
+inline void assertEnd(TokenizedDataReader &reader)
 {
 	Token token;
 	ASSERT_TRUE(reader.atEnd());
 	ASSERT_FALSE(reader.read(token));
 }
-
 }
 
 #endif /* _OUSIA_TOKENIZED_DATA_TEST_UTILS_HPP_ */