From ec31aae293f88e36190aa32169a97a776873567a Mon Sep 17 00:00:00 2001 From: Benjamin Paassen Date: Wed, 19 Nov 2014 19:42:07 +0100 Subject: fixed a bug preventing the Tokenizer from finding the right token if a parse was incomplete beforehand. Also cleared the buffers if a subclass returns false from doPrepare. Failing to clear the buffers lead to subsequent problems. --- test/core/utils/TokenizerTest.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'test/core/utils/TokenizerTest.cpp') diff --git a/test/core/utils/TokenizerTest.cpp b/test/core/utils/TokenizerTest.cpp index ba06c33..79cc01d 100644 --- a/test/core/utils/TokenizerTest.cpp +++ b/test/core/utils/TokenizerTest.cpp @@ -93,5 +93,33 @@ TEST(Tokenizer, testTokenization) } ASSERT_FALSE(tokenizer.next(t)); } + +TEST(Tokenizer, testIncompleteTokens) +{ + TokenTreeNode root{{{"ab", 1}, {"c", 2}}}; + + BufferedCharReader reader; + reader.feed("ac"); + // 1234567890 + // 0 1 + + std::vector expected = { + {TOKEN_TEXT, "a", 1, 1, 2, 1}, + {2, "c", 2, 1, 3, 1}}; + + Tokenizer tokenizer{reader, root}; + + Token t; + for (auto &te : expected) { + ASSERT_TRUE(tokenizer.next(t)); + ASSERT_EQ(te.tokenId, t.tokenId); + ASSERT_EQ(te.content, t.content); + ASSERT_EQ(te.startColumn, t.startColumn); + ASSERT_EQ(te.startLine, t.startLine); + ASSERT_EQ(te.endColumn, t.endColumn); + ASSERT_EQ(te.endLine, t.endLine); + } + ASSERT_FALSE(tokenizer.next(t)); +} } } -- cgit v1.2.3