diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/core/parser/utils/TokenizedDataTest.cpp | 43 | ||||
| -rw-r--r-- | test/core/parser/utils/TokenizedDataTestUtils.hpp | 41 |
2 files changed, 59 insertions, 25 deletions
diff --git a/test/core/parser/utils/TokenizedDataTest.cpp b/test/core/parser/utils/TokenizedDataTest.cpp index 31346bd..e468a50 100644 --- a/test/core/parser/utils/TokenizedDataTest.cpp +++ b/test/core/parser/utils/TokenizedDataTest.cpp @@ -348,12 +348,13 @@ TEST(TokenizedData, specialTokenIndent) const TokenSet tokens{Tokens::Indent, Tokens::Dedent}; TokenizedDataReader reader = data.reader(); - assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, - 4, 4); + assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 4, + 4); assertText(reader, "test", tokens, WhitespaceMode::COLLAPSE, 4, 8); assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 10, 10); - assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, 10, 37); + assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, + 10, 37); assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE, 37, 37); assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE, @@ -372,11 +373,11 @@ TEST(TokenizedData, specialTokenIndent2) TokenizedDataReader reader = data.reader(); assertText(reader, "a", tokens, WhitespaceMode::COLLAPSE, 0, 1); - assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, - 3, 3); + assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 3, + 3); assertText(reader, "b", tokens, WhitespaceMode::COLLAPSE, 3, 4); - assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, - 7, 7); + assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 7, + 7); assertText(reader, "c", tokens, WhitespaceMode::COLLAPSE, 7, 8); assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 12, 12); @@ -403,12 +404,13 @@ TEST(TokenizedData, specialTokenIndentOverlap) data.mark(5, 4, 4); TokenizedDataReader reader = data.reader(); - assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, - 4, 4); + assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 4, + 4); assertToken(reader, 5, "test", tokens, WhitespaceMode::COLLAPSE, 4, 8); assertToken(reader, Tokens::Indent, "", tokens, WhitespaceMode::COLLAPSE, 10, 10); - assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, 10, 37); + assertText(reader, "test2 test3 test4", tokens, WhitespaceMode::COLLAPSE, + 10, 37); assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE, 37, 37); assertToken(reader, Tokens::Dedent, "", tokens, WhitespaceMode::COLLAPSE, @@ -417,5 +419,26 @@ TEST(TokenizedData, specialTokenIndentOverlap) assertEnd(reader); } +TEST(TokenizedData, endAtWhitespace) +{ + TokenizedData data; + data.append(" a b c d"); + // 0123456789012345 + // 0 1 + const TokenSet tokens{}; + + data.mark(5, 4, 4); + + TokenizedDataReader reader = data.reader(); + assertTextEndAtWhitespace(reader, " a", tokens, WhitespaceMode::PRESERVE, + 0, 5); + assertTextEndAtWhitespace(reader, " b", tokens, WhitespaceMode::PRESERVE, + 5, 10); + assertTextEndAtWhitespace(reader, " c", tokens, WhitespaceMode::PRESERVE, + 10, 12); + assertTextEndAtWhitespace(reader, " d", tokens, WhitespaceMode::PRESERVE, + 12, 16); + assertEnd(reader); +} } diff --git a/test/core/parser/utils/TokenizedDataTestUtils.hpp b/test/core/parser/utils/TokenizedDataTestUtils.hpp index c384f9d..30f72ae 100644 --- a/test/core/parser/utils/TokenizedDataTestUtils.hpp +++ b/test/core/parser/utils/TokenizedDataTestUtils.hpp @@ -21,15 +21,17 @@ namespace ousia { -static void assertToken(TokenizedDataReader &reader, TokenId id, - const std::string &text, const TokenSet &tokens = TokenSet{}, - WhitespaceMode mode = WhitespaceMode::TRIM, - SourceOffset start = InvalidSourceOffset, - SourceOffset end = InvalidSourceOffset, - SourceId sourceId = InvalidSourceId) +inline void assertToken(TokenizedDataReader &reader, TokenId id, + const std::string &text, + const TokenSet &tokens = TokenSet{}, + WhitespaceMode mode = WhitespaceMode::TRIM, + SourceOffset start = InvalidSourceOffset, + SourceOffset end = InvalidSourceOffset, + SourceId sourceId = InvalidSourceId, + bool endAtWhitespace = false) { Token token; - ASSERT_TRUE(reader.read(token, tokens, mode)); + ASSERT_TRUE(reader.read(token, tokens, mode, endAtWhitespace)); EXPECT_EQ(id, token.id); EXPECT_EQ(text, token.content); if (start != InvalidSourceOffset) { @@ -41,23 +43,32 @@ static void assertToken(TokenizedDataReader &reader, TokenId id, EXPECT_EQ(sourceId, token.getLocation().getSourceId()); } -static void assertText(TokenizedDataReader &reader, const std::string &text, - const TokenSet &tokens = TokenSet{}, - WhitespaceMode mode = WhitespaceMode::TRIM, - SourceOffset start = InvalidSourceOffset, - SourceOffset end = InvalidSourceOffset, - SourceId id = InvalidSourceId) +inline void assertText(TokenizedDataReader &reader, const std::string &text, + const TokenSet &tokens = TokenSet{}, + WhitespaceMode mode = WhitespaceMode::TRIM, + SourceOffset start = InvalidSourceOffset, + SourceOffset end = InvalidSourceOffset, + SourceId id = InvalidSourceId) { assertToken(reader, Tokens::Data, text, tokens, mode, start, end, id); } -static void assertEnd(TokenizedDataReader &reader) +inline void assertTextEndAtWhitespace( + TokenizedDataReader &reader, const std::string &text, + const TokenSet &tokens = TokenSet{}, + WhitespaceMode mode = WhitespaceMode::TRIM, + SourceOffset start = InvalidSourceOffset, + SourceOffset end = InvalidSourceOffset, SourceId id = InvalidSourceId) +{ + assertToken(reader, Tokens::Data, text, tokens, mode, start, end, id, true); +} + +inline void assertEnd(TokenizedDataReader &reader) { Token token; ASSERT_TRUE(reader.atEnd()); ASSERT_FALSE(reader.read(token)); } - } #endif /* _OUSIA_TOKENIZED_DATA_TEST_UTILS_HPP_ */ |