summaryrefslogtreecommitdiff
path: root/test/core
diff options
context:
space:
mode:
Diffstat (limited to 'test/core')
-rw-r--r--test/core/CodeTokenizerTest.cpp100
-rw-r--r--test/core/RangeSetTest.cpp10
-rw-r--r--test/core/StandaloneEnvironment.hpp4
-rw-r--r--test/core/TokenizerTest.cpp118
-rw-r--r--test/core/common/UtilsTest.cpp58
-rw-r--r--test/core/model/DomainTest.cpp12
-rw-r--r--test/core/parser/ParserStackTest.cpp177
-rw-r--r--test/core/parser/ParserStateTest.cpp77
-rw-r--r--test/core/parser/stack/StackTest.cpp666
-rw-r--r--test/core/parser/stack/StateTest.cpp79
-rw-r--r--test/core/parser/utils/TokenTrieTest.cpp92
-rw-r--r--test/core/parser/utils/TokenizerTest.cpp414
12 files changed, 1313 insertions, 494 deletions
diff --git a/test/core/CodeTokenizerTest.cpp b/test/core/CodeTokenizerTest.cpp
deleted file mode 100644
index 2d4d5a7..0000000
--- a/test/core/CodeTokenizerTest.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- Ousía
- Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/CodeTokenizer.hpp>
-
-namespace ousia {
-
-static const int BLOCK_COMMENT = 30;
-static const int LINE_COMMENT = 31;
-static const int STRING = 20;
-static const int ESCAPE = 21;
-static const int LINEBREAK = 21;
-static const int CURLY_OPEN = 40;
-static const int CURLY_CLOSE = 41;
-
-TEST(CodeTokenizer, testTokenizer)
-{
- CharReader reader{
- "/**\n" // 1
- " * Some Block Comment\n" // 2
- " */\n" // 3
- "var my_string = 'My \\'String\\'';\n" // 4
- "// and a line comment\n" // 5
- "var my_obj = { a = 4;}", 0}; // 6
- // 123456789012345678901234567890123456789
- // 0 1 2 3
- TokenTreeNode root{{{"/*", 1},
- {"*/", 2},
- {"//", 3},
- {"'", 4},
- {"\\", 5},
- {"{", CURLY_OPEN},
- {"}", CURLY_CLOSE},
- {"\n", 6}}};
- std::map<int, CodeTokenDescriptor> descriptors{
- // the block comment start Token has the id 1 and if the Tokenizer
- // returns a Block Comment Token that should have the id 10.
- {1, {CodeTokenMode::BLOCK_COMMENT_START, BLOCK_COMMENT}},
- {2, {CodeTokenMode::BLOCK_COMMENT_END, BLOCK_COMMENT}},
- {3, {CodeTokenMode::LINE_COMMENT, LINE_COMMENT}},
- {4, {CodeTokenMode::STRING_START_END, STRING}},
- {5, {CodeTokenMode::ESCAPE, ESCAPE}},
- {6, {CodeTokenMode::LINEBREAK, LINEBREAK}}};
-
- std::vector<Token> expected = {
- {BLOCK_COMMENT, "*\n * Some Block Comment\n ", SourceLocation{0, 0, 29}},
- {LINEBREAK, "\n", SourceLocation{0, 29, 30}},
- {TOKEN_TEXT, "var", SourceLocation{0, 30, 33}},
- {TOKEN_TEXT, "my_string", SourceLocation{0, 34, 43}},
- {TOKEN_TEXT, "=", SourceLocation{0, 44, 45}},
- {STRING, "My 'String'", SourceLocation{0, 46, 61}},
- {TOKEN_TEXT, ";", SourceLocation{0, 61, 62}},
- {LINEBREAK, "\n", SourceLocation{0, 62, 63}},
- // this is slightly counter-intuitive but makes sense if you think about
- // it: As a line comment is ended by a line break the line break is
- // technically still a part of the line comment and thus the ending
- // is in the next line.
- {LINE_COMMENT, " and a line comment", SourceLocation{0, 63, 85}},
- {TOKEN_TEXT, "var", SourceLocation{0, 85, 88}},
- {TOKEN_TEXT, "my_obj", SourceLocation{0, 89, 95}},
- {TOKEN_TEXT, "=", SourceLocation{0, 96, 97}},
- {CURLY_OPEN, "{", SourceLocation{0, 98, 99}},
- {TOKEN_TEXT, "a", SourceLocation{0, 100, 101}},
- {TOKEN_TEXT, "=", SourceLocation{0, 102, 103}},
- {TOKEN_TEXT, "4;", SourceLocation{0, 104, 106}},
- {CURLY_CLOSE, "}", SourceLocation{0, 106, 107}},
- };
-
- CodeTokenizer tokenizer{reader, root, descriptors};
-
- Token t;
- for (auto &te : expected) {
- EXPECT_TRUE(tokenizer.next(t));
- EXPECT_EQ(te.tokenId, t.tokenId);
- EXPECT_EQ(te.content, t.content);
- EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
- EXPECT_EQ(te.location.getStart(), t.location.getStart());
- EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
- }
- ASSERT_FALSE(tokenizer.next(t));
-}
-}
-
diff --git a/test/core/RangeSetTest.cpp b/test/core/RangeSetTest.cpp
index cbf8f59..446ee51 100644
--- a/test/core/RangeSetTest.cpp
+++ b/test/core/RangeSetTest.cpp
@@ -110,7 +110,7 @@ TEST(RangeSet, Merge)
s.merge(Range<int>(40, 50));
s.merge(Range<int>(60, 70));
{
- ASSERT_EQ(ranges.size(), 4);
+ ASSERT_EQ(ranges.size(), 4U);
auto it = ranges.begin();
ASSERT_EQ((*it).start, 0);
@@ -132,7 +132,7 @@ TEST(RangeSet, Merge)
// Now insert an element which spans the second and third element
s.merge(Range<int>(15, 55));
{
- ASSERT_EQ(ranges.size(), 3);
+ ASSERT_EQ(ranges.size(), 3U);
auto it = ranges.begin();
ASSERT_EQ((*it).start, 0);
@@ -150,7 +150,7 @@ TEST(RangeSet, Merge)
// Now insert an element which expands the first element
s.merge(Range<int>(-10, 11));
{
- ASSERT_EQ(ranges.size(), 3);
+ ASSERT_EQ(ranges.size(), 3U);
auto it = ranges.begin();
ASSERT_EQ((*it).start, -10);
@@ -168,7 +168,7 @@ TEST(RangeSet, Merge)
// Now insert an element which merges the last two elements
s.merge(Range<int>(13, 70));
{
- ASSERT_EQ(ranges.size(), 2);
+ ASSERT_EQ(ranges.size(), 2U);
auto it = ranges.begin();
ASSERT_EQ((*it).start, -10);
@@ -182,7 +182,7 @@ TEST(RangeSet, Merge)
// Now insert an element which merges the remaining elements
s.merge(Range<int>(-9, 12));
{
- ASSERT_EQ(ranges.size(), 1);
+ ASSERT_EQ(ranges.size(), 1U);
auto it = ranges.begin();
ASSERT_EQ((*it).start, -10);
diff --git a/test/core/StandaloneEnvironment.hpp b/test/core/StandaloneEnvironment.hpp
index a9dcdce..790bad4 100644
--- a/test/core/StandaloneEnvironment.hpp
+++ b/test/core/StandaloneEnvironment.hpp
@@ -31,6 +31,10 @@
namespace ousia {
+/**
+ * StandaloneEnvironment is a class used for quickly setting up an entire
+ * environment needed for running an Ousia instance.
+ */
struct StandaloneEnvironment {
ConcreteLogger &logger;
Manager manager;
diff --git a/test/core/TokenizerTest.cpp b/test/core/TokenizerTest.cpp
deleted file mode 100644
index c53f93d..0000000
--- a/test/core/TokenizerTest.cpp
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- Ousía
- Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/common/CharReader.hpp>
-
-#include <core/Tokenizer.hpp>
-
-namespace ousia {
-TEST(TokenTreeNode, testConstructor)
-{
- TokenTreeNode root{{{"a", 1}, {"aab", 2}, {"aac", 3}, {"abd", 4}}};
-
- ASSERT_EQ(-1, root.tokenId);
- ASSERT_EQ(1U, root.children.size());
- ASSERT_TRUE(root.children.find('a') != root.children.end());
-
- const TokenTreeNode &a = root.children.at('a');
- ASSERT_EQ(1, a.tokenId);
- ASSERT_EQ(2U, a.children.size());
- ASSERT_TRUE(a.children.find('a') != a.children.end());
- ASSERT_TRUE(a.children.find('b') != a.children.end());
-
- const TokenTreeNode &aa = a.children.at('a');
- ASSERT_EQ(-1, aa.tokenId);
- ASSERT_EQ(2U, aa.children.size());
- ASSERT_TRUE(aa.children.find('b') != aa.children.end());
- ASSERT_TRUE(aa.children.find('c') != aa.children.end());
-
- const TokenTreeNode &aab = aa.children.at('b');
- ASSERT_EQ(2, aab.tokenId);
- ASSERT_EQ(0U, aab.children.size());
-
- const TokenTreeNode &aac = aa.children.at('c');
- ASSERT_EQ(3, aac.tokenId);
- ASSERT_EQ(0U, aac.children.size());
-
- const TokenTreeNode &ab = a.children.at('b');
- ASSERT_EQ(-1, ab.tokenId);
- ASSERT_EQ(1U, ab.children.size());
- ASSERT_TRUE(ab.children.find('d') != ab.children.end());
-
- const TokenTreeNode &abd = ab.children.at('d');
- ASSERT_EQ(4, abd.tokenId);
- ASSERT_EQ(0U, abd.children.size());
-}
-
-TEST(Tokenizer, testTokenization)
-{
- TokenTreeNode root{{{"/", 1}, {"/*", 2}, {"*/", 3}}};
-
- CharReader reader{"Test/Test /* Block Comment */", 0};
- // 012345678901234567890123456789
- // 0 1 2
-
- std::vector<Token> expected = {
- {TOKEN_TEXT, "Test", SourceLocation{0, 0, 4}},
- {1, "/", SourceLocation{0, 4, 5}},
- {TOKEN_TEXT, "Test ", SourceLocation{0, 5, 10}},
- {2, "/*", SourceLocation{0, 10, 12}},
- {TOKEN_TEXT, " Block Comment ", SourceLocation{0, 12, 27}},
- {3, "*/", SourceLocation{0, 27, 29}}};
-
- Tokenizer tokenizer{reader, root};
-
- Token t;
- for (auto &te : expected) {
- EXPECT_TRUE(tokenizer.next(t));
- EXPECT_EQ(te.tokenId, t.tokenId);
- EXPECT_EQ(te.content, t.content);
- EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
- EXPECT_EQ(te.location.getStart(), t.location.getStart());
- EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
- }
- ASSERT_FALSE(tokenizer.next(t));
-}
-
-TEST(Tokenizer, testIncompleteTokens)
-{
- TokenTreeNode root{{{"ab", 1}, {"c", 2}}};
-
- CharReader reader{"ac", 0};
-
- std::vector<Token> expected = {
- {TOKEN_TEXT, "a", SourceLocation{0, 0, 1}},
- {2, "c", SourceLocation{0, 1, 2}}};
-
- Tokenizer tokenizer{reader, root};
-
- Token t;
- for (auto &te : expected) {
- EXPECT_TRUE(tokenizer.next(t));
- EXPECT_EQ(te.tokenId, t.tokenId);
- EXPECT_EQ(te.content, t.content);
- EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
- EXPECT_EQ(te.location.getStart(), t.location.getStart());
- EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
- }
- ASSERT_FALSE(tokenizer.next(t));
-}
-}
-
diff --git a/test/core/common/UtilsTest.cpp b/test/core/common/UtilsTest.cpp
index 917f45c..7801296 100644
--- a/test/core/common/UtilsTest.cpp
+++ b/test/core/common/UtilsTest.cpp
@@ -24,22 +24,40 @@ namespace ousia {
TEST(Utils, isIdentifier)
{
- ASSERT_TRUE(Utils::isIdentifier("test"));
- ASSERT_TRUE(Utils::isIdentifier("t0-_est"));
- ASSERT_FALSE(Utils::isIdentifier("_t0-_EST"));
- ASSERT_FALSE(Utils::isIdentifier("-t0-_EST"));
- ASSERT_FALSE(Utils::isIdentifier("0t-_EST"));
- ASSERT_FALSE(Utils::isIdentifier("invalid key"));
+ EXPECT_TRUE(Utils::isIdentifier("test"));
+ EXPECT_TRUE(Utils::isIdentifier("t0-_est"));
+ EXPECT_FALSE(Utils::isIdentifier("_t0-_EST"));
+ EXPECT_FALSE(Utils::isIdentifier("-t0-_EST"));
+ EXPECT_FALSE(Utils::isIdentifier("0t-_EST"));
+ EXPECT_FALSE(Utils::isIdentifier("_A"));
+ EXPECT_FALSE(Utils::isIdentifier("invalid key"));
+ EXPECT_FALSE(Utils::isIdentifier(""));
}
-TEST(Utils, trim)
+
+TEST(Utils, isNamespacedIdentifier)
{
- ASSERT_EQ("hello world", Utils::trim("\t hello world \n\r\t"));
- ASSERT_EQ("hello world", Utils::trim("hello world \n\r\t"));
- ASSERT_EQ("hello world", Utils::trim(" hello world"));
- ASSERT_EQ("hello world", Utils::trim("hello world"));
+ EXPECT_TRUE(Utils::isNamespacedIdentifier("test"));
+ EXPECT_TRUE(Utils::isNamespacedIdentifier("t0-_est"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("_t0-_EST"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("-t0-_EST"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("0t-_EST"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("invalid key"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("_A"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier(""));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier(":"));
+ EXPECT_TRUE(Utils::isNamespacedIdentifier("test:a"));
+ EXPECT_TRUE(Utils::isNamespacedIdentifier("t0-_est:b"));
+ EXPECT_TRUE(Utils::isNamespacedIdentifier("test:test"));
+ EXPECT_TRUE(Utils::isNamespacedIdentifier("t0-_est:t0-_est"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("test:_A"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("test::a"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier(":test"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("t0-_est:_t0-_EST"));
+ EXPECT_FALSE(Utils::isNamespacedIdentifier("t0-_est: b"));
}
+
TEST(Utils, split)
{
ASSERT_EQ(std::vector<std::string>({"ab"}), Utils::split("ab", '.'));
@@ -73,5 +91,23 @@ TEST(Utils, extractFileExtension)
ASSERT_EQ("ext", Utils::extractFileExtension("foo.bar/test.EXT"));
}
+TEST(Utils, startsWith)
+{
+ ASSERT_TRUE(Utils::startsWith("foobar", "foo"));
+ ASSERT_TRUE(Utils::startsWith("foo", "foo"));
+ ASSERT_FALSE(Utils::startsWith("foo", "foobar"));
+ ASSERT_FALSE(Utils::startsWith("foobar", "bar"));
+ ASSERT_TRUE(Utils::startsWith("foo", ""));
+}
+
+TEST(Utils, endsWith)
+{
+ ASSERT_FALSE(Utils::endsWith("foobar", "foo"));
+ ASSERT_TRUE(Utils::endsWith("foo", "foo"));
+ ASSERT_FALSE(Utils::endsWith("foo", "foobar"));
+ ASSERT_TRUE(Utils::endsWith("foobar", "bar"));
+ ASSERT_TRUE(Utils::endsWith("foo", ""));
+}
+
}
diff --git a/test/core/model/DomainTest.cpp b/test/core/model/DomainTest.cpp
index 8fcbdf2..4cb4331 100644
--- a/test/core/model/DomainTest.cpp
+++ b/test/core/model/DomainTest.cpp
@@ -242,7 +242,7 @@ TEST(Descriptor, getDefaultFields)
A->createPrimitiveFieldDescriptor(sys->getStringType(), logger);
// now we should find that.
auto fields = A->getDefaultFields();
- ASSERT_EQ(1, fields.size());
+ ASSERT_EQ(1U, fields.size());
ASSERT_EQ(A_prim_field, fields[0]);
// remove that field from A and add it to another class.
@@ -258,7 +258,7 @@ TEST(Descriptor, getDefaultFields)
// but we should find it again if we set B as superclass of A.
A->setSuperclass(B, logger);
fields = A->getDefaultFields();
- ASSERT_EQ(1, fields.size());
+ ASSERT_EQ(1U, fields.size());
ASSERT_EQ(A_prim_field, fields[0]);
// and we should not be able to find it if we override the field.
@@ -277,7 +277,7 @@ TEST(Descriptor, getDefaultFields)
// now we should find that.
fields = A->getDefaultFields();
- ASSERT_EQ(1, fields.size());
+ ASSERT_EQ(1U, fields.size());
ASSERT_EQ(C_field, fields[0]);
// add another transparent child class to A with a daughter class that has
@@ -296,7 +296,7 @@ TEST(Descriptor, getDefaultFields)
// now we should find both primitive fields, but the C field first.
fields = A->getDefaultFields();
- ASSERT_EQ(2, fields.size());
+ ASSERT_EQ(2U, fields.size());
ASSERT_EQ(C_field, fields[0]);
ASSERT_EQ(F_field, fields[1]);
}
@@ -321,7 +321,7 @@ TEST(Descriptor, getPermittedChildren)
* in between.
*/
NodeVector<StructuredClass> children = book->getPermittedChildren();
- ASSERT_EQ(3, children.size());
+ ASSERT_EQ(3U, children.size());
ASSERT_EQ(section, children[0]);
ASSERT_EQ(paragraph, children[1]);
ASSERT_EQ(text, children[2]);
@@ -331,7 +331,7 @@ TEST(Descriptor, getPermittedChildren)
mgr, "Subclass", domain, Cardinality::any(), text, true, false)};
// And that should be in the result list as well now.
children = book->getPermittedChildren();
- ASSERT_EQ(4, children.size());
+ ASSERT_EQ(4U, children.size());
ASSERT_EQ(section, children[0]);
ASSERT_EQ(paragraph, children[1]);
ASSERT_EQ(text, children[2]);
diff --git a/test/core/parser/ParserStackTest.cpp b/test/core/parser/ParserStackTest.cpp
deleted file mode 100644
index 3a0decb..0000000
--- a/test/core/parser/ParserStackTest.cpp
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- Ousía
- Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <iostream>
-
-#include <gtest/gtest.h>
-
-#include <core/parser/ParserStack.hpp>
-#include <core/StandaloneEnvironment.hpp>
-
-namespace ousia {
-
-ConcreteLogger logger;
-
-static int startCount = 0;
-static int endCount = 0;
-static int dataCount = 0;
-
-class TestHandler : public Handler {
-public:
- using Handler::Handler;
-
- void start(Variant::mapType &args) override { startCount++; }
-
- void end() override { endCount++; }
-
- void data(const std::string &data, int field) override { dataCount++; }
-
- static Handler *create(const HandlerData &data)
- {
- return new TestHandler(data);
- }
-};
-
-namespace ParserStates {
-static const ParserState Document =
- ParserStateBuilder().parent(&None).elementHandler(TestHandler::create);
-static const ParserState Body = ParserStateBuilder()
- .parent(&Document)
- .elementHandler(TestHandler::create);
-static const ParserState Empty =
- ParserStateBuilder().parent(&Document).elementHandler(TestHandler::create);
-static const ParserState Special =
- ParserStateBuilder().parent(&All).elementHandler(TestHandler::create);
-static const ParserState Arguments =
- ParserStateBuilder()
- .parent(&None)
- .elementHandler(TestHandler::create)
- .arguments({Argument::Int("a"), Argument::String("b")});
-static const ParserState BodyChildren =
- ParserStateBuilder()
- .parent(&Body)
- .elementHandler(TestHandler::create);
-
-static const std::multimap<std::string, const ParserState *> TestHandlers{
- {"document", &Document},
- {"body", &Body},
- {"empty", &Empty},
- {"special", &Special},
- {"arguments", &Arguments},
- {"*", &BodyChildren}};
-}
-
-TEST(ParserStack, simpleTest)
-{
- StandaloneEnvironment env(logger);
- ParserStack s{env.context, ParserStates::TestHandlers};
-
- startCount = 0;
- endCount = 0;
- dataCount = 0;
-
- EXPECT_EQ("", s.currentCommandName());
- EXPECT_EQ(&ParserStates::None, &s.currentState());
-
- s.start("document", {});
- s.data("test1");
-
- EXPECT_EQ("document", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Document, &s.currentState());
- EXPECT_EQ(1, startCount);
- EXPECT_EQ(1, dataCount);
-
- s.start("body", {});
- s.data("test2");
- EXPECT_EQ("body", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Body, &s.currentState());
- EXPECT_EQ(2, startCount);
- EXPECT_EQ(2, dataCount);
-
- s.start("inner", {});
- EXPECT_EQ("inner", s.currentCommandName());
- EXPECT_EQ(&ParserStates::BodyChildren, &s.currentState());
- s.end();
- EXPECT_EQ(3, startCount);
- EXPECT_EQ(1, endCount);
-
- s.end();
- EXPECT_EQ(2, endCount);
-
- EXPECT_EQ("document", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
- s.start("body", {});
- s.data("test3");
- EXPECT_EQ("body", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Body, &s.currentState());
- s.end();
- EXPECT_EQ(4, startCount);
- EXPECT_EQ(3, dataCount);
- EXPECT_EQ(3, endCount);
-
- EXPECT_EQ("document", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
- s.end();
- EXPECT_EQ(4, endCount);
-
- EXPECT_EQ("", s.currentCommandName());
- EXPECT_EQ(&ParserStates::None, &s.currentState());
-}
-
-TEST(ParserStack, errorHandling)
-{
- StandaloneEnvironment env(logger);
- ParserStack s{env.context, ParserStates::TestHandlers};
-
- EXPECT_THROW(s.start("body", {}), OusiaException);
- s.start("document", {});
- EXPECT_THROW(s.start("document", {}), OusiaException);
- s.start("empty", {});
- EXPECT_THROW(s.start("body", {}), OusiaException);
- s.start("special", {});
- s.end();
- s.end();
- s.end();
- EXPECT_EQ(&ParserStates::None, &s.currentState());
- ASSERT_THROW(s.end(), OusiaException);
- ASSERT_THROW(s.data("test", 1), OusiaException);
-}
-
-TEST(ParserStack, validation)
-{
- StandaloneEnvironment env(logger);
- ParserStack s{env.context, ParserStates::TestHandlers};
-
- logger.reset();
- s.start("arguments", {});
- EXPECT_TRUE(logger.hasError());
- s.end();
-
- s.start("arguments", {{"a", 5}});
- EXPECT_TRUE(logger.hasError());
- s.end();
-
- logger.reset();
- s.start("arguments", {{"a", 5}, {"b", "test"}});
- EXPECT_FALSE(logger.hasError());
- s.end();
-}
-}
-
diff --git a/test/core/parser/ParserStateTest.cpp b/test/core/parser/ParserStateTest.cpp
deleted file mode 100644
index 91d8dcd..0000000
--- a/test/core/parser/ParserStateTest.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- Ousía
- Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/common/Rtti.hpp>
-#include <core/parser/ParserState.hpp>
-
-namespace ousia {
-
-static const Rtti t1;
-static const Rtti t2;
-static const Rtti t3;
-static const Rtti t4;
-static const Rtti t5;
-
-static const ParserState s1 = ParserStateBuilder().createdNodeType(&t1);
-static const ParserState s2a =
- ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s2b =
- ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s3 =
- ParserStateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
-static const ParserState s4 =
- ParserStateBuilder().parent(&s3).createdNodeType(&t4);
-static const ParserState s5 =
- ParserStateBuilder().parent(&s2b).createdNodeType(&t5);
-
-TEST(ParserStateDeductor, deduce)
-{
- using Result = std::vector<const ParserState *>;
- using Signature = std::vector<const Rtti *>;
- std::vector<const ParserState *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
-
- // Should not crash on empty signature
- ASSERT_EQ(Result{}, ParserStateDeductor(Signature{}, states).deduce());
-
- // Try repeating signature elements
- ASSERT_EQ(Result({&s1}),
- ParserStateDeductor(Signature({&t1}), states).deduce());
- ASSERT_EQ(Result({&s1}),
- ParserStateDeductor(Signature({&t1, &t1}), states).deduce());
- ASSERT_EQ(Result({&s1}),
- ParserStateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
-
- // Go to another state
- ASSERT_EQ(Result({&s2a, &s2b}),
- ParserStateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
- ASSERT_EQ(Result({&s4}),
- ParserStateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
-
- // Skip one state
- ASSERT_EQ(Result({&s4}),
- ParserStateDeductor(Signature({&t2, &t4}), states).deduce());
-
- // Impossible signature
- ASSERT_EQ(Result({}),
- ParserStateDeductor(Signature({&t4, &t5}), states).deduce());
-
-}
-}
-
diff --git a/test/core/parser/stack/StackTest.cpp b/test/core/parser/stack/StackTest.cpp
new file mode 100644
index 0000000..321d471
--- /dev/null
+++ b/test/core/parser/stack/StackTest.cpp
@@ -0,0 +1,666 @@
+/*
+ Ousía
+ Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <iostream>
+
+#include <gtest/gtest.h>
+
+#include <core/frontend/TerminalLogger.hpp>
+#include <core/parser/stack/Handler.hpp>
+#include <core/parser/stack/Stack.hpp>
+#include <core/parser/stack/State.hpp>
+
+#include <core/StandaloneEnvironment.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+// Build an instance of the StandaloneEnvironment used for this unit test
+static TerminalLogger logger(std::cerr, true);
+// static ConcreteLogger logger;
+static StandaloneEnvironment env(logger);
+
+namespace {
+
+struct Tracker {
+ int startCount;
+ int endCount;
+ int fieldStartCount;
+ int fieldEndCount;
+ int annotationStartCount;
+ int annotationEndCount;
+ int dataCount;
+
+ Variant::mapType startArgs;
+ bool fieldStartIsDefault;
+ size_t fieldStartIdx;
+ Variant annotationStartClassName;
+ Variant::mapType annotationStartArgs;
+ Variant annotationEndClassName;
+ Variant annotationEndElementName;
+ Variant dataData;
+
+ bool startResult;
+ bool fieldStartSetIsDefault;
+ bool fieldStartResult;
+ bool annotationStartResult;
+ bool annotationEndResult;
+ bool dataResult;
+
+ Tracker() { reset(); }
+
+ void reset()
+ {
+ startCount = 0;
+ endCount = 0;
+ fieldStartCount = 0;
+ fieldEndCount = 0;
+ annotationStartCount = 0;
+ annotationEndCount = 0;
+ dataCount = 0;
+
+ startArgs = Variant::mapType{};
+ fieldStartIsDefault = false;
+ fieldStartIdx = 0;
+ annotationStartClassName = Variant::fromString(std::string{});
+ annotationStartArgs = Variant::mapType{};
+ annotationEndClassName = Variant::fromString(std::string{});
+ annotationEndElementName = Variant::fromString(std::string{});
+ dataData = Variant::fromString(std::string{});
+
+ startResult = true;
+ fieldStartSetIsDefault = false;
+ fieldStartResult = true;
+ annotationStartResult = true;
+ annotationEndResult = true;
+ dataResult = true;
+ }
+
+ void expect(int startCount, int endCount, int fieldStartCount,
+ int fieldEndCount, int annotationStartCount,
+ int annotationEndCount, int dataCount)
+ {
+ EXPECT_EQ(startCount, this->startCount);
+ EXPECT_EQ(endCount, this->endCount);
+ EXPECT_EQ(fieldStartCount, this->fieldStartCount);
+ EXPECT_EQ(fieldEndCount, this->fieldEndCount);
+ EXPECT_EQ(annotationStartCount, this->annotationStartCount);
+ EXPECT_EQ(annotationEndCount, this->annotationEndCount);
+ EXPECT_EQ(dataCount, this->dataCount);
+ }
+};
+
+static Tracker tracker;
+
+class TestHandler : public Handler {
+private:
+ TestHandler(const HandlerData &handlerData) : Handler(handlerData) {}
+
+public:
+ bool start(Variant::mapType &args) override
+ {
+ tracker.startCount++;
+ tracker.startArgs = args;
+ if (!tracker.startResult) {
+ logger().error(
+ "The TestHandler was told not to allow a field start. So it "
+ "doesn't. The TestHandler always obeys its master.");
+ }
+ return tracker.startResult;
+ }
+
+ void end() override { tracker.endCount++; }
+
+ bool fieldStart(bool &isDefault, size_t fieldIdx) override
+ {
+ tracker.fieldStartCount++;
+ tracker.fieldStartIsDefault = isDefault;
+ tracker.fieldStartIdx = fieldIdx;
+ if (tracker.fieldStartSetIsDefault) {
+ isDefault = true;
+ }
+ return tracker.fieldStartResult;
+ }
+
+ void fieldEnd() override { tracker.fieldEndCount++; }
+
+ bool annotationStart(const Variant &className,
+ Variant::mapType &args) override
+ {
+ tracker.annotationStartCount++;
+ tracker.annotationStartClassName = className;
+ tracker.annotationStartArgs = args;
+ return tracker.annotationStartResult;
+ }
+
+ bool annotationEnd(const Variant &className,
+ const Variant &elementName) override
+ {
+ tracker.annotationEndCount++;
+ tracker.annotationEndClassName = className;
+ tracker.annotationEndElementName = elementName;
+ return tracker.annotationEndResult;
+ }
+
+ bool data(Variant &data) override
+ {
+ tracker.dataCount++;
+ tracker.dataData = data;
+ return tracker.dataResult;
+ }
+
+ static Handler *create(const HandlerData &handlerData)
+ {
+ return new TestHandler(handlerData);
+ }
+};
+}
+
+namespace States {
+static const State Document =
+ StateBuilder().parent(&None).elementHandler(TestHandler::create);
+static const State Body =
+ StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Empty =
+ StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Special =
+ StateBuilder().parent(&All).elementHandler(TestHandler::create);
+static const State Arguments =
+ StateBuilder().parent(&None).elementHandler(TestHandler::create).arguments(
+ {Argument::Int("a"), Argument::String("b")});
+static const State BodyChildren =
+ StateBuilder().parent(&Body).elementHandler(TestHandler::create);
+static const State Any =
+ StateBuilder().parents({&None, &Any}).elementHandler(TestHandler::create);
+
+static const std::multimap<std::string, const State *> TestHandlers{
+ {"document", &Document},
+ {"body", &Body},
+ {"empty", &Empty},
+ {"special", &Special},
+ {"arguments", &Arguments},
+ {"*", &BodyChildren}};
+
+static const std::multimap<std::string, const State *> AnyHandlers{{"*", &Any}};
+}
+
+TEST(Stack, basicTest)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::TestHandlers};
+
+ EXPECT_EQ("", s.currentCommandName());
+ EXPECT_EQ(&States::None, &s.currentState());
+
+ s.command("document", {});
+ s.fieldStart(true);
+ s.data("test1");
+
+ EXPECT_EQ("document", s.currentCommandName());
+ EXPECT_EQ(&States::Document, &s.currentState());
+ tracker.expect(1, 0, 1, 0, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("body", {});
+ s.fieldStart(true);
+ s.data("test2");
+ EXPECT_EQ("body", s.currentCommandName());
+ EXPECT_EQ(&States::Body, &s.currentState());
+ tracker.expect(2, 0, 2, 0, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("inner", {});
+ s.fieldStart(true);
+ EXPECT_EQ("inner", s.currentCommandName());
+ EXPECT_EQ(&States::BodyChildren, &s.currentState());
+
+ s.fieldEnd();
+ tracker.expect(3, 1, 3, 1, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldEnd();
+ EXPECT_EQ("document", s.currentCommandName());
+ EXPECT_EQ(&States::Document, &s.currentState());
+ tracker.expect(3, 2, 3, 2, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("body", {});
+ s.fieldStart(true);
+ s.data("test3");
+ EXPECT_EQ("body", s.currentCommandName());
+ EXPECT_EQ(&States::Body, &s.currentState());
+ s.fieldEnd();
+ tracker.expect(4, 3, 4, 3, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+
+ EXPECT_EQ("document", s.currentCommandName());
+ EXPECT_EQ(&States::Document, &s.currentState());
+
+ s.fieldEnd();
+ tracker.expect(4, 4, 4, 4, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+
+ EXPECT_EQ("", s.currentCommandName());
+ EXPECT_EQ(&States::None, &s.currentState());
+ }
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidCommands)
+{
+ Stack s{env.context, States::TestHandlers};
+ tracker.reset();
+ EXPECT_THROW(s.command("body", {}), LoggableException);
+ s.command("document", {});
+ s.fieldStart(true);
+ EXPECT_THROW(s.command("document", {}), LoggableException);
+ s.command("empty", {});
+ s.fieldStart(true);
+ EXPECT_THROW(s.command("body", {}), LoggableException);
+ s.command("special", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ s.fieldEnd();
+ s.fieldEnd();
+ EXPECT_EQ(&States::None, &s.currentState());
+ ASSERT_THROW(s.fieldEnd(), LoggableException);
+ ASSERT_THROW(s.data("test"), LoggableException);
+}
+
+TEST(Stack, validation)
+{
+ Stack s{env.context, States::TestHandlers};
+ tracker.reset();
+ logger.reset();
+
+ s.command("arguments", {});
+ EXPECT_TRUE(logger.hasError());
+ s.fieldStart(true);
+ s.fieldEnd();
+
+ logger.reset();
+ s.command("arguments", {{"a", 5}});
+ EXPECT_TRUE(logger.hasError());
+ s.fieldStart(true);
+ s.fieldEnd();
+
+ logger.reset();
+ s.command("arguments", {{"a", 5}, {"b", "test"}});
+ EXPECT_FALSE(logger.hasError());
+ s.fieldStart(true);
+ s.fieldEnd();
+}
+
+TEST(Stack, invalidCommandName)
+{
+ Stack s{env.context, States::AnyHandlers};
+ tracker.reset();
+ logger.reset();
+
+ s.command("a", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("a_", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(2, 2, 2, 2, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("a_:b", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(3, 3, 3, 3, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ ASSERT_THROW(s.command("_a", {}), LoggableException);
+ ASSERT_THROW(s.command("a:", {}), LoggableException);
+ ASSERT_THROW(s.command("a:_b", {}), LoggableException);
+ tracker.expect(3, 3, 3, 3, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, multipleFields)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {{"a", false}});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("a", s.currentCommandName());
+ EXPECT_EQ(Variant::mapType({{"a", false}}), tracker.startArgs);
+
+ s.fieldStart(false);
+ tracker.expect(1, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_FALSE(tracker.fieldStartIsDefault);
+ EXPECT_EQ(0U, tracker.fieldStartIdx);
+
+ s.data("test");
+ tracker.expect(1, 0, 1, 0, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("test", tracker.dataData);
+
+ s.fieldEnd();
+ tracker.expect(1, 0, 1, 1, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ tracker.expect(1, 0, 2, 1, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_FALSE(tracker.fieldStartIsDefault);
+ EXPECT_EQ(1U, tracker.fieldStartIdx);
+
+ s.data("test2");
+ tracker.expect(1, 0, 2, 1, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("test2", tracker.dataData);
+
+ s.fieldEnd();
+ tracker.expect(1, 0, 2, 2, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(true);
+ tracker.expect(1, 0, 3, 2, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_TRUE(tracker.fieldStartIsDefault);
+ EXPECT_EQ(2U, tracker.fieldStartIdx);
+
+ s.data("test3");
+ tracker.expect(1, 0, 3, 2, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("test3", tracker.dataData);
+
+ s.fieldEnd();
+ tracker.expect(1, 1, 3, 3, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommand)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("b", {});
+ tracker.expect(2, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(2, 2, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommandWithExplicitDefaultField)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+
+ s.command("b", {});
+ tracker.expect(2, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(2, 1, 2, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+ }
+ tracker.expect(2, 2, 2, 2, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldOnIncompatibleCommand)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+
+ tracker.fieldStartResult = false;
+ s.command("b", {});
+ tracker.expect(2, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ }
+ tracker.expect(2, 2, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldIfDefaultFieldGiven)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+ s.fieldStart(true);
+ tracker.expect(1, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("", s.currentCommandName());
+
+ s.command("b", {});
+ tracker.expect(2, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ }
+ tracker.expect(2, 2, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noEndIfStartFails)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+
+ tracker.startResult = false;
+ s.command("b", {});
+ tracker.expect(3, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ }
+ tracker.expect(3, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_TRUE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnData)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.data("test");
+ tracker.expect(1, 0, 1, 0, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoFieldEnd)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoImplicitFieldEnd)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ s.command("b", {});
+ s.command("c", {});
+ s.command("d", {});
+ s.command("e", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(5, 1, 5, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(5, 5, 5, 5, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, invalidDefaultField)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.fieldStartResult = false;
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidDefaultFieldData)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.fieldStartResult = false;
+ s.fieldStart(true);
+ ASSERT_FALSE(logger.hasError());
+ s.data("test");
+ ASSERT_TRUE(logger.hasError());
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorInvalidFieldData)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.fieldStartResult = false;
+ ASSERT_FALSE(logger.hasError());
+ s.fieldStart(false);
+ ASSERT_TRUE(logger.hasError());
+ s.data("test");
+ s.fieldEnd();
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorFieldStartNoCommand)
+{
+ tracker.reset();
+ logger.reset();
+
+ Stack s{env.context, States::AnyHandlers};
+ ASSERT_THROW(s.fieldStart(false), LoggableException);
+ ASSERT_THROW(s.fieldStart(true), LoggableException);
+ tracker.expect(0, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldStarts)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ ASSERT_FALSE(logger.hasError());
+ s.fieldStart(false);
+ ASSERT_TRUE(logger.hasError());
+ tracker.expect(1, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldEnd();
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldEnds)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ s.fieldEnd();
+ ASSERT_FALSE(logger.hasError());
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ s.fieldEnd();
+ ASSERT_TRUE(logger.hasError());
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorOpenField)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ ASSERT_FALSE(logger.hasError());
+ }
+ ASSERT_TRUE(logger.hasError());
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+}
+}
+
diff --git a/test/core/parser/stack/StateTest.cpp b/test/core/parser/stack/StateTest.cpp
new file mode 100644
index 0000000..e503d30
--- /dev/null
+++ b/test/core/parser/stack/StateTest.cpp
@@ -0,0 +1,79 @@
+/*
+ Ousía
+ Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/Rtti.hpp>
+#include <core/parser/stack/State.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+static const Rtti t1;
+static const Rtti t2;
+static const Rtti t3;
+static const Rtti t4;
+static const Rtti t5;
+
+static const State s1 = StateBuilder().createdNodeType(&t1);
+static const State s2a =
+ StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s2b =
+ StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s3 =
+ StateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
+static const State s4 =
+ StateBuilder().parent(&s3).createdNodeType(&t4);
+static const State s5 =
+ StateBuilder().parent(&s2b).createdNodeType(&t5);
+
+TEST(StateDeductor, deduce)
+{
+ using Result = std::vector<const State *>;
+ using Signature = std::vector<const Rtti *>;
+ std::vector<const State *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
+
+ // Should not crash on empty signature
+ ASSERT_EQ(Result{}, StateDeductor(Signature{}, states).deduce());
+
+ // Try repeating signature elements
+ ASSERT_EQ(Result({&s1}),
+ StateDeductor(Signature({&t1}), states).deduce());
+ ASSERT_EQ(Result({&s1}),
+ StateDeductor(Signature({&t1, &t1}), states).deduce());
+ ASSERT_EQ(Result({&s1}),
+ StateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
+
+ // Go to another state
+ ASSERT_EQ(Result({&s2a, &s2b}),
+ StateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
+ ASSERT_EQ(Result({&s4}),
+ StateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
+
+ // Skip one state
+ ASSERT_EQ(Result({&s4}),
+ StateDeductor(Signature({&t2, &t4}), states).deduce());
+
+ // Impossible signature
+ ASSERT_EQ(Result({}),
+ StateDeductor(Signature({&t4, &t5}), states).deduce());
+
+}
+}
+}
+
diff --git a/test/core/parser/utils/TokenTrieTest.cpp b/test/core/parser/utils/TokenTrieTest.cpp
new file mode 100644
index 0000000..087e6e6
--- /dev/null
+++ b/test/core/parser/utils/TokenTrieTest.cpp
@@ -0,0 +1,92 @@
+/*
+ Ousía
+ Copyright (C) 2014 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/parser/utils/TokenTrie.hpp>
+
+namespace ousia {
+
+static const TokenTypeId t1 = 0;
+static const TokenTypeId t2 = 1;
+static const TokenTypeId t3 = 2;
+static const TokenTypeId t4 = 3;
+
+TEST(TokenTrie, registerToken)
+{
+ TokenTrie tree;
+
+ ASSERT_TRUE(tree.registerToken("a", t1));
+ ASSERT_TRUE(tree.registerToken("ab", t2));
+ ASSERT_TRUE(tree.registerToken("b", t3));
+ ASSERT_TRUE(tree.registerToken("hello", t4));
+
+ ASSERT_FALSE(tree.registerToken("", t1));
+ ASSERT_FALSE(tree.registerToken("a", t4));
+ ASSERT_FALSE(tree.registerToken("ab", t4));
+ ASSERT_FALSE(tree.registerToken("b", t4));
+ ASSERT_FALSE(tree.registerToken("hello", t4));
+
+ ASSERT_EQ(t1, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(t3, tree.hasToken("b"));
+ ASSERT_EQ(t4, tree.hasToken("hello"));
+ ASSERT_EQ(EmptyToken, tree.hasToken(""));
+ ASSERT_EQ(EmptyToken, tree.hasToken("abc"));
+}
+
+TEST(TokenTrie, unregisterToken)
+{
+ TokenTrie tree;
+
+ ASSERT_TRUE(tree.registerToken("a", t1));
+ ASSERT_FALSE(tree.registerToken("a", t4));
+
+ ASSERT_TRUE(tree.registerToken("ab", t2));
+ ASSERT_FALSE(tree.registerToken("ab", t4));
+
+ ASSERT_TRUE(tree.registerToken("b", t3));
+ ASSERT_FALSE(tree.registerToken("b", t4));
+
+ ASSERT_EQ(t1, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(t3, tree.hasToken("b"));
+
+ ASSERT_TRUE(tree.unregisterToken("a"));
+ ASSERT_FALSE(tree.unregisterToken("a"));
+
+ ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(t3, tree.hasToken("b"));
+
+ ASSERT_TRUE(tree.unregisterToken("b"));
+ ASSERT_FALSE(tree.unregisterToken("b"));
+
+ ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+
+ ASSERT_TRUE(tree.unregisterToken("ab"));
+ ASSERT_FALSE(tree.unregisterToken("ab"));
+
+ ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+ ASSERT_EQ(EmptyToken, tree.hasToken("ab"));
+ ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+}
+}
+
diff --git a/test/core/parser/utils/TokenizerTest.cpp b/test/core/parser/utils/TokenizerTest.cpp
new file mode 100644
index 0000000..8565057
--- /dev/null
+++ b/test/core/parser/utils/TokenizerTest.cpp
@@ -0,0 +1,414 @@
+/*
+ Ousía
+ Copyright (C) 2014 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/CharReader.hpp>
+#include <core/parser/utils/Tokenizer.hpp>
+
+namespace ousia {
+
+TEST(Tokenizer, tokenRegistration)
+{
+ Tokenizer tokenizer;
+
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken(""));
+
+ ASSERT_EQ(0U, tokenizer.registerToken("a"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("a"));
+ ASSERT_EQ("a", tokenizer.getTokenString(0U));
+
+ ASSERT_EQ(1U, tokenizer.registerToken("b"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("b"));
+ ASSERT_EQ("b", tokenizer.getTokenString(1U));
+
+ ASSERT_EQ(2U, tokenizer.registerToken("c"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("c"));
+ ASSERT_EQ("c", tokenizer.getTokenString(2U));
+
+ ASSERT_TRUE(tokenizer.unregisterToken(1U));
+ ASSERT_FALSE(tokenizer.unregisterToken(1U));
+ ASSERT_EQ("", tokenizer.getTokenString(1U));
+
+ ASSERT_EQ(1U, tokenizer.registerToken("d"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("d"));
+ ASSERT_EQ("d", tokenizer.getTokenString(1U));
+}
+
+TEST(Tokenizer, textTokenPreserveWhitespace)
+{
+ {
+ CharReader reader{" this \t is only a \n\n test text "};
+ // 012345 6789012345678 9 0123456789012345
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ(" this \t is only a \n\n test text ", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(36U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+
+ {
+ CharReader reader{"this \t is only a \n\n test text"};
+ // 01234 5678901234567 8 9012345678901
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this \t is only a \n\n test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(32U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+}
+
+TEST(Tokenizer, textTokenTrimWhitespace)
+{
+ {
+ CharReader reader{" this \t is only a \n\n test text "};
+ // 012345 6789012345678 9 0123456789012345
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this \t is only a \n\n test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(1U, loc.getStart());
+ ASSERT_EQ(33U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+
+ {
+ CharReader reader{"this \t is only a \n\n test text"};
+ // 01234 5678901234567 8 9012345678901
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this \t is only a \n\n test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(32U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+}
+
+TEST(Tokenizer, textTokenCollapseWhitespace)
+{
+ {
+ CharReader reader{" this \t is only a \n\n test text "};
+ // 012345 6789012345678 9 0123456789012345
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this is only a test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(1U, loc.getStart());
+ ASSERT_EQ(33U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+
+ {
+ CharReader reader{"this \t is only a \n\n test text"};
+ // 01234 5678901234567 8 9012345678901
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this is only a test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(32U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+}
+
+TEST(Tokenizer, simpleReadToken)
+{
+ CharReader reader{"test1:test2"};
+ Tokenizer tokenizer;
+
+ const TokenTypeId tid = tokenizer.registerToken(":");
+ ASSERT_EQ(0U, tid);
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test1", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(5U, loc.getEnd());
+
+ char c;
+ ASSERT_TRUE(reader.peek(c));
+ ASSERT_EQ(':', c);
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(tid, token.type);
+ ASSERT_EQ(":", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(5U, loc.getStart());
+ ASSERT_EQ(6U, loc.getEnd());
+
+ char c;
+ ASSERT_TRUE(reader.peek(c));
+ ASSERT_EQ('t', c);
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test2", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(6U, loc.getStart());
+ ASSERT_EQ(11U, loc.getEnd());
+
+ char c;
+ ASSERT_FALSE(reader.peek(c));
+ }
+}
+
+TEST(Tokenizer, simplePeekToken)
+{
+ CharReader reader{"test1:test2"};
+ Tokenizer tokenizer;
+
+ const TokenTypeId tid = tokenizer.registerToken(":");
+ ASSERT_EQ(0U, tid);
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.peek(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test1", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(5U, loc.getEnd());
+ ASSERT_EQ(0U, reader.getOffset());
+ ASSERT_EQ(5U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.peek(reader, token));
+
+ ASSERT_EQ(tid, token.type);
+ ASSERT_EQ(":", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(5U, loc.getStart());
+ ASSERT_EQ(6U, loc.getEnd());
+ ASSERT_EQ(0U, reader.getOffset());
+ ASSERT_EQ(6U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.peek(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test2", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(6U, loc.getStart());
+ ASSERT_EQ(11U, loc.getEnd());
+ ASSERT_EQ(0U, reader.getOffset());
+ ASSERT_EQ(11U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test1", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(5U, loc.getEnd());
+ ASSERT_EQ(5U, reader.getOffset());
+ ASSERT_EQ(5U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(tid, token.type);
+ ASSERT_EQ(":", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(5U, loc.getStart());
+ ASSERT_EQ(6U, loc.getEnd());
+ ASSERT_EQ(6U, reader.getOffset());
+ ASSERT_EQ(6U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test2", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(6U, loc.getStart());
+ ASSERT_EQ(11U, loc.getEnd());
+ ASSERT_EQ(11U, reader.getOffset());
+ ASSERT_EQ(11U, reader.getPeekOffset());
+ }
+}
+
+TEST(Tokenizer, ambiguousTokens)
+{
+ CharReader reader{"abc"};
+ Tokenizer tokenizer;
+
+ TokenTypeId t1 = tokenizer.registerToken("abd");
+ TokenTypeId t2 = tokenizer.registerToken("bc");
+
+ ASSERT_EQ(0U, t1);
+ ASSERT_EQ(1U, t2);
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("a", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(1U, loc.getEnd());
+
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(t2, token.type);
+ ASSERT_EQ("bc", token.content);
+
+ loc = token.location;
+ ASSERT_EQ(1U, loc.getStart());
+ ASSERT_EQ(3U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+}
+
+TEST(Tokenizer, commentTestWhitespacePreserve)
+{
+ CharReader reader{"Test/Test /* Block Comment */", 0};
+ // 012345678901234567890123456789
+ // 0 1 2
+ Tokenizer tokenizer(WhitespaceMode::PRESERVE);
+
+ const TokenTypeId t1 = tokenizer.registerToken("/");
+ const TokenTypeId t2 = tokenizer.registerToken("/*");
+ const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+ std::vector<Token> expected = {
+ {TextToken, "Test", SourceLocation{0, 0, 4}},
+ {t1, "/", SourceLocation{0, 4, 5}},
+ {TextToken, "Test ", SourceLocation{0, 5, 10}},
+ {t2, "/*", SourceLocation{0, 10, 12}},
+ {TextToken, " Block Comment ", SourceLocation{0, 12, 27}},
+ {t3, "*/", SourceLocation{0, 27, 29}}};
+
+ Token t;
+ for (auto &te : expected) {
+ EXPECT_TRUE(tokenizer.read(reader, t));
+ EXPECT_EQ(te.type, t.type);
+ EXPECT_EQ(te.content, t.content);
+ EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+ EXPECT_EQ(te.location.getStart(), t.location.getStart());
+ EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+ }
+ ASSERT_FALSE(tokenizer.read(reader, t));
+}
+
+TEST(Tokenizer, commentTestWhitespaceCollapse)
+{
+ CharReader reader{"Test/Test /* Block Comment */", 0};
+ // 012345678901234567890123456789
+ // 0 1 2
+ Tokenizer tokenizer(WhitespaceMode::COLLAPSE);
+
+ const TokenTypeId t1 = tokenizer.registerToken("/");
+ const TokenTypeId t2 = tokenizer.registerToken("/*");
+ const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+ std::vector<Token> expected = {
+ {TextToken, "Test", SourceLocation{0, 0, 4}},
+ {t1, "/", SourceLocation{0, 4, 5}},
+ {TextToken, "Test", SourceLocation{0, 5, 9}},
+ {t2, "/*", SourceLocation{0, 10, 12}},
+ {TextToken, "Block Comment", SourceLocation{0, 13, 26}},
+ {t3, "*/", SourceLocation{0, 27, 29}}};
+
+ Token t;
+ for (auto &te : expected) {
+ EXPECT_TRUE(tokenizer.read(reader, t));
+ EXPECT_EQ(te.type, t.type);
+ EXPECT_EQ(te.content, t.content);
+ EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+ EXPECT_EQ(te.location.getStart(), t.location.getStart());
+ EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+ }
+ ASSERT_FALSE(tokenizer.read(reader, t));
+}
+}
+
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-01 09:12:24 +0000