summaryrefslogtreecommitdiff
path: root/test/core/parser
diff options
context:
space:
mode:
Diffstat (limited to 'test/core/parser')
-rw-r--r--test/core/parser/ParserStackTest.cpp177
-rw-r--r--test/core/parser/ParserStateTest.cpp77
-rw-r--r--test/core/parser/stack/StackTest.cpp666
-rw-r--r--test/core/parser/stack/StateTest.cpp79
-rw-r--r--test/core/parser/utils/TokenTrieTest.cpp92
-rw-r--r--test/core/parser/utils/TokenizerTest.cpp414
6 files changed, 1251 insertions, 254 deletions
diff --git a/test/core/parser/ParserStackTest.cpp b/test/core/parser/ParserStackTest.cpp
deleted file mode 100644
index 3a0decb..0000000
--- a/test/core/parser/ParserStackTest.cpp
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- Ousía
- Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <iostream>
-
-#include <gtest/gtest.h>
-
-#include <core/parser/ParserStack.hpp>
-#include <core/StandaloneEnvironment.hpp>
-
-namespace ousia {
-
-ConcreteLogger logger;
-
-static int startCount = 0;
-static int endCount = 0;
-static int dataCount = 0;
-
-class TestHandler : public Handler {
-public:
- using Handler::Handler;
-
- void start(Variant::mapType &args) override { startCount++; }
-
- void end() override { endCount++; }
-
- void data(const std::string &data, int field) override { dataCount++; }
-
- static Handler *create(const HandlerData &data)
- {
- return new TestHandler(data);
- }
-};
-
-namespace ParserStates {
-static const ParserState Document =
- ParserStateBuilder().parent(&None).elementHandler(TestHandler::create);
-static const ParserState Body = ParserStateBuilder()
- .parent(&Document)
- .elementHandler(TestHandler::create);
-static const ParserState Empty =
- ParserStateBuilder().parent(&Document).elementHandler(TestHandler::create);
-static const ParserState Special =
- ParserStateBuilder().parent(&All).elementHandler(TestHandler::create);
-static const ParserState Arguments =
- ParserStateBuilder()
- .parent(&None)
- .elementHandler(TestHandler::create)
- .arguments({Argument::Int("a"), Argument::String("b")});
-static const ParserState BodyChildren =
- ParserStateBuilder()
- .parent(&Body)
- .elementHandler(TestHandler::create);
-
-static const std::multimap<std::string, const ParserState *> TestHandlers{
- {"document", &Document},
- {"body", &Body},
- {"empty", &Empty},
- {"special", &Special},
- {"arguments", &Arguments},
- {"*", &BodyChildren}};
-}
-
-TEST(ParserStack, simpleTest)
-{
- StandaloneEnvironment env(logger);
- ParserStack s{env.context, ParserStates::TestHandlers};
-
- startCount = 0;
- endCount = 0;
- dataCount = 0;
-
- EXPECT_EQ("", s.currentCommandName());
- EXPECT_EQ(&ParserStates::None, &s.currentState());
-
- s.start("document", {});
- s.data("test1");
-
- EXPECT_EQ("document", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Document, &s.currentState());
- EXPECT_EQ(1, startCount);
- EXPECT_EQ(1, dataCount);
-
- s.start("body", {});
- s.data("test2");
- EXPECT_EQ("body", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Body, &s.currentState());
- EXPECT_EQ(2, startCount);
- EXPECT_EQ(2, dataCount);
-
- s.start("inner", {});
- EXPECT_EQ("inner", s.currentCommandName());
- EXPECT_EQ(&ParserStates::BodyChildren, &s.currentState());
- s.end();
- EXPECT_EQ(3, startCount);
- EXPECT_EQ(1, endCount);
-
- s.end();
- EXPECT_EQ(2, endCount);
-
- EXPECT_EQ("document", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
- s.start("body", {});
- s.data("test3");
- EXPECT_EQ("body", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Body, &s.currentState());
- s.end();
- EXPECT_EQ(4, startCount);
- EXPECT_EQ(3, dataCount);
- EXPECT_EQ(3, endCount);
-
- EXPECT_EQ("document", s.currentCommandName());
- EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
- s.end();
- EXPECT_EQ(4, endCount);
-
- EXPECT_EQ("", s.currentCommandName());
- EXPECT_EQ(&ParserStates::None, &s.currentState());
-}
-
-TEST(ParserStack, errorHandling)
-{
- StandaloneEnvironment env(logger);
- ParserStack s{env.context, ParserStates::TestHandlers};
-
- EXPECT_THROW(s.start("body", {}), OusiaException);
- s.start("document", {});
- EXPECT_THROW(s.start("document", {}), OusiaException);
- s.start("empty", {});
- EXPECT_THROW(s.start("body", {}), OusiaException);
- s.start("special", {});
- s.end();
- s.end();
- s.end();
- EXPECT_EQ(&ParserStates::None, &s.currentState());
- ASSERT_THROW(s.end(), OusiaException);
- ASSERT_THROW(s.data("test", 1), OusiaException);
-}
-
-TEST(ParserStack, validation)
-{
- StandaloneEnvironment env(logger);
- ParserStack s{env.context, ParserStates::TestHandlers};
-
- logger.reset();
- s.start("arguments", {});
- EXPECT_TRUE(logger.hasError());
- s.end();
-
- s.start("arguments", {{"a", 5}});
- EXPECT_TRUE(logger.hasError());
- s.end();
-
- logger.reset();
- s.start("arguments", {{"a", 5}, {"b", "test"}});
- EXPECT_FALSE(logger.hasError());
- s.end();
-}
-}
-
diff --git a/test/core/parser/ParserStateTest.cpp b/test/core/parser/ParserStateTest.cpp
deleted file mode 100644
index 91d8dcd..0000000
--- a/test/core/parser/ParserStateTest.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- Ousía
- Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/common/Rtti.hpp>
-#include <core/parser/ParserState.hpp>
-
-namespace ousia {
-
-static const Rtti t1;
-static const Rtti t2;
-static const Rtti t3;
-static const Rtti t4;
-static const Rtti t5;
-
-static const ParserState s1 = ParserStateBuilder().createdNodeType(&t1);
-static const ParserState s2a =
- ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s2b =
- ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s3 =
- ParserStateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
-static const ParserState s4 =
- ParserStateBuilder().parent(&s3).createdNodeType(&t4);
-static const ParserState s5 =
- ParserStateBuilder().parent(&s2b).createdNodeType(&t5);
-
-TEST(ParserStateDeductor, deduce)
-{
- using Result = std::vector<const ParserState *>;
- using Signature = std::vector<const Rtti *>;
- std::vector<const ParserState *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
-
- // Should not crash on empty signature
- ASSERT_EQ(Result{}, ParserStateDeductor(Signature{}, states).deduce());
-
- // Try repeating signature elements
- ASSERT_EQ(Result({&s1}),
- ParserStateDeductor(Signature({&t1}), states).deduce());
- ASSERT_EQ(Result({&s1}),
- ParserStateDeductor(Signature({&t1, &t1}), states).deduce());
- ASSERT_EQ(Result({&s1}),
- ParserStateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
-
- // Go to another state
- ASSERT_EQ(Result({&s2a, &s2b}),
- ParserStateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
- ASSERT_EQ(Result({&s4}),
- ParserStateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
-
- // Skip one state
- ASSERT_EQ(Result({&s4}),
- ParserStateDeductor(Signature({&t2, &t4}), states).deduce());
-
- // Impossible signature
- ASSERT_EQ(Result({}),
- ParserStateDeductor(Signature({&t4, &t5}), states).deduce());
-
-}
-}
-
diff --git a/test/core/parser/stack/StackTest.cpp b/test/core/parser/stack/StackTest.cpp
new file mode 100644
index 0000000..321d471
--- /dev/null
+++ b/test/core/parser/stack/StackTest.cpp
@@ -0,0 +1,666 @@
+/*
+ Ousía
+ Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <iostream>
+
+#include <gtest/gtest.h>
+
+#include <core/frontend/TerminalLogger.hpp>
+#include <core/parser/stack/Handler.hpp>
+#include <core/parser/stack/Stack.hpp>
+#include <core/parser/stack/State.hpp>
+
+#include <core/StandaloneEnvironment.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+// Build an instance of the StandaloneEnvironment used for this unit test
+static TerminalLogger logger(std::cerr, true);
+// static ConcreteLogger logger;
+static StandaloneEnvironment env(logger);
+
+namespace {
+
+struct Tracker {
+ int startCount;
+ int endCount;
+ int fieldStartCount;
+ int fieldEndCount;
+ int annotationStartCount;
+ int annotationEndCount;
+ int dataCount;
+
+ Variant::mapType startArgs;
+ bool fieldStartIsDefault;
+ size_t fieldStartIdx;
+ Variant annotationStartClassName;
+ Variant::mapType annotationStartArgs;
+ Variant annotationEndClassName;
+ Variant annotationEndElementName;
+ Variant dataData;
+
+ bool startResult;
+ bool fieldStartSetIsDefault;
+ bool fieldStartResult;
+ bool annotationStartResult;
+ bool annotationEndResult;
+ bool dataResult;
+
+ Tracker() { reset(); }
+
+ void reset()
+ {
+ startCount = 0;
+ endCount = 0;
+ fieldStartCount = 0;
+ fieldEndCount = 0;
+ annotationStartCount = 0;
+ annotationEndCount = 0;
+ dataCount = 0;
+
+ startArgs = Variant::mapType{};
+ fieldStartIsDefault = false;
+ fieldStartIdx = 0;
+ annotationStartClassName = Variant::fromString(std::string{});
+ annotationStartArgs = Variant::mapType{};
+ annotationEndClassName = Variant::fromString(std::string{});
+ annotationEndElementName = Variant::fromString(std::string{});
+ dataData = Variant::fromString(std::string{});
+
+ startResult = true;
+ fieldStartSetIsDefault = false;
+ fieldStartResult = true;
+ annotationStartResult = true;
+ annotationEndResult = true;
+ dataResult = true;
+ }
+
+ void expect(int startCount, int endCount, int fieldStartCount,
+ int fieldEndCount, int annotationStartCount,
+ int annotationEndCount, int dataCount)
+ {
+ EXPECT_EQ(startCount, this->startCount);
+ EXPECT_EQ(endCount, this->endCount);
+ EXPECT_EQ(fieldStartCount, this->fieldStartCount);
+ EXPECT_EQ(fieldEndCount, this->fieldEndCount);
+ EXPECT_EQ(annotationStartCount, this->annotationStartCount);
+ EXPECT_EQ(annotationEndCount, this->annotationEndCount);
+ EXPECT_EQ(dataCount, this->dataCount);
+ }
+};
+
+static Tracker tracker;
+
+class TestHandler : public Handler {
+private:
+ TestHandler(const HandlerData &handlerData) : Handler(handlerData) {}
+
+public:
+ bool start(Variant::mapType &args) override
+ {
+ tracker.startCount++;
+ tracker.startArgs = args;
+ if (!tracker.startResult) {
+ logger().error(
+ "The TestHandler was told not to allow a field start. So it "
+ "doesn't. The TestHandler always obeys its master.");
+ }
+ return tracker.startResult;
+ }
+
+ void end() override { tracker.endCount++; }
+
+ bool fieldStart(bool &isDefault, size_t fieldIdx) override
+ {
+ tracker.fieldStartCount++;
+ tracker.fieldStartIsDefault = isDefault;
+ tracker.fieldStartIdx = fieldIdx;
+ if (tracker.fieldStartSetIsDefault) {
+ isDefault = true;
+ }
+ return tracker.fieldStartResult;
+ }
+
+ void fieldEnd() override { tracker.fieldEndCount++; }
+
+ bool annotationStart(const Variant &className,
+ Variant::mapType &args) override
+ {
+ tracker.annotationStartCount++;
+ tracker.annotationStartClassName = className;
+ tracker.annotationStartArgs = args;
+ return tracker.annotationStartResult;
+ }
+
+ bool annotationEnd(const Variant &className,
+ const Variant &elementName) override
+ {
+ tracker.annotationEndCount++;
+ tracker.annotationEndClassName = className;
+ tracker.annotationEndElementName = elementName;
+ return tracker.annotationEndResult;
+ }
+
+ bool data(Variant &data) override
+ {
+ tracker.dataCount++;
+ tracker.dataData = data;
+ return tracker.dataResult;
+ }
+
+ static Handler *create(const HandlerData &handlerData)
+ {
+ return new TestHandler(handlerData);
+ }
+};
+}
+
+namespace States {
+static const State Document =
+ StateBuilder().parent(&None).elementHandler(TestHandler::create);
+static const State Body =
+ StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Empty =
+ StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Special =
+ StateBuilder().parent(&All).elementHandler(TestHandler::create);
+static const State Arguments =
+ StateBuilder().parent(&None).elementHandler(TestHandler::create).arguments(
+ {Argument::Int("a"), Argument::String("b")});
+static const State BodyChildren =
+ StateBuilder().parent(&Body).elementHandler(TestHandler::create);
+static const State Any =
+ StateBuilder().parents({&None, &Any}).elementHandler(TestHandler::create);
+
+static const std::multimap<std::string, const State *> TestHandlers{
+ {"document", &Document},
+ {"body", &Body},
+ {"empty", &Empty},
+ {"special", &Special},
+ {"arguments", &Arguments},
+ {"*", &BodyChildren}};
+
+static const std::multimap<std::string, const State *> AnyHandlers{{"*", &Any}};
+}
+
+TEST(Stack, basicTest)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::TestHandlers};
+
+ EXPECT_EQ("", s.currentCommandName());
+ EXPECT_EQ(&States::None, &s.currentState());
+
+ s.command("document", {});
+ s.fieldStart(true);
+ s.data("test1");
+
+ EXPECT_EQ("document", s.currentCommandName());
+ EXPECT_EQ(&States::Document, &s.currentState());
+ tracker.expect(1, 0, 1, 0, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("body", {});
+ s.fieldStart(true);
+ s.data("test2");
+ EXPECT_EQ("body", s.currentCommandName());
+ EXPECT_EQ(&States::Body, &s.currentState());
+ tracker.expect(2, 0, 2, 0, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("inner", {});
+ s.fieldStart(true);
+ EXPECT_EQ("inner", s.currentCommandName());
+ EXPECT_EQ(&States::BodyChildren, &s.currentState());
+
+ s.fieldEnd();
+ tracker.expect(3, 1, 3, 1, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldEnd();
+ EXPECT_EQ("document", s.currentCommandName());
+ EXPECT_EQ(&States::Document, &s.currentState());
+ tracker.expect(3, 2, 3, 2, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("body", {});
+ s.fieldStart(true);
+ s.data("test3");
+ EXPECT_EQ("body", s.currentCommandName());
+ EXPECT_EQ(&States::Body, &s.currentState());
+ s.fieldEnd();
+ tracker.expect(4, 3, 4, 3, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+
+ EXPECT_EQ("document", s.currentCommandName());
+ EXPECT_EQ(&States::Document, &s.currentState());
+
+ s.fieldEnd();
+ tracker.expect(4, 4, 4, 4, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+
+ EXPECT_EQ("", s.currentCommandName());
+ EXPECT_EQ(&States::None, &s.currentState());
+ }
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidCommands)
+{
+ Stack s{env.context, States::TestHandlers};
+ tracker.reset();
+ EXPECT_THROW(s.command("body", {}), LoggableException);
+ s.command("document", {});
+ s.fieldStart(true);
+ EXPECT_THROW(s.command("document", {}), LoggableException);
+ s.command("empty", {});
+ s.fieldStart(true);
+ EXPECT_THROW(s.command("body", {}), LoggableException);
+ s.command("special", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ s.fieldEnd();
+ s.fieldEnd();
+ EXPECT_EQ(&States::None, &s.currentState());
+ ASSERT_THROW(s.fieldEnd(), LoggableException);
+ ASSERT_THROW(s.data("test"), LoggableException);
+}
+
+TEST(Stack, validation)
+{
+ Stack s{env.context, States::TestHandlers};
+ tracker.reset();
+ logger.reset();
+
+ s.command("arguments", {});
+ EXPECT_TRUE(logger.hasError());
+ s.fieldStart(true);
+ s.fieldEnd();
+
+ logger.reset();
+ s.command("arguments", {{"a", 5}});
+ EXPECT_TRUE(logger.hasError());
+ s.fieldStart(true);
+ s.fieldEnd();
+
+ logger.reset();
+ s.command("arguments", {{"a", 5}, {"b", "test"}});
+ EXPECT_FALSE(logger.hasError());
+ s.fieldStart(true);
+ s.fieldEnd();
+}
+
+TEST(Stack, invalidCommandName)
+{
+ Stack s{env.context, States::AnyHandlers};
+ tracker.reset();
+ logger.reset();
+
+ s.command("a", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("a_", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(2, 2, 2, 2, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("a_:b", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(3, 3, 3, 3, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ ASSERT_THROW(s.command("_a", {}), LoggableException);
+ ASSERT_THROW(s.command("a:", {}), LoggableException);
+ ASSERT_THROW(s.command("a:_b", {}), LoggableException);
+ tracker.expect(3, 3, 3, 3, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, multipleFields)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {{"a", false}});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("a", s.currentCommandName());
+ EXPECT_EQ(Variant::mapType({{"a", false}}), tracker.startArgs);
+
+ s.fieldStart(false);
+ tracker.expect(1, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_FALSE(tracker.fieldStartIsDefault);
+ EXPECT_EQ(0U, tracker.fieldStartIdx);
+
+ s.data("test");
+ tracker.expect(1, 0, 1, 0, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("test", tracker.dataData);
+
+ s.fieldEnd();
+ tracker.expect(1, 0, 1, 1, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ tracker.expect(1, 0, 2, 1, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_FALSE(tracker.fieldStartIsDefault);
+ EXPECT_EQ(1U, tracker.fieldStartIdx);
+
+ s.data("test2");
+ tracker.expect(1, 0, 2, 1, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("test2", tracker.dataData);
+
+ s.fieldEnd();
+ tracker.expect(1, 0, 2, 2, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(true);
+ tracker.expect(1, 0, 3, 2, 0, 0, 2); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_TRUE(tracker.fieldStartIsDefault);
+ EXPECT_EQ(2U, tracker.fieldStartIdx);
+
+ s.data("test3");
+ tracker.expect(1, 0, 3, 2, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+ EXPECT_EQ("test3", tracker.dataData);
+
+ s.fieldEnd();
+ tracker.expect(1, 1, 3, 3, 0, 0, 3); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommand)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.command("b", {});
+ tracker.expect(2, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(2, 2, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommandWithExplicitDefaultField)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+
+ s.command("b", {});
+ tracker.expect(2, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(2, 1, 2, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+ }
+ tracker.expect(2, 2, 2, 2, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldOnIncompatibleCommand)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+
+ tracker.fieldStartResult = false;
+ s.command("b", {});
+ tracker.expect(2, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ }
+ tracker.expect(2, 2, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldIfDefaultFieldGiven)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+ s.fieldStart(true);
+ tracker.expect(1, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("", s.currentCommandName());
+
+ s.command("b", {});
+ tracker.expect(2, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ }
+ tracker.expect(2, 2, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noEndIfStartFails)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("a", s.currentCommandName());
+
+ tracker.startResult = false;
+ s.command("b", {});
+ tracker.expect(3, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_EQ("b", s.currentCommandName());
+ }
+ tracker.expect(3, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_TRUE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnData)
+{
+ tracker.reset();
+ logger.reset();
+ {
+ Stack s{env.context, States::AnyHandlers};
+
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.data("test");
+ tracker.expect(1, 0, 1, 0, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 1); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoFieldEnd)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoImplicitFieldEnd)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ s.command("b", {});
+ s.command("c", {});
+ s.command("d", {});
+ s.command("e", {});
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(5, 1, 5, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(5, 5, 5, 5, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, invalidDefaultField)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.fieldStartResult = false;
+ s.fieldStart(true);
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidDefaultFieldData)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.fieldStartResult = false;
+ s.fieldStart(true);
+ ASSERT_FALSE(logger.hasError());
+ s.data("test");
+ ASSERT_TRUE(logger.hasError());
+ s.fieldEnd();
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorInvalidFieldData)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.fieldStartResult = false;
+ ASSERT_FALSE(logger.hasError());
+ s.fieldStart(false);
+ ASSERT_TRUE(logger.hasError());
+ s.data("test");
+ s.fieldEnd();
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorFieldStartNoCommand)
+{
+ tracker.reset();
+ logger.reset();
+
+ Stack s{env.context, States::AnyHandlers};
+ ASSERT_THROW(s.fieldStart(false), LoggableException);
+ ASSERT_THROW(s.fieldStart(true), LoggableException);
+ tracker.expect(0, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldStarts)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ ASSERT_FALSE(logger.hasError());
+ s.fieldStart(false);
+ ASSERT_TRUE(logger.hasError());
+ tracker.expect(1, 0, 1, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldEnd();
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldEnds)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ s.fieldEnd();
+ ASSERT_FALSE(logger.hasError());
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ s.fieldEnd();
+ ASSERT_TRUE(logger.hasError());
+ tracker.expect(1, 0, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+ }
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorOpenField)
+{
+ tracker.reset();
+ logger.reset();
+
+ {
+ Stack s{env.context, States::AnyHandlers};
+ s.command("a", {});
+ tracker.expect(1, 0, 0, 0, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+
+ s.fieldStart(false);
+ ASSERT_FALSE(logger.hasError());
+ }
+ ASSERT_TRUE(logger.hasError());
+ tracker.expect(1, 1, 1, 1, 0, 0, 0); // sc, ec, fsc, fse, asc, aec, dc
+}
+}
+}
+
diff --git a/test/core/parser/stack/StateTest.cpp b/test/core/parser/stack/StateTest.cpp
new file mode 100644
index 0000000..e503d30
--- /dev/null
+++ b/test/core/parser/stack/StateTest.cpp
@@ -0,0 +1,79 @@
+/*
+ Ousía
+ Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/Rtti.hpp>
+#include <core/parser/stack/State.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+static const Rtti t1;
+static const Rtti t2;
+static const Rtti t3;
+static const Rtti t4;
+static const Rtti t5;
+
+static const State s1 = StateBuilder().createdNodeType(&t1);
+static const State s2a =
+ StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s2b =
+ StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s3 =
+ StateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
+static const State s4 =
+ StateBuilder().parent(&s3).createdNodeType(&t4);
+static const State s5 =
+ StateBuilder().parent(&s2b).createdNodeType(&t5);
+
+TEST(StateDeductor, deduce)
+{
+ using Result = std::vector<const State *>;
+ using Signature = std::vector<const Rtti *>;
+ std::vector<const State *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
+
+ // Should not crash on empty signature
+ ASSERT_EQ(Result{}, StateDeductor(Signature{}, states).deduce());
+
+ // Try repeating signature elements
+ ASSERT_EQ(Result({&s1}),
+ StateDeductor(Signature({&t1}), states).deduce());
+ ASSERT_EQ(Result({&s1}),
+ StateDeductor(Signature({&t1, &t1}), states).deduce());
+ ASSERT_EQ(Result({&s1}),
+ StateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
+
+ // Go to another state
+ ASSERT_EQ(Result({&s2a, &s2b}),
+ StateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
+ ASSERT_EQ(Result({&s4}),
+ StateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
+
+ // Skip one state
+ ASSERT_EQ(Result({&s4}),
+ StateDeductor(Signature({&t2, &t4}), states).deduce());
+
+ // Impossible signature
+ ASSERT_EQ(Result({}),
+ StateDeductor(Signature({&t4, &t5}), states).deduce());
+
+}
+}
+}
+
diff --git a/test/core/parser/utils/TokenTrieTest.cpp b/test/core/parser/utils/TokenTrieTest.cpp
new file mode 100644
index 0000000..087e6e6
--- /dev/null
+++ b/test/core/parser/utils/TokenTrieTest.cpp
@@ -0,0 +1,92 @@
+/*
+ Ousía
+ Copyright (C) 2014 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/parser/utils/TokenTrie.hpp>
+
+namespace ousia {
+
+static const TokenTypeId t1 = 0;
+static const TokenTypeId t2 = 1;
+static const TokenTypeId t3 = 2;
+static const TokenTypeId t4 = 3;
+
+TEST(TokenTrie, registerToken)
+{
+ TokenTrie tree;
+
+ ASSERT_TRUE(tree.registerToken("a", t1));
+ ASSERT_TRUE(tree.registerToken("ab", t2));
+ ASSERT_TRUE(tree.registerToken("b", t3));
+ ASSERT_TRUE(tree.registerToken("hello", t4));
+
+ ASSERT_FALSE(tree.registerToken("", t1));
+ ASSERT_FALSE(tree.registerToken("a", t4));
+ ASSERT_FALSE(tree.registerToken("ab", t4));
+ ASSERT_FALSE(tree.registerToken("b", t4));
+ ASSERT_FALSE(tree.registerToken("hello", t4));
+
+ ASSERT_EQ(t1, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(t3, tree.hasToken("b"));
+ ASSERT_EQ(t4, tree.hasToken("hello"));
+ ASSERT_EQ(EmptyToken, tree.hasToken(""));
+ ASSERT_EQ(EmptyToken, tree.hasToken("abc"));
+}
+
+TEST(TokenTrie, unregisterToken)
+{
+ TokenTrie tree;
+
+ ASSERT_TRUE(tree.registerToken("a", t1));
+ ASSERT_FALSE(tree.registerToken("a", t4));
+
+ ASSERT_TRUE(tree.registerToken("ab", t2));
+ ASSERT_FALSE(tree.registerToken("ab", t4));
+
+ ASSERT_TRUE(tree.registerToken("b", t3));
+ ASSERT_FALSE(tree.registerToken("b", t4));
+
+ ASSERT_EQ(t1, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(t3, tree.hasToken("b"));
+
+ ASSERT_TRUE(tree.unregisterToken("a"));
+ ASSERT_FALSE(tree.unregisterToken("a"));
+
+ ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(t3, tree.hasToken("b"));
+
+ ASSERT_TRUE(tree.unregisterToken("b"));
+ ASSERT_FALSE(tree.unregisterToken("b"));
+
+ ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+ ASSERT_EQ(t2, tree.hasToken("ab"));
+ ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+
+ ASSERT_TRUE(tree.unregisterToken("ab"));
+ ASSERT_FALSE(tree.unregisterToken("ab"));
+
+ ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+ ASSERT_EQ(EmptyToken, tree.hasToken("ab"));
+ ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+}
+}
+
diff --git a/test/core/parser/utils/TokenizerTest.cpp b/test/core/parser/utils/TokenizerTest.cpp
new file mode 100644
index 0000000..8565057
--- /dev/null
+++ b/test/core/parser/utils/TokenizerTest.cpp
@@ -0,0 +1,414 @@
+/*
+ Ousía
+ Copyright (C) 2014 Benjamin Paaßen, Andreas Stöckel
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/CharReader.hpp>
+#include <core/parser/utils/Tokenizer.hpp>
+
+namespace ousia {
+
+TEST(Tokenizer, tokenRegistration)
+{
+ Tokenizer tokenizer;
+
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken(""));
+
+ ASSERT_EQ(0U, tokenizer.registerToken("a"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("a"));
+ ASSERT_EQ("a", tokenizer.getTokenString(0U));
+
+ ASSERT_EQ(1U, tokenizer.registerToken("b"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("b"));
+ ASSERT_EQ("b", tokenizer.getTokenString(1U));
+
+ ASSERT_EQ(2U, tokenizer.registerToken("c"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("c"));
+ ASSERT_EQ("c", tokenizer.getTokenString(2U));
+
+ ASSERT_TRUE(tokenizer.unregisterToken(1U));
+ ASSERT_FALSE(tokenizer.unregisterToken(1U));
+ ASSERT_EQ("", tokenizer.getTokenString(1U));
+
+ ASSERT_EQ(1U, tokenizer.registerToken("d"));
+ ASSERT_EQ(EmptyToken, tokenizer.registerToken("d"));
+ ASSERT_EQ("d", tokenizer.getTokenString(1U));
+}
+
+TEST(Tokenizer, textTokenPreserveWhitespace)
+{
+ {
+ CharReader reader{" this \t is only a \n\n test text "};
+ // 012345 6789012345678 9 0123456789012345
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ(" this \t is only a \n\n test text ", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(36U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+
+ {
+ CharReader reader{"this \t is only a \n\n test text"};
+ // 01234 5678901234567 8 9012345678901
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this \t is only a \n\n test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(32U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+}
+
+TEST(Tokenizer, textTokenTrimWhitespace)
+{
+ {
+ CharReader reader{" this \t is only a \n\n test text "};
+ // 012345 6789012345678 9 0123456789012345
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this \t is only a \n\n test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(1U, loc.getStart());
+ ASSERT_EQ(33U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+
+ {
+ CharReader reader{"this \t is only a \n\n test text"};
+ // 01234 5678901234567 8 9012345678901
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this \t is only a \n\n test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(32U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+}
+
+TEST(Tokenizer, textTokenCollapseWhitespace)
+{
+ {
+ CharReader reader{" this \t is only a \n\n test text "};
+ // 012345 6789012345678 9 0123456789012345
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this is only a test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(1U, loc.getStart());
+ ASSERT_EQ(33U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+
+ {
+ CharReader reader{"this \t is only a \n\n test text"};
+ // 01234 5678901234567 8 9012345678901
+ // 0 1 2 3
+ Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("this is only a test text", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(32U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+ }
+}
+
+TEST(Tokenizer, simpleReadToken)
+{
+ CharReader reader{"test1:test2"};
+ Tokenizer tokenizer;
+
+ const TokenTypeId tid = tokenizer.registerToken(":");
+ ASSERT_EQ(0U, tid);
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test1", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(5U, loc.getEnd());
+
+ char c;
+ ASSERT_TRUE(reader.peek(c));
+ ASSERT_EQ(':', c);
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(tid, token.type);
+ ASSERT_EQ(":", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(5U, loc.getStart());
+ ASSERT_EQ(6U, loc.getEnd());
+
+ char c;
+ ASSERT_TRUE(reader.peek(c));
+ ASSERT_EQ('t', c);
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test2", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(6U, loc.getStart());
+ ASSERT_EQ(11U, loc.getEnd());
+
+ char c;
+ ASSERT_FALSE(reader.peek(c));
+ }
+}
+
+TEST(Tokenizer, simplePeekToken)
+{
+ CharReader reader{"test1:test2"};
+ Tokenizer tokenizer;
+
+ const TokenTypeId tid = tokenizer.registerToken(":");
+ ASSERT_EQ(0U, tid);
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.peek(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test1", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(5U, loc.getEnd());
+ ASSERT_EQ(0U, reader.getOffset());
+ ASSERT_EQ(5U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.peek(reader, token));
+
+ ASSERT_EQ(tid, token.type);
+ ASSERT_EQ(":", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(5U, loc.getStart());
+ ASSERT_EQ(6U, loc.getEnd());
+ ASSERT_EQ(0U, reader.getOffset());
+ ASSERT_EQ(6U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.peek(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test2", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(6U, loc.getStart());
+ ASSERT_EQ(11U, loc.getEnd());
+ ASSERT_EQ(0U, reader.getOffset());
+ ASSERT_EQ(11U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test1", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(5U, loc.getEnd());
+ ASSERT_EQ(5U, reader.getOffset());
+ ASSERT_EQ(5U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(tid, token.type);
+ ASSERT_EQ(":", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(5U, loc.getStart());
+ ASSERT_EQ(6U, loc.getEnd());
+ ASSERT_EQ(6U, reader.getOffset());
+ ASSERT_EQ(6U, reader.getPeekOffset());
+ }
+
+ {
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("test2", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(6U, loc.getStart());
+ ASSERT_EQ(11U, loc.getEnd());
+ ASSERT_EQ(11U, reader.getOffset());
+ ASSERT_EQ(11U, reader.getPeekOffset());
+ }
+}
+
+TEST(Tokenizer, ambiguousTokens)
+{
+ CharReader reader{"abc"};
+ Tokenizer tokenizer;
+
+ TokenTypeId t1 = tokenizer.registerToken("abd");
+ TokenTypeId t2 = tokenizer.registerToken("bc");
+
+ ASSERT_EQ(0U, t1);
+ ASSERT_EQ(1U, t2);
+
+ Token token;
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(TextToken, token.type);
+ ASSERT_EQ("a", token.content);
+
+ SourceLocation loc = token.location;
+ ASSERT_EQ(0U, loc.getStart());
+ ASSERT_EQ(1U, loc.getEnd());
+
+ ASSERT_TRUE(tokenizer.read(reader, token));
+
+ ASSERT_EQ(t2, token.type);
+ ASSERT_EQ("bc", token.content);
+
+ loc = token.location;
+ ASSERT_EQ(1U, loc.getStart());
+ ASSERT_EQ(3U, loc.getEnd());
+
+ ASSERT_FALSE(tokenizer.read(reader, token));
+}
+
+TEST(Tokenizer, commentTestWhitespacePreserve)
+{
+ CharReader reader{"Test/Test /* Block Comment */", 0};
+ // 012345678901234567890123456789
+ // 0 1 2
+ Tokenizer tokenizer(WhitespaceMode::PRESERVE);
+
+ const TokenTypeId t1 = tokenizer.registerToken("/");
+ const TokenTypeId t2 = tokenizer.registerToken("/*");
+ const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+ std::vector<Token> expected = {
+ {TextToken, "Test", SourceLocation{0, 0, 4}},
+ {t1, "/", SourceLocation{0, 4, 5}},
+ {TextToken, "Test ", SourceLocation{0, 5, 10}},
+ {t2, "/*", SourceLocation{0, 10, 12}},
+ {TextToken, " Block Comment ", SourceLocation{0, 12, 27}},
+ {t3, "*/", SourceLocation{0, 27, 29}}};
+
+ Token t;
+ for (auto &te : expected) {
+ EXPECT_TRUE(tokenizer.read(reader, t));
+ EXPECT_EQ(te.type, t.type);
+ EXPECT_EQ(te.content, t.content);
+ EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+ EXPECT_EQ(te.location.getStart(), t.location.getStart());
+ EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+ }
+ ASSERT_FALSE(tokenizer.read(reader, t));
+}
+
+TEST(Tokenizer, commentTestWhitespaceCollapse)
+{
+ CharReader reader{"Test/Test /* Block Comment */", 0};
+ // 012345678901234567890123456789
+ // 0 1 2
+ Tokenizer tokenizer(WhitespaceMode::COLLAPSE);
+
+ const TokenTypeId t1 = tokenizer.registerToken("/");
+ const TokenTypeId t2 = tokenizer.registerToken("/*");
+ const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+ std::vector<Token> expected = {
+ {TextToken, "Test", SourceLocation{0, 0, 4}},
+ {t1, "/", SourceLocation{0, 4, 5}},
+ {TextToken, "Test", SourceLocation{0, 5, 9}},
+ {t2, "/*", SourceLocation{0, 10, 12}},
+ {TextToken, "Block Comment", SourceLocation{0, 13, 26}},
+ {t3, "*/", SourceLocation{0, 27, 29}}};
+
+ Token t;
+ for (auto &te : expected) {
+ EXPECT_TRUE(tokenizer.read(reader, t));
+ EXPECT_EQ(te.type, t.type);
+ EXPECT_EQ(te.content, t.content);
+ EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+ EXPECT_EQ(te.location.getStart(), t.location.getStart());
+ EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+ }
+ ASSERT_FALSE(tokenizer.read(reader, t));
+}
+}
+
generated by cgit v1.2.3 (git 2.39.1) at 2025-11-04 10:15:33 +0000