Merge branch 'astoecke_parser_stack_new'

Conflicts:
	application/src/core/parser/stack/DocumentHandler.cpp
	application/src/core/parser/stack/DocumentHandler.hpp

6 files changed, 1251 insertions, 254 deletions

diff --git a/test/core/parser/ParserStackTest.cpp b/test/core/parser/ParserStackTest.cpp
deleted file mode 100644
index 3a0decb..0000000
--- a/test/core/parser/ParserStackTest.cpp
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
-    Ousía
-    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <iostream>
-
-#include <gtest/gtest.h>
-
-#include <core/parser/ParserStack.hpp>
-#include <core/StandaloneEnvironment.hpp>
-
-namespace ousia {
-
-ConcreteLogger logger;
-
-static int startCount = 0;
-static int endCount = 0;
-static int dataCount = 0;
-
-class TestHandler : public Handler {
-public:
-	using Handler::Handler;
-
-	void start(Variant::mapType &args) override { startCount++; }
-
-	void end() override { endCount++; }
-
-	void data(const std::string &data, int field) override { dataCount++; }
-
-	static Handler *create(const HandlerData &data)
-	{
-		return new TestHandler(data);
-	}
-};
-
-namespace ParserStates {
-static const ParserState Document =
-    ParserStateBuilder().parent(&None).elementHandler(TestHandler::create);
-static const ParserState Body = ParserStateBuilder()
-                                    .parent(&Document)
-                                    .elementHandler(TestHandler::create);
-static const ParserState Empty =
-    ParserStateBuilder().parent(&Document).elementHandler(TestHandler::create);
-static const ParserState Special =
-    ParserStateBuilder().parent(&All).elementHandler(TestHandler::create);
-static const ParserState Arguments =
-    ParserStateBuilder()
-        .parent(&None)
-        .elementHandler(TestHandler::create)
-        .arguments({Argument::Int("a"), Argument::String("b")});
-static const ParserState BodyChildren =
-    ParserStateBuilder()
-        .parent(&Body)
-        .elementHandler(TestHandler::create);
-
-static const std::multimap<std::string, const ParserState *> TestHandlers{
-    {"document", &Document},
-    {"body", &Body},
-    {"empty", &Empty},
-    {"special", &Special},
-    {"arguments", &Arguments},
-    {"*", &BodyChildren}};
-}
-
-TEST(ParserStack, simpleTest)
-{
-	StandaloneEnvironment env(logger);
-	ParserStack s{env.context, ParserStates::TestHandlers};
-
-	startCount = 0;
-	endCount = 0;
-	dataCount = 0;
-
-	EXPECT_EQ("", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::None, &s.currentState());
-
-	s.start("document", {});
-	s.data("test1");
-
-	EXPECT_EQ("document", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Document, &s.currentState());
-	EXPECT_EQ(1, startCount);
-	EXPECT_EQ(1, dataCount);
-
-	s.start("body", {});
-	s.data("test2");
-	EXPECT_EQ("body", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Body, &s.currentState());
-	EXPECT_EQ(2, startCount);
-	EXPECT_EQ(2, dataCount);
-
-	s.start("inner", {});
-	EXPECT_EQ("inner", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::BodyChildren, &s.currentState());
-	s.end();
-	EXPECT_EQ(3, startCount);
-	EXPECT_EQ(1, endCount);
-
-	s.end();
-	EXPECT_EQ(2, endCount);
-
-	EXPECT_EQ("document", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
-	s.start("body", {});
-	s.data("test3");
-	EXPECT_EQ("body", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Body, &s.currentState());
-	s.end();
-	EXPECT_EQ(4, startCount);
-	EXPECT_EQ(3, dataCount);
-	EXPECT_EQ(3, endCount);
-
-	EXPECT_EQ("document", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
-	s.end();
-	EXPECT_EQ(4, endCount);
-
-	EXPECT_EQ("", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::None, &s.currentState());
-}
-
-TEST(ParserStack, errorHandling)
-{
-	StandaloneEnvironment env(logger);
-	ParserStack s{env.context, ParserStates::TestHandlers};
-
-	EXPECT_THROW(s.start("body", {}), OusiaException);
-	s.start("document", {});
-	EXPECT_THROW(s.start("document", {}), OusiaException);
-	s.start("empty", {});
-	EXPECT_THROW(s.start("body", {}), OusiaException);
-	s.start("special", {});
-	s.end();
-	s.end();
-	s.end();
-	EXPECT_EQ(&ParserStates::None, &s.currentState());
-	ASSERT_THROW(s.end(), OusiaException);
-	ASSERT_THROW(s.data("test", 1), OusiaException);
-}
-
-TEST(ParserStack, validation)
-{
-	StandaloneEnvironment env(logger);
-	ParserStack s{env.context, ParserStates::TestHandlers};
-
-	logger.reset();
-	s.start("arguments", {});
-	EXPECT_TRUE(logger.hasError());
-	s.end();
-
-	s.start("arguments", {{"a", 5}});
-	EXPECT_TRUE(logger.hasError());
-	s.end();
-
-	logger.reset();
-	s.start("arguments", {{"a", 5}, {"b", "test"}});
-	EXPECT_FALSE(logger.hasError());
-	s.end();
-}
-}
-
diff --git a/test/core/parser/ParserStateTest.cpp b/test/core/parser/ParserStateTest.cpp
deleted file mode 100644
index 91d8dcd..0000000
--- a/test/core/parser/ParserStateTest.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
-    Ousía
-    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/common/Rtti.hpp>
-#include <core/parser/ParserState.hpp>
-
-namespace ousia {
-
-static const Rtti t1;
-static const Rtti t2;
-static const Rtti t3;
-static const Rtti t4;
-static const Rtti t5;
-
-static const ParserState s1 = ParserStateBuilder().createdNodeType(&t1);
-static const ParserState s2a =
-    ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s2b =
-    ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s3 =
-    ParserStateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
-static const ParserState s4 =
-    ParserStateBuilder().parent(&s3).createdNodeType(&t4);
-static const ParserState s5 =
-    ParserStateBuilder().parent(&s2b).createdNodeType(&t5);
-
-TEST(ParserStateDeductor, deduce)
-{
-	using Result = std::vector<const ParserState *>;
-	using Signature = std::vector<const Rtti *>;
-	std::vector<const ParserState *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
-
-	// Should not crash on empty signature
-	ASSERT_EQ(Result{}, ParserStateDeductor(Signature{}, states).deduce());
-
-	// Try repeating signature elements
-	ASSERT_EQ(Result({&s1}),
-	          ParserStateDeductor(Signature({&t1}), states).deduce());
-	ASSERT_EQ(Result({&s1}),
-	          ParserStateDeductor(Signature({&t1, &t1}), states).deduce());
-	ASSERT_EQ(Result({&s1}),
-	          ParserStateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
-
-	// Go to another state
-	ASSERT_EQ(Result({&s2a, &s2b}),
-	          ParserStateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
-	ASSERT_EQ(Result({&s4}),
-	          ParserStateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
-
-	// Skip one state
-	ASSERT_EQ(Result({&s4}),
-	          ParserStateDeductor(Signature({&t2, &t4}), states).deduce());
-
-	// Impossible signature
-	ASSERT_EQ(Result({}),
-	          ParserStateDeductor(Signature({&t4, &t5}), states).deduce());
-
-}
-}
-
diff --git a/test/core/parser/stack/StackTest.cpp b/test/core/parser/stack/StackTest.cpp
new file mode 100644
index 0000000..321d471
--- /dev/null
+++ b/test/core/parser/stack/StackTest.cpp
@@ -0,0 +1,666 @@
+/*
+    Ousía
+    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <iostream>
+
+#include <gtest/gtest.h>
+
+#include <core/frontend/TerminalLogger.hpp>
+#include <core/parser/stack/Handler.hpp>
+#include <core/parser/stack/Stack.hpp>
+#include <core/parser/stack/State.hpp>
+
+#include <core/StandaloneEnvironment.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+// Build an instance of the StandaloneEnvironment used for this unit test
+static TerminalLogger logger(std::cerr, true);
+// static ConcreteLogger logger;
+static StandaloneEnvironment env(logger);
+
+namespace {
+
+struct Tracker {
+	int startCount;
+	int endCount;
+	int fieldStartCount;
+	int fieldEndCount;
+	int annotationStartCount;
+	int annotationEndCount;
+	int dataCount;
+
+	Variant::mapType startArgs;
+	bool fieldStartIsDefault;
+	size_t fieldStartIdx;
+	Variant annotationStartClassName;
+	Variant::mapType annotationStartArgs;
+	Variant annotationEndClassName;
+	Variant annotationEndElementName;
+	Variant dataData;
+
+	bool startResult;
+	bool fieldStartSetIsDefault;
+	bool fieldStartResult;
+	bool annotationStartResult;
+	bool annotationEndResult;
+	bool dataResult;
+
+	Tracker() { reset(); }
+
+	void reset()
+	{
+		startCount = 0;
+		endCount = 0;
+		fieldStartCount = 0;
+		fieldEndCount = 0;
+		annotationStartCount = 0;
+		annotationEndCount = 0;
+		dataCount = 0;
+
+		startArgs = Variant::mapType{};
+		fieldStartIsDefault = false;
+		fieldStartIdx = 0;
+		annotationStartClassName = Variant::fromString(std::string{});
+		annotationStartArgs = Variant::mapType{};
+		annotationEndClassName = Variant::fromString(std::string{});
+		annotationEndElementName = Variant::fromString(std::string{});
+		dataData = Variant::fromString(std::string{});
+
+		startResult = true;
+		fieldStartSetIsDefault = false;
+		fieldStartResult = true;
+		annotationStartResult = true;
+		annotationEndResult = true;
+		dataResult = true;
+	}
+
+	void expect(int startCount, int endCount, int fieldStartCount,
+	            int fieldEndCount, int annotationStartCount,
+	            int annotationEndCount, int dataCount)
+	{
+		EXPECT_EQ(startCount, this->startCount);
+		EXPECT_EQ(endCount, this->endCount);
+		EXPECT_EQ(fieldStartCount, this->fieldStartCount);
+		EXPECT_EQ(fieldEndCount, this->fieldEndCount);
+		EXPECT_EQ(annotationStartCount, this->annotationStartCount);
+		EXPECT_EQ(annotationEndCount, this->annotationEndCount);
+		EXPECT_EQ(dataCount, this->dataCount);
+	}
+};
+
+static Tracker tracker;
+
+class TestHandler : public Handler {
+private:
+	TestHandler(const HandlerData &handlerData) : Handler(handlerData) {}
+
+public:
+	bool start(Variant::mapType &args) override
+	{
+		tracker.startCount++;
+		tracker.startArgs = args;
+		if (!tracker.startResult) {
+			logger().error(
+			    "The TestHandler was told not to allow a field start. So it "
+			    "doesn't. The TestHandler always obeys its master.");
+		}
+		return tracker.startResult;
+	}
+
+	void end() override { tracker.endCount++; }
+
+	bool fieldStart(bool &isDefault, size_t fieldIdx) override
+	{
+		tracker.fieldStartCount++;
+		tracker.fieldStartIsDefault = isDefault;
+		tracker.fieldStartIdx = fieldIdx;
+		if (tracker.fieldStartSetIsDefault) {
+			isDefault = true;
+		}
+		return tracker.fieldStartResult;
+	}
+
+	void fieldEnd() override { tracker.fieldEndCount++; }
+
+	bool annotationStart(const Variant &className,
+	                     Variant::mapType &args) override
+	{
+		tracker.annotationStartCount++;
+		tracker.annotationStartClassName = className;
+		tracker.annotationStartArgs = args;
+		return tracker.annotationStartResult;
+	}
+
+	bool annotationEnd(const Variant &className,
+	                   const Variant &elementName) override
+	{
+		tracker.annotationEndCount++;
+		tracker.annotationEndClassName = className;
+		tracker.annotationEndElementName = elementName;
+		return tracker.annotationEndResult;
+	}
+
+	bool data(Variant &data) override
+	{
+		tracker.dataCount++;
+		tracker.dataData = data;
+		return tracker.dataResult;
+	}
+
+	static Handler *create(const HandlerData &handlerData)
+	{
+		return new TestHandler(handlerData);
+	}
+};
+}
+
+namespace States {
+static const State Document =
+    StateBuilder().parent(&None).elementHandler(TestHandler::create);
+static const State Body =
+    StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Empty =
+    StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Special =
+    StateBuilder().parent(&All).elementHandler(TestHandler::create);
+static const State Arguments =
+    StateBuilder().parent(&None).elementHandler(TestHandler::create).arguments(
+        {Argument::Int("a"), Argument::String("b")});
+static const State BodyChildren =
+    StateBuilder().parent(&Body).elementHandler(TestHandler::create);
+static const State Any =
+    StateBuilder().parents({&None, &Any}).elementHandler(TestHandler::create);
+
+static const std::multimap<std::string, const State *> TestHandlers{
+    {"document", &Document},
+    {"body", &Body},
+    {"empty", &Empty},
+    {"special", &Special},
+    {"arguments", &Arguments},
+    {"*", &BodyChildren}};
+
+static const std::multimap<std::string, const State *> AnyHandlers{{"*", &Any}};
+}
+
+TEST(Stack, basicTest)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::TestHandlers};
+
+		EXPECT_EQ("", s.currentCommandName());
+		EXPECT_EQ(&States::None, &s.currentState());
+
+		s.command("document", {});
+		s.fieldStart(true);
+		s.data("test1");
+
+		EXPECT_EQ("document", s.currentCommandName());
+		EXPECT_EQ(&States::Document, &s.currentState());
+		tracker.expect(1, 0, 1, 0, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("body", {});
+		s.fieldStart(true);
+		s.data("test2");
+		EXPECT_EQ("body", s.currentCommandName());
+		EXPECT_EQ(&States::Body, &s.currentState());
+		tracker.expect(2, 0, 2, 0, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("inner", {});
+		s.fieldStart(true);
+		EXPECT_EQ("inner", s.currentCommandName());
+		EXPECT_EQ(&States::BodyChildren, &s.currentState());
+
+		s.fieldEnd();
+		tracker.expect(3, 1, 3, 1, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldEnd();
+		EXPECT_EQ("document", s.currentCommandName());
+		EXPECT_EQ(&States::Document, &s.currentState());
+		tracker.expect(3, 2, 3, 2, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("body", {});
+		s.fieldStart(true);
+		s.data("test3");
+		EXPECT_EQ("body", s.currentCommandName());
+		EXPECT_EQ(&States::Body, &s.currentState());
+		s.fieldEnd();
+		tracker.expect(4, 3, 4, 3, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+
+		EXPECT_EQ("document", s.currentCommandName());
+		EXPECT_EQ(&States::Document, &s.currentState());
+
+		s.fieldEnd();
+		tracker.expect(4, 4, 4, 4, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+
+		EXPECT_EQ("", s.currentCommandName());
+		EXPECT_EQ(&States::None, &s.currentState());
+	}
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidCommands)
+{
+	Stack s{env.context, States::TestHandlers};
+	tracker.reset();
+	EXPECT_THROW(s.command("body", {}), LoggableException);
+	s.command("document", {});
+	s.fieldStart(true);
+	EXPECT_THROW(s.command("document", {}), LoggableException);
+	s.command("empty", {});
+	s.fieldStart(true);
+	EXPECT_THROW(s.command("body", {}), LoggableException);
+	s.command("special", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	s.fieldEnd();
+	s.fieldEnd();
+	EXPECT_EQ(&States::None, &s.currentState());
+	ASSERT_THROW(s.fieldEnd(), LoggableException);
+	ASSERT_THROW(s.data("test"), LoggableException);
+}
+
+TEST(Stack, validation)
+{
+	Stack s{env.context, States::TestHandlers};
+	tracker.reset();
+	logger.reset();
+
+	s.command("arguments", {});
+	EXPECT_TRUE(logger.hasError());
+	s.fieldStart(true);
+	s.fieldEnd();
+
+	logger.reset();
+	s.command("arguments", {{"a", 5}});
+	EXPECT_TRUE(logger.hasError());
+	s.fieldStart(true);
+	s.fieldEnd();
+
+	logger.reset();
+	s.command("arguments", {{"a", 5}, {"b", "test"}});
+	EXPECT_FALSE(logger.hasError());
+	s.fieldStart(true);
+	s.fieldEnd();
+}
+
+TEST(Stack, invalidCommandName)
+{
+	Stack s{env.context, States::AnyHandlers};
+	tracker.reset();
+	logger.reset();
+
+	s.command("a", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+	s.command("a_", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	tracker.expect(2, 2, 2, 2, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+	s.command("a_:b", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	tracker.expect(3, 3, 3, 3, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+	ASSERT_THROW(s.command("_a", {}), LoggableException);
+	ASSERT_THROW(s.command("a:", {}), LoggableException);
+	ASSERT_THROW(s.command("a:_b", {}), LoggableException);
+	tracker.expect(3, 3, 3, 3, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, multipleFields)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {{"a", false}});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("a", s.currentCommandName());
+		EXPECT_EQ(Variant::mapType({{"a", false}}), tracker.startArgs);
+
+		s.fieldStart(false);
+		tracker.expect(1, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_FALSE(tracker.fieldStartIsDefault);
+		EXPECT_EQ(0U, tracker.fieldStartIdx);
+
+		s.data("test");
+		tracker.expect(1, 0, 1, 0, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("test", tracker.dataData);
+
+		s.fieldEnd();
+		tracker.expect(1, 0, 1, 1, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		tracker.expect(1, 0, 2, 1, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_FALSE(tracker.fieldStartIsDefault);
+		EXPECT_EQ(1U, tracker.fieldStartIdx);
+
+		s.data("test2");
+		tracker.expect(1, 0, 2, 1, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("test2", tracker.dataData);
+
+		s.fieldEnd();
+		tracker.expect(1, 0, 2, 2, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(true);
+		tracker.expect(1, 0, 3, 2, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_TRUE(tracker.fieldStartIsDefault);
+		EXPECT_EQ(2U, tracker.fieldStartIdx);
+
+		s.data("test3");
+		tracker.expect(1, 0, 3, 2, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("test3", tracker.dataData);
+
+		s.fieldEnd();
+		tracker.expect(1, 1, 3, 3, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommand)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("b", {});
+		tracker.expect(2, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(2, 2, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommandWithExplicitDefaultField)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+
+		s.command("b", {});
+		tracker.expect(2, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+		s.fieldStart(true);
+		s.fieldEnd();
+		tracker.expect(2, 1, 2, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+	}
+	tracker.expect(2, 2, 2, 2, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldOnIncompatibleCommand)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+
+		tracker.fieldStartResult = false;
+		s.command("b", {});
+		tracker.expect(2, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+	}
+	tracker.expect(2, 2, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldIfDefaultFieldGiven)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+		s.fieldStart(true);
+		tracker.expect(1, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+		s.fieldEnd();
+		tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("", s.currentCommandName());
+
+		s.command("b", {});
+		tracker.expect(2, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+	}
+	tracker.expect(2, 2, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noEndIfStartFails)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+
+		tracker.startResult = false;
+		s.command("b", {});
+		tracker.expect(3, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+	}
+	tracker.expect(3, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_TRUE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnData)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.data("test");
+		tracker.expect(1, 0, 1, 0, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoFieldEnd)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoImplicitFieldEnd)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		s.command("b", {});
+		s.command("c", {});
+		s.command("d", {});
+		s.command("e", {});
+		s.fieldStart(true);
+		s.fieldEnd();
+		tracker.expect(5, 1, 5, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(5, 5, 5, 5, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, invalidDefaultField)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.fieldStartResult = false;
+		s.fieldStart(true);
+		s.fieldEnd();
+		tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidDefaultFieldData)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.fieldStartResult = false;
+		s.fieldStart(true);
+		ASSERT_FALSE(logger.hasError());
+		s.data("test");
+		ASSERT_TRUE(logger.hasError());
+		s.fieldEnd();
+		tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorInvalidFieldData)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.fieldStartResult = false;
+		ASSERT_FALSE(logger.hasError());
+		s.fieldStart(false);
+		ASSERT_TRUE(logger.hasError());
+		s.data("test");
+		s.fieldEnd();
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorFieldStartNoCommand)
+{
+	tracker.reset();
+	logger.reset();
+
+	Stack s{env.context, States::AnyHandlers};
+	ASSERT_THROW(s.fieldStart(false), LoggableException);
+	ASSERT_THROW(s.fieldStart(true), LoggableException);
+	tracker.expect(0, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldStarts)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		ASSERT_FALSE(logger.hasError());
+		s.fieldStart(false);
+		ASSERT_TRUE(logger.hasError());
+		tracker.expect(1, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldEnd();
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldEnds)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		s.fieldEnd();
+		ASSERT_FALSE(logger.hasError());
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		s.fieldEnd();
+		ASSERT_TRUE(logger.hasError());
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorOpenField)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		ASSERT_FALSE(logger.hasError());
+	}
+	ASSERT_TRUE(logger.hasError());
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+}
+}
+
diff --git a/test/core/parser/stack/StateTest.cpp b/test/core/parser/stack/StateTest.cpp
new file mode 100644
index 0000000..e503d30
--- /dev/null
+++ b/test/core/parser/stack/StateTest.cpp
@@ -0,0 +1,79 @@
+/*
+    Ousía
+    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/Rtti.hpp>
+#include <core/parser/stack/State.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+static const Rtti t1;
+static const Rtti t2;
+static const Rtti t3;
+static const Rtti t4;
+static const Rtti t5;
+
+static const State s1 = StateBuilder().createdNodeType(&t1);
+static const State s2a =
+    StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s2b =
+    StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s3 =
+    StateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
+static const State s4 =
+    StateBuilder().parent(&s3).createdNodeType(&t4);
+static const State s5 =
+    StateBuilder().parent(&s2b).createdNodeType(&t5);
+
+TEST(StateDeductor, deduce)
+{
+	using Result = std::vector<const State *>;
+	using Signature = std::vector<const Rtti *>;
+	std::vector<const State *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
+
+	// Should not crash on empty signature
+	ASSERT_EQ(Result{}, StateDeductor(Signature{}, states).deduce());
+
+	// Try repeating signature elements
+	ASSERT_EQ(Result({&s1}),
+	          StateDeductor(Signature({&t1}), states).deduce());
+	ASSERT_EQ(Result({&s1}),
+	          StateDeductor(Signature({&t1, &t1}), states).deduce());
+	ASSERT_EQ(Result({&s1}),
+	          StateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
+
+	// Go to another state
+	ASSERT_EQ(Result({&s2a, &s2b}),
+	          StateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
+	ASSERT_EQ(Result({&s4}),
+	          StateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
+
+	// Skip one state
+	ASSERT_EQ(Result({&s4}),
+	          StateDeductor(Signature({&t2, &t4}), states).deduce());
+
+	// Impossible signature
+	ASSERT_EQ(Result({}),
+	          StateDeductor(Signature({&t4, &t5}), states).deduce());
+
+}
+}
+}
+
diff --git a/test/core/parser/utils/TokenTrieTest.cpp b/test/core/parser/utils/TokenTrieTest.cpp
new file mode 100644
index 0000000..087e6e6
--- /dev/null
+++ b/test/core/parser/utils/TokenTrieTest.cpp
@@ -0,0 +1,92 @@
+/*
+    Ousía
+    Copyright (C) 2014  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/parser/utils/TokenTrie.hpp>
+
+namespace ousia {
+
+static const TokenTypeId t1 = 0;
+static const TokenTypeId t2 = 1;
+static const TokenTypeId t3 = 2;
+static const TokenTypeId t4 = 3;
+
+TEST(TokenTrie, registerToken)
+{
+	TokenTrie tree;
+
+	ASSERT_TRUE(tree.registerToken("a", t1));
+	ASSERT_TRUE(tree.registerToken("ab", t2));
+	ASSERT_TRUE(tree.registerToken("b", t3));
+	ASSERT_TRUE(tree.registerToken("hello", t4));
+
+	ASSERT_FALSE(tree.registerToken("", t1));
+	ASSERT_FALSE(tree.registerToken("a", t4));
+	ASSERT_FALSE(tree.registerToken("ab", t4));
+	ASSERT_FALSE(tree.registerToken("b", t4));
+	ASSERT_FALSE(tree.registerToken("hello", t4));
+
+	ASSERT_EQ(t1, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(t3, tree.hasToken("b"));
+	ASSERT_EQ(t4, tree.hasToken("hello"));
+	ASSERT_EQ(EmptyToken, tree.hasToken(""));
+	ASSERT_EQ(EmptyToken, tree.hasToken("abc"));
+}
+
+TEST(TokenTrie, unregisterToken)
+{
+	TokenTrie tree;
+
+	ASSERT_TRUE(tree.registerToken("a", t1));
+	ASSERT_FALSE(tree.registerToken("a", t4));
+
+	ASSERT_TRUE(tree.registerToken("ab", t2));
+	ASSERT_FALSE(tree.registerToken("ab", t4));
+
+	ASSERT_TRUE(tree.registerToken("b", t3));
+	ASSERT_FALSE(tree.registerToken("b", t4));
+
+	ASSERT_EQ(t1, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(t3, tree.hasToken("b"));
+
+	ASSERT_TRUE(tree.unregisterToken("a"));
+	ASSERT_FALSE(tree.unregisterToken("a"));
+
+	ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(t3, tree.hasToken("b"));
+
+	ASSERT_TRUE(tree.unregisterToken("b"));
+	ASSERT_FALSE(tree.unregisterToken("b"));
+
+	ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+
+	ASSERT_TRUE(tree.unregisterToken("ab"));
+	ASSERT_FALSE(tree.unregisterToken("ab"));
+
+	ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+	ASSERT_EQ(EmptyToken, tree.hasToken("ab"));
+	ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+}
+}
+
diff --git a/test/core/parser/utils/TokenizerTest.cpp b/test/core/parser/utils/TokenizerTest.cpp
new file mode 100644
index 0000000..8565057
--- /dev/null
+++ b/test/core/parser/utils/TokenizerTest.cpp
@@ -0,0 +1,414 @@
+/*
+    Ousía
+    Copyright (C) 2014  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/CharReader.hpp>
+#include <core/parser/utils/Tokenizer.hpp>
+
+namespace ousia {
+
+TEST(Tokenizer, tokenRegistration)
+{
+	Tokenizer tokenizer;
+
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken(""));
+
+	ASSERT_EQ(0U, tokenizer.registerToken("a"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("a"));
+	ASSERT_EQ("a", tokenizer.getTokenString(0U));
+
+	ASSERT_EQ(1U, tokenizer.registerToken("b"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("b"));
+	ASSERT_EQ("b", tokenizer.getTokenString(1U));
+
+	ASSERT_EQ(2U, tokenizer.registerToken("c"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("c"));
+	ASSERT_EQ("c", tokenizer.getTokenString(2U));
+
+	ASSERT_TRUE(tokenizer.unregisterToken(1U));
+	ASSERT_FALSE(tokenizer.unregisterToken(1U));
+	ASSERT_EQ("", tokenizer.getTokenString(1U));
+
+	ASSERT_EQ(1U, tokenizer.registerToken("d"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("d"));
+	ASSERT_EQ("d", tokenizer.getTokenString(1U));
+}
+
+TEST(Tokenizer, textTokenPreserveWhitespace)
+{
+	{
+		CharReader reader{" this \t is only a  \n\n test   text   "};
+		//                 012345 6789012345678 9 0123456789012345
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ(" this \t is only a  \n\n test   text   ", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(36U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+
+	{
+		CharReader reader{"this \t is only a  \n\n test   text"};
+		//                 01234 5678901234567 8 9012345678901
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this \t is only a  \n\n test   text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(32U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+}
+
+TEST(Tokenizer, textTokenTrimWhitespace)
+{
+	{
+		CharReader reader{" this \t is only a  \n\n test   text   "};
+		//                 012345 6789012345678 9 0123456789012345
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this \t is only a  \n\n test   text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(33U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+
+	{
+		CharReader reader{"this \t is only a  \n\n test   text"};
+		//                 01234 5678901234567 8 9012345678901
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this \t is only a  \n\n test   text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(32U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+}
+
+TEST(Tokenizer, textTokenCollapseWhitespace)
+{
+	{
+		CharReader reader{" this \t is only a  \n\n test   text   "};
+		//                 012345 6789012345678 9 0123456789012345
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this is only a test text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(33U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+
+	{
+		CharReader reader{"this \t is only a  \n\n test   text"};
+		//                 01234 5678901234567 8 9012345678901
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this is only a test text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(32U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+}
+
+TEST(Tokenizer, simpleReadToken)
+{
+	CharReader reader{"test1:test2"};
+	Tokenizer tokenizer;
+
+	const TokenTypeId tid = tokenizer.registerToken(":");
+	ASSERT_EQ(0U, tid);
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test1", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
+
+		char c;
+		ASSERT_TRUE(reader.peek(c));
+		ASSERT_EQ(':', c);
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(tid, token.type);
+		ASSERT_EQ(":", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(5U, loc.getStart());
+		ASSERT_EQ(6U, loc.getEnd());
+
+		char c;
+		ASSERT_TRUE(reader.peek(c));
+		ASSERT_EQ('t', c);
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test2", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(6U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
+
+		char c;
+		ASSERT_FALSE(reader.peek(c));
+	}
+}
+
+TEST(Tokenizer, simplePeekToken)
+{
+	CharReader reader{"test1:test2"};
+	Tokenizer tokenizer;
+
+	const TokenTypeId tid = tokenizer.registerToken(":");
+	ASSERT_EQ(0U, tid);
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.peek(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test1", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
+		ASSERT_EQ(0U, reader.getOffset());
+		ASSERT_EQ(5U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.peek(reader, token));
+
+		ASSERT_EQ(tid, token.type);
+		ASSERT_EQ(":", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(5U, loc.getStart());
+		ASSERT_EQ(6U, loc.getEnd());
+		ASSERT_EQ(0U, reader.getOffset());
+		ASSERT_EQ(6U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.peek(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test2", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(6U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
+		ASSERT_EQ(0U, reader.getOffset());
+		ASSERT_EQ(11U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test1", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
+		ASSERT_EQ(5U, reader.getOffset());
+		ASSERT_EQ(5U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(tid, token.type);
+		ASSERT_EQ(":", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(5U, loc.getStart());
+		ASSERT_EQ(6U, loc.getEnd());
+		ASSERT_EQ(6U, reader.getOffset());
+		ASSERT_EQ(6U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test2", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(6U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
+		ASSERT_EQ(11U, reader.getOffset());
+		ASSERT_EQ(11U, reader.getPeekOffset());
+	}
+}
+
+TEST(Tokenizer, ambiguousTokens)
+{
+	CharReader reader{"abc"};
+	Tokenizer tokenizer;
+
+	TokenTypeId t1 = tokenizer.registerToken("abd");
+	TokenTypeId t2 = tokenizer.registerToken("bc");
+
+	ASSERT_EQ(0U, t1);
+	ASSERT_EQ(1U, t2);
+
+	Token token;
+	ASSERT_TRUE(tokenizer.read(reader, token));
+
+	ASSERT_EQ(TextToken, token.type);
+	ASSERT_EQ("a", token.content);
+
+	SourceLocation loc = token.location;
+	ASSERT_EQ(0U, loc.getStart());
+	ASSERT_EQ(1U, loc.getEnd());
+
+	ASSERT_TRUE(tokenizer.read(reader, token));
+
+	ASSERT_EQ(t2, token.type);
+	ASSERT_EQ("bc", token.content);
+
+	loc = token.location;
+	ASSERT_EQ(1U, loc.getStart());
+	ASSERT_EQ(3U, loc.getEnd());
+
+	ASSERT_FALSE(tokenizer.read(reader, token));
+}
+
+TEST(Tokenizer, commentTestWhitespacePreserve)
+{
+	CharReader reader{"Test/Test /* Block Comment */", 0};
+	//                 012345678901234567890123456789
+	//                 0        1         2
+	Tokenizer tokenizer(WhitespaceMode::PRESERVE);
+
+	const TokenTypeId t1 = tokenizer.registerToken("/");
+	const TokenTypeId t2 = tokenizer.registerToken("/*");
+	const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+	std::vector<Token> expected = {
+	    {TextToken, "Test", SourceLocation{0, 0, 4}},
+	    {t1, "/", SourceLocation{0, 4, 5}},
+	    {TextToken, "Test ", SourceLocation{0, 5, 10}},
+	    {t2, "/*", SourceLocation{0, 10, 12}},
+	    {TextToken, " Block Comment ", SourceLocation{0, 12, 27}},
+	    {t3, "*/", SourceLocation{0, 27, 29}}};
+
+	Token t;
+	for (auto &te : expected) {
+		EXPECT_TRUE(tokenizer.read(reader, t));
+		EXPECT_EQ(te.type, t.type);
+		EXPECT_EQ(te.content, t.content);
+		EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+		EXPECT_EQ(te.location.getStart(), t.location.getStart());
+		EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+	}
+	ASSERT_FALSE(tokenizer.read(reader, t));
+}
+
+TEST(Tokenizer, commentTestWhitespaceCollapse)
+{
+	CharReader reader{"Test/Test /* Block Comment */", 0};
+	//                 012345678901234567890123456789
+	//                 0        1         2
+	Tokenizer tokenizer(WhitespaceMode::COLLAPSE);
+
+	const TokenTypeId t1 = tokenizer.registerToken("/");
+	const TokenTypeId t2 = tokenizer.registerToken("/*");
+	const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+	std::vector<Token> expected = {
+	    {TextToken, "Test", SourceLocation{0, 0, 4}},
+	    {t1, "/", SourceLocation{0, 4, 5}},
+	    {TextToken, "Test", SourceLocation{0, 5, 9}},
+	    {t2, "/*", SourceLocation{0, 10, 12}},
+	    {TextToken, "Block Comment", SourceLocation{0, 13, 26}},
+	    {t3, "*/", SourceLocation{0, 27, 29}}};
+
+	Token t;
+	for (auto &te : expected) {
+		EXPECT_TRUE(tokenizer.read(reader, t));
+		EXPECT_EQ(te.type, t.type);
+		EXPECT_EQ(te.content, t.content);
+		EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+		EXPECT_EQ(te.location.getStart(), t.location.getStart());
+		EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+	}
+	ASSERT_FALSE(tokenizer.read(reader, t));
+}
+}
+