Merge branch 'master' of somweyr.de:ousia

12 files changed, 1313 insertions, 494 deletions

diff --git a/test/core/CodeTokenizerTest.cpp b/test/core/CodeTokenizerTest.cpp
deleted file mode 100644
index 2d4d5a7..0000000
--- a/test/core/CodeTokenizerTest.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
-    Ousía
-    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/CodeTokenizer.hpp>
-
-namespace ousia {
-
-static const int BLOCK_COMMENT = 30;
-static const int LINE_COMMENT = 31;
-static const int STRING = 20;
-static const int ESCAPE = 21;
-static const int LINEBREAK = 21;
-static const int CURLY_OPEN = 40;
-static const int CURLY_CLOSE = 41;
-
-TEST(CodeTokenizer, testTokenizer)
-{
-	CharReader reader{
-	    "/**\n"                                 // 1
-	    " * Some Block Comment\n"               // 2
-	    " */\n"                                 // 3
-	    "var my_string = 'My \\'String\\'';\n"  // 4
-	    "// and a line comment\n"               // 5
-	    "var my_obj = { a = 4;}", 0};              // 6
-	//   123456789012345678901234567890123456789
-	//   0        1         2         3
-	TokenTreeNode root{{{"/*", 1},
-	                    {"*/", 2},
-	                    {"//", 3},
-	                    {"'", 4},
-	                    {"\\", 5},
-	                    {"{", CURLY_OPEN},
-	                    {"}", CURLY_CLOSE},
-	                    {"\n", 6}}};
-	std::map<int, CodeTokenDescriptor> descriptors{
-	    // the block comment start Token has the id 1 and if the Tokenizer
-	    // returns a Block Comment Token that should have the id 10.
-	    {1, {CodeTokenMode::BLOCK_COMMENT_START, BLOCK_COMMENT}},
-	    {2, {CodeTokenMode::BLOCK_COMMENT_END, BLOCK_COMMENT}},
-	    {3, {CodeTokenMode::LINE_COMMENT, LINE_COMMENT}},
-	    {4, {CodeTokenMode::STRING_START_END, STRING}},
-	    {5, {CodeTokenMode::ESCAPE, ESCAPE}},
-	    {6, {CodeTokenMode::LINEBREAK, LINEBREAK}}};
-
-	std::vector<Token> expected = {
-	    {BLOCK_COMMENT, "*\n * Some Block Comment\n ", SourceLocation{0, 0, 29}},
-	    {LINEBREAK, "\n", SourceLocation{0, 29, 30}},
-	    {TOKEN_TEXT, "var", SourceLocation{0, 30, 33}},
-	    {TOKEN_TEXT, "my_string", SourceLocation{0, 34, 43}},
-	    {TOKEN_TEXT, "=", SourceLocation{0, 44, 45}},
-	    {STRING, "My 'String'", SourceLocation{0, 46, 61}},
-	    {TOKEN_TEXT, ";", SourceLocation{0, 61, 62}},
-	    {LINEBREAK, "\n", SourceLocation{0, 62, 63}},
-	    // this is slightly counter-intuitive but makes sense if you think about
-	    // it: As a line comment is ended by a line break the line break is
-	    // technically still a part of the line comment and thus the ending
-	    // is in the next line.
-	    {LINE_COMMENT, " and a line comment", SourceLocation{0, 63, 85}},
-	    {TOKEN_TEXT, "var", SourceLocation{0, 85, 88}},
-	    {TOKEN_TEXT, "my_obj", SourceLocation{0, 89, 95}},
-	    {TOKEN_TEXT, "=", SourceLocation{0, 96, 97}},
-	    {CURLY_OPEN, "{", SourceLocation{0, 98, 99}},
-	    {TOKEN_TEXT, "a", SourceLocation{0, 100, 101}},
-	    {TOKEN_TEXT, "=", SourceLocation{0, 102, 103}},
-	    {TOKEN_TEXT, "4;", SourceLocation{0, 104, 106}},
-	    {CURLY_CLOSE, "}", SourceLocation{0, 106, 107}},
-	};
-
-	CodeTokenizer tokenizer{reader, root, descriptors};
-
-	Token t;
-	for (auto &te : expected) {
-		EXPECT_TRUE(tokenizer.next(t));
-		EXPECT_EQ(te.tokenId, t.tokenId);
-		EXPECT_EQ(te.content, t.content);
-		EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
-		EXPECT_EQ(te.location.getStart(), t.location.getStart());
-		EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
-	}
-	ASSERT_FALSE(tokenizer.next(t));
-}
-}
-
diff --git a/test/core/RangeSetTest.cpp b/test/core/RangeSetTest.cpp
index cbf8f59..446ee51 100644
--- a/test/core/RangeSetTest.cpp
+++ b/test/core/RangeSetTest.cpp
@@ -110,7 +110,7 @@ TEST(RangeSet, Merge)
 	s.merge(Range<int>(40, 50));
 	s.merge(Range<int>(60, 70));
 	{
-		ASSERT_EQ(ranges.size(), 4);
+		ASSERT_EQ(ranges.size(), 4U);
 
 		auto it = ranges.begin();
 		ASSERT_EQ((*it).start, 0);
@@ -132,7 +132,7 @@ TEST(RangeSet, Merge)
 	// Now insert an element which spans the second and third element
 	s.merge(Range<int>(15, 55));
 	{
-		ASSERT_EQ(ranges.size(), 3);
+		ASSERT_EQ(ranges.size(), 3U);
 
 		auto it = ranges.begin();
 		ASSERT_EQ((*it).start, 0);
@@ -150,7 +150,7 @@ TEST(RangeSet, Merge)
 	// Now insert an element which expands the first element
 	s.merge(Range<int>(-10, 11));
 	{
-		ASSERT_EQ(ranges.size(), 3);
+		ASSERT_EQ(ranges.size(), 3U);
 
 		auto it = ranges.begin();
 		ASSERT_EQ((*it).start, -10);
@@ -168,7 +168,7 @@ TEST(RangeSet, Merge)
 	// Now insert an element which merges the last two elements
 	s.merge(Range<int>(13, 70));
 	{
-		ASSERT_EQ(ranges.size(), 2);
+		ASSERT_EQ(ranges.size(), 2U);
 
 		auto it = ranges.begin();
 		ASSERT_EQ((*it).start, -10);
@@ -182,7 +182,7 @@ TEST(RangeSet, Merge)
 	// Now insert an element which merges the remaining elements
 	s.merge(Range<int>(-9, 12));
 	{
-		ASSERT_EQ(ranges.size(), 1);
+		ASSERT_EQ(ranges.size(), 1U);
 
 		auto it = ranges.begin();
 		ASSERT_EQ((*it).start, -10);
diff --git a/test/core/StandaloneEnvironment.hpp b/test/core/StandaloneEnvironment.hpp
index a9dcdce..790bad4 100644
--- a/test/core/StandaloneEnvironment.hpp
+++ b/test/core/StandaloneEnvironment.hpp
@@ -31,6 +31,10 @@
 
 namespace ousia {
 
+/**
+ * StandaloneEnvironment is a class used for quickly setting up an entire
+ * environment needed for running an Ousia instance.
+ */
 struct StandaloneEnvironment {
 	ConcreteLogger &logger;
 	Manager manager;
diff --git a/test/core/TokenizerTest.cpp b/test/core/TokenizerTest.cpp
deleted file mode 100644
index c53f93d..0000000
--- a/test/core/TokenizerTest.cpp
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
-    Ousía
-    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/common/CharReader.hpp>
-
-#include <core/Tokenizer.hpp>
-
-namespace ousia {
-TEST(TokenTreeNode, testConstructor)
-{
-	TokenTreeNode root{{{"a", 1}, {"aab", 2}, {"aac", 3}, {"abd", 4}}};
-
-	ASSERT_EQ(-1, root.tokenId);
-	ASSERT_EQ(1U, root.children.size());
-	ASSERT_TRUE(root.children.find('a') != root.children.end());
-
-	const TokenTreeNode &a = root.children.at('a');
-	ASSERT_EQ(1, a.tokenId);
-	ASSERT_EQ(2U, a.children.size());
-	ASSERT_TRUE(a.children.find('a') != a.children.end());
-	ASSERT_TRUE(a.children.find('b') != a.children.end());
-
-	const TokenTreeNode &aa = a.children.at('a');
-	ASSERT_EQ(-1, aa.tokenId);
-	ASSERT_EQ(2U, aa.children.size());
-	ASSERT_TRUE(aa.children.find('b') != aa.children.end());
-	ASSERT_TRUE(aa.children.find('c') != aa.children.end());
-
-	const TokenTreeNode &aab = aa.children.at('b');
-	ASSERT_EQ(2, aab.tokenId);
-	ASSERT_EQ(0U, aab.children.size());
-
-	const TokenTreeNode &aac = aa.children.at('c');
-	ASSERT_EQ(3, aac.tokenId);
-	ASSERT_EQ(0U, aac.children.size());
-
-	const TokenTreeNode &ab = a.children.at('b');
-	ASSERT_EQ(-1, ab.tokenId);
-	ASSERT_EQ(1U, ab.children.size());
-	ASSERT_TRUE(ab.children.find('d') != ab.children.end());
-
-	const TokenTreeNode &abd = ab.children.at('d');
-	ASSERT_EQ(4, abd.tokenId);
-	ASSERT_EQ(0U, abd.children.size());
-}
-
-TEST(Tokenizer, testTokenization)
-{
-	TokenTreeNode root{{{"/", 1}, {"/*", 2}, {"*/", 3}}};
-
-	CharReader reader{"Test/Test /* Block Comment */", 0};
-	//                 012345678901234567890123456789
-	//                 0        1         2
-
-	std::vector<Token> expected = {
-	    {TOKEN_TEXT, "Test", SourceLocation{0, 0, 4}},
-	    {1, "/", SourceLocation{0, 4, 5}},
-	    {TOKEN_TEXT, "Test ", SourceLocation{0, 5, 10}},
-	    {2, "/*", SourceLocation{0, 10, 12}},
-	    {TOKEN_TEXT, " Block Comment ", SourceLocation{0, 12, 27}},
-	    {3, "*/", SourceLocation{0, 27, 29}}};
-
-	Tokenizer tokenizer{reader, root};
-
-	Token t;
-	for (auto &te : expected) {
-		EXPECT_TRUE(tokenizer.next(t));
-		EXPECT_EQ(te.tokenId, t.tokenId);
-		EXPECT_EQ(te.content, t.content);
-		EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
-		EXPECT_EQ(te.location.getStart(), t.location.getStart());
-		EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
-	}
-	ASSERT_FALSE(tokenizer.next(t));
-}
-
-TEST(Tokenizer, testIncompleteTokens)
-{
-	TokenTreeNode root{{{"ab", 1}, {"c", 2}}};
-
-	CharReader reader{"ac", 0};
-
-	std::vector<Token> expected = {
-	    {TOKEN_TEXT, "a", SourceLocation{0, 0, 1}},
-	    {2, "c", SourceLocation{0, 1, 2}}};
-
-	Tokenizer tokenizer{reader, root};
-
-	Token t;
-	for (auto &te : expected) {
-		EXPECT_TRUE(tokenizer.next(t));
-		EXPECT_EQ(te.tokenId, t.tokenId);
-		EXPECT_EQ(te.content, t.content);
-		EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
-		EXPECT_EQ(te.location.getStart(), t.location.getStart());
-		EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
-	}
-	ASSERT_FALSE(tokenizer.next(t));
-}
-}
-
diff --git a/test/core/common/UtilsTest.cpp b/test/core/common/UtilsTest.cpp
index 917f45c..7801296 100644
--- a/test/core/common/UtilsTest.cpp
+++ b/test/core/common/UtilsTest.cpp
@@ -24,22 +24,40 @@ namespace ousia {
 
 TEST(Utils, isIdentifier)
 {
-	ASSERT_TRUE(Utils::isIdentifier("test"));
-	ASSERT_TRUE(Utils::isIdentifier("t0-_est"));
-	ASSERT_FALSE(Utils::isIdentifier("_t0-_EST"));
-	ASSERT_FALSE(Utils::isIdentifier("-t0-_EST"));
-	ASSERT_FALSE(Utils::isIdentifier("0t-_EST"));
-	ASSERT_FALSE(Utils::isIdentifier("invalid key"));
+	EXPECT_TRUE(Utils::isIdentifier("test"));
+	EXPECT_TRUE(Utils::isIdentifier("t0-_est"));
+	EXPECT_FALSE(Utils::isIdentifier("_t0-_EST"));
+	EXPECT_FALSE(Utils::isIdentifier("-t0-_EST"));
+	EXPECT_FALSE(Utils::isIdentifier("0t-_EST"));
+	EXPECT_FALSE(Utils::isIdentifier("_A"));
+	EXPECT_FALSE(Utils::isIdentifier("invalid key"));
+	EXPECT_FALSE(Utils::isIdentifier(""));
 }
 
-TEST(Utils, trim)
+
+TEST(Utils, isNamespacedIdentifier)
 {
-	ASSERT_EQ("hello world", Utils::trim("\t hello world   \n\r\t"));
-	ASSERT_EQ("hello world", Utils::trim("hello world   \n\r\t"));
-	ASSERT_EQ("hello world", Utils::trim("   hello world"));
-	ASSERT_EQ("hello world", Utils::trim("hello world"));
+	EXPECT_TRUE(Utils::isNamespacedIdentifier("test"));
+	EXPECT_TRUE(Utils::isNamespacedIdentifier("t0-_est"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("_t0-_EST"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("-t0-_EST"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("0t-_EST"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("invalid key"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("_A"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier(""));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier(":"));
+	EXPECT_TRUE(Utils::isNamespacedIdentifier("test:a"));
+	EXPECT_TRUE(Utils::isNamespacedIdentifier("t0-_est:b"));
+	EXPECT_TRUE(Utils::isNamespacedIdentifier("test:test"));
+	EXPECT_TRUE(Utils::isNamespacedIdentifier("t0-_est:t0-_est"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("test:_A"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("test::a"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier(":test"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("t0-_est:_t0-_EST"));
+	EXPECT_FALSE(Utils::isNamespacedIdentifier("t0-_est: b"));
 }
 
+
 TEST(Utils, split)
 {
 	ASSERT_EQ(std::vector<std::string>({"ab"}), Utils::split("ab", '.'));
@@ -73,5 +91,23 @@ TEST(Utils, extractFileExtension)
 	ASSERT_EQ("ext", Utils::extractFileExtension("foo.bar/test.EXT"));
 }
 
+TEST(Utils, startsWith)
+{
+	ASSERT_TRUE(Utils::startsWith("foobar", "foo"));
+	ASSERT_TRUE(Utils::startsWith("foo", "foo"));
+	ASSERT_FALSE(Utils::startsWith("foo", "foobar"));
+	ASSERT_FALSE(Utils::startsWith("foobar", "bar"));
+	ASSERT_TRUE(Utils::startsWith("foo", ""));
+}
+
+TEST(Utils, endsWith)
+{
+	ASSERT_FALSE(Utils::endsWith("foobar", "foo"));
+	ASSERT_TRUE(Utils::endsWith("foo", "foo"));
+	ASSERT_FALSE(Utils::endsWith("foo", "foobar"));
+	ASSERT_TRUE(Utils::endsWith("foobar", "bar"));
+	ASSERT_TRUE(Utils::endsWith("foo", ""));
+}
+
 }
 
diff --git a/test/core/model/DomainTest.cpp b/test/core/model/DomainTest.cpp
index 8fcbdf2..4cb4331 100644
--- a/test/core/model/DomainTest.cpp
+++ b/test/core/model/DomainTest.cpp
@@ -242,7 +242,7 @@ TEST(Descriptor, getDefaultFields)
 	    A->createPrimitiveFieldDescriptor(sys->getStringType(), logger);
 	// now we should find that.
 	auto fields = A->getDefaultFields();
-	ASSERT_EQ(1, fields.size());
+	ASSERT_EQ(1U, fields.size());
 	ASSERT_EQ(A_prim_field, fields[0]);
 
 	// remove that field from A and add it to another class.
@@ -258,7 +258,7 @@ TEST(Descriptor, getDefaultFields)
 	// but we should find it again if we set B as superclass of A.
 	A->setSuperclass(B, logger);
 	fields = A->getDefaultFields();
-	ASSERT_EQ(1, fields.size());
+	ASSERT_EQ(1U, fields.size());
 	ASSERT_EQ(A_prim_field, fields[0]);
 
 	// and we should not be able to find it if we override the field.
@@ -277,7 +277,7 @@ TEST(Descriptor, getDefaultFields)
 
 	// now we should find that.
 	fields = A->getDefaultFields();
-	ASSERT_EQ(1, fields.size());
+	ASSERT_EQ(1U, fields.size());
 	ASSERT_EQ(C_field, fields[0]);
 
 	// add another transparent child class to A with a daughter class that has
@@ -296,7 +296,7 @@ TEST(Descriptor, getDefaultFields)
 
 	// now we should find both primitive fields, but the C field first.
 	fields = A->getDefaultFields();
-	ASSERT_EQ(2, fields.size());
+	ASSERT_EQ(2U, fields.size());
 	ASSERT_EQ(C_field, fields[0]);
 	ASSERT_EQ(F_field, fields[1]);
 }
@@ -321,7 +321,7 @@ TEST(Descriptor, getPermittedChildren)
 	 * in between.
 	 */
 	NodeVector<StructuredClass> children = book->getPermittedChildren();
-	ASSERT_EQ(3, children.size());
+	ASSERT_EQ(3U, children.size());
 	ASSERT_EQ(section, children[0]);
 	ASSERT_EQ(paragraph, children[1]);
 	ASSERT_EQ(text, children[2]);
@@ -331,7 +331,7 @@ TEST(Descriptor, getPermittedChildren)
 	    mgr, "Subclass", domain, Cardinality::any(), text, true, false)};
 	// And that should be in the result list as well now.
 	children = book->getPermittedChildren();
-	ASSERT_EQ(4, children.size());
+	ASSERT_EQ(4U, children.size());
 	ASSERT_EQ(section, children[0]);
 	ASSERT_EQ(paragraph, children[1]);
 	ASSERT_EQ(text, children[2]);
diff --git a/test/core/parser/ParserStackTest.cpp b/test/core/parser/ParserStackTest.cpp
deleted file mode 100644
index 3a0decb..0000000
--- a/test/core/parser/ParserStackTest.cpp
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
-    Ousía
-    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <iostream>
-
-#include <gtest/gtest.h>
-
-#include <core/parser/ParserStack.hpp>
-#include <core/StandaloneEnvironment.hpp>
-
-namespace ousia {
-
-ConcreteLogger logger;
-
-static int startCount = 0;
-static int endCount = 0;
-static int dataCount = 0;
-
-class TestHandler : public Handler {
-public:
-	using Handler::Handler;
-
-	void start(Variant::mapType &args) override { startCount++; }
-
-	void end() override { endCount++; }
-
-	void data(const std::string &data, int field) override { dataCount++; }
-
-	static Handler *create(const HandlerData &data)
-	{
-		return new TestHandler(data);
-	}
-};
-
-namespace ParserStates {
-static const ParserState Document =
-    ParserStateBuilder().parent(&None).elementHandler(TestHandler::create);
-static const ParserState Body = ParserStateBuilder()
-                                    .parent(&Document)
-                                    .elementHandler(TestHandler::create);
-static const ParserState Empty =
-    ParserStateBuilder().parent(&Document).elementHandler(TestHandler::create);
-static const ParserState Special =
-    ParserStateBuilder().parent(&All).elementHandler(TestHandler::create);
-static const ParserState Arguments =
-    ParserStateBuilder()
-        .parent(&None)
-        .elementHandler(TestHandler::create)
-        .arguments({Argument::Int("a"), Argument::String("b")});
-static const ParserState BodyChildren =
-    ParserStateBuilder()
-        .parent(&Body)
-        .elementHandler(TestHandler::create);
-
-static const std::multimap<std::string, const ParserState *> TestHandlers{
-    {"document", &Document},
-    {"body", &Body},
-    {"empty", &Empty},
-    {"special", &Special},
-    {"arguments", &Arguments},
-    {"*", &BodyChildren}};
-}
-
-TEST(ParserStack, simpleTest)
-{
-	StandaloneEnvironment env(logger);
-	ParserStack s{env.context, ParserStates::TestHandlers};
-
-	startCount = 0;
-	endCount = 0;
-	dataCount = 0;
-
-	EXPECT_EQ("", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::None, &s.currentState());
-
-	s.start("document", {});
-	s.data("test1");
-
-	EXPECT_EQ("document", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Document, &s.currentState());
-	EXPECT_EQ(1, startCount);
-	EXPECT_EQ(1, dataCount);
-
-	s.start("body", {});
-	s.data("test2");
-	EXPECT_EQ("body", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Body, &s.currentState());
-	EXPECT_EQ(2, startCount);
-	EXPECT_EQ(2, dataCount);
-
-	s.start("inner", {});
-	EXPECT_EQ("inner", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::BodyChildren, &s.currentState());
-	s.end();
-	EXPECT_EQ(3, startCount);
-	EXPECT_EQ(1, endCount);
-
-	s.end();
-	EXPECT_EQ(2, endCount);
-
-	EXPECT_EQ("document", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
-	s.start("body", {});
-	s.data("test3");
-	EXPECT_EQ("body", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Body, &s.currentState());
-	s.end();
-	EXPECT_EQ(4, startCount);
-	EXPECT_EQ(3, dataCount);
-	EXPECT_EQ(3, endCount);
-
-	EXPECT_EQ("document", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::Document, &s.currentState());
-
-	s.end();
-	EXPECT_EQ(4, endCount);
-
-	EXPECT_EQ("", s.currentCommandName());
-	EXPECT_EQ(&ParserStates::None, &s.currentState());
-}
-
-TEST(ParserStack, errorHandling)
-{
-	StandaloneEnvironment env(logger);
-	ParserStack s{env.context, ParserStates::TestHandlers};
-
-	EXPECT_THROW(s.start("body", {}), OusiaException);
-	s.start("document", {});
-	EXPECT_THROW(s.start("document", {}), OusiaException);
-	s.start("empty", {});
-	EXPECT_THROW(s.start("body", {}), OusiaException);
-	s.start("special", {});
-	s.end();
-	s.end();
-	s.end();
-	EXPECT_EQ(&ParserStates::None, &s.currentState());
-	ASSERT_THROW(s.end(), OusiaException);
-	ASSERT_THROW(s.data("test", 1), OusiaException);
-}
-
-TEST(ParserStack, validation)
-{
-	StandaloneEnvironment env(logger);
-	ParserStack s{env.context, ParserStates::TestHandlers};
-
-	logger.reset();
-	s.start("arguments", {});
-	EXPECT_TRUE(logger.hasError());
-	s.end();
-
-	s.start("arguments", {{"a", 5}});
-	EXPECT_TRUE(logger.hasError());
-	s.end();
-
-	logger.reset();
-	s.start("arguments", {{"a", 5}, {"b", "test"}});
-	EXPECT_FALSE(logger.hasError());
-	s.end();
-}
-}
-
diff --git a/test/core/parser/ParserStateTest.cpp b/test/core/parser/ParserStateTest.cpp
deleted file mode 100644
index 91d8dcd..0000000
--- a/test/core/parser/ParserStateTest.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
-    Ousía
-    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <gtest/gtest.h>
-
-#include <core/common/Rtti.hpp>
-#include <core/parser/ParserState.hpp>
-
-namespace ousia {
-
-static const Rtti t1;
-static const Rtti t2;
-static const Rtti t3;
-static const Rtti t4;
-static const Rtti t5;
-
-static const ParserState s1 = ParserStateBuilder().createdNodeType(&t1);
-static const ParserState s2a =
-    ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s2b =
-    ParserStateBuilder().parent(&s1).createdNodeType(&t2);
-static const ParserState s3 =
-    ParserStateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
-static const ParserState s4 =
-    ParserStateBuilder().parent(&s3).createdNodeType(&t4);
-static const ParserState s5 =
-    ParserStateBuilder().parent(&s2b).createdNodeType(&t5);
-
-TEST(ParserStateDeductor, deduce)
-{
-	using Result = std::vector<const ParserState *>;
-	using Signature = std::vector<const Rtti *>;
-	std::vector<const ParserState *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
-
-	// Should not crash on empty signature
-	ASSERT_EQ(Result{}, ParserStateDeductor(Signature{}, states).deduce());
-
-	// Try repeating signature elements
-	ASSERT_EQ(Result({&s1}),
-	          ParserStateDeductor(Signature({&t1}), states).deduce());
-	ASSERT_EQ(Result({&s1}),
-	          ParserStateDeductor(Signature({&t1, &t1}), states).deduce());
-	ASSERT_EQ(Result({&s1}),
-	          ParserStateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
-
-	// Go to another state
-	ASSERT_EQ(Result({&s2a, &s2b}),
-	          ParserStateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
-	ASSERT_EQ(Result({&s4}),
-	          ParserStateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
-
-	// Skip one state
-	ASSERT_EQ(Result({&s4}),
-	          ParserStateDeductor(Signature({&t2, &t4}), states).deduce());
-
-	// Impossible signature
-	ASSERT_EQ(Result({}),
-	          ParserStateDeductor(Signature({&t4, &t5}), states).deduce());
-
-}
-}
-
diff --git a/test/core/parser/stack/StackTest.cpp b/test/core/parser/stack/StackTest.cpp
new file mode 100644
index 0000000..321d471
--- /dev/null
+++ b/test/core/parser/stack/StackTest.cpp
@@ -0,0 +1,666 @@
+/*
+    Ousía
+    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <iostream>
+
+#include <gtest/gtest.h>
+
+#include <core/frontend/TerminalLogger.hpp>
+#include <core/parser/stack/Handler.hpp>
+#include <core/parser/stack/Stack.hpp>
+#include <core/parser/stack/State.hpp>
+
+#include <core/StandaloneEnvironment.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+// Build an instance of the StandaloneEnvironment used for this unit test
+static TerminalLogger logger(std::cerr, true);
+// static ConcreteLogger logger;
+static StandaloneEnvironment env(logger);
+
+namespace {
+
+struct Tracker {
+	int startCount;
+	int endCount;
+	int fieldStartCount;
+	int fieldEndCount;
+	int annotationStartCount;
+	int annotationEndCount;
+	int dataCount;
+
+	Variant::mapType startArgs;
+	bool fieldStartIsDefault;
+	size_t fieldStartIdx;
+	Variant annotationStartClassName;
+	Variant::mapType annotationStartArgs;
+	Variant annotationEndClassName;
+	Variant annotationEndElementName;
+	Variant dataData;
+
+	bool startResult;
+	bool fieldStartSetIsDefault;
+	bool fieldStartResult;
+	bool annotationStartResult;
+	bool annotationEndResult;
+	bool dataResult;
+
+	Tracker() { reset(); }
+
+	void reset()
+	{
+		startCount = 0;
+		endCount = 0;
+		fieldStartCount = 0;
+		fieldEndCount = 0;
+		annotationStartCount = 0;
+		annotationEndCount = 0;
+		dataCount = 0;
+
+		startArgs = Variant::mapType{};
+		fieldStartIsDefault = false;
+		fieldStartIdx = 0;
+		annotationStartClassName = Variant::fromString(std::string{});
+		annotationStartArgs = Variant::mapType{};
+		annotationEndClassName = Variant::fromString(std::string{});
+		annotationEndElementName = Variant::fromString(std::string{});
+		dataData = Variant::fromString(std::string{});
+
+		startResult = true;
+		fieldStartSetIsDefault = false;
+		fieldStartResult = true;
+		annotationStartResult = true;
+		annotationEndResult = true;
+		dataResult = true;
+	}
+
+	void expect(int startCount, int endCount, int fieldStartCount,
+	            int fieldEndCount, int annotationStartCount,
+	            int annotationEndCount, int dataCount)
+	{
+		EXPECT_EQ(startCount, this->startCount);
+		EXPECT_EQ(endCount, this->endCount);
+		EXPECT_EQ(fieldStartCount, this->fieldStartCount);
+		EXPECT_EQ(fieldEndCount, this->fieldEndCount);
+		EXPECT_EQ(annotationStartCount, this->annotationStartCount);
+		EXPECT_EQ(annotationEndCount, this->annotationEndCount);
+		EXPECT_EQ(dataCount, this->dataCount);
+	}
+};
+
+static Tracker tracker;
+
+class TestHandler : public Handler {
+private:
+	TestHandler(const HandlerData &handlerData) : Handler(handlerData) {}
+
+public:
+	bool start(Variant::mapType &args) override
+	{
+		tracker.startCount++;
+		tracker.startArgs = args;
+		if (!tracker.startResult) {
+			logger().error(
+			    "The TestHandler was told not to allow a field start. So it "
+			    "doesn't. The TestHandler always obeys its master.");
+		}
+		return tracker.startResult;
+	}
+
+	void end() override { tracker.endCount++; }
+
+	bool fieldStart(bool &isDefault, size_t fieldIdx) override
+	{
+		tracker.fieldStartCount++;
+		tracker.fieldStartIsDefault = isDefault;
+		tracker.fieldStartIdx = fieldIdx;
+		if (tracker.fieldStartSetIsDefault) {
+			isDefault = true;
+		}
+		return tracker.fieldStartResult;
+	}
+
+	void fieldEnd() override { tracker.fieldEndCount++; }
+
+	bool annotationStart(const Variant &className,
+	                     Variant::mapType &args) override
+	{
+		tracker.annotationStartCount++;
+		tracker.annotationStartClassName = className;
+		tracker.annotationStartArgs = args;
+		return tracker.annotationStartResult;
+	}
+
+	bool annotationEnd(const Variant &className,
+	                   const Variant &elementName) override
+	{
+		tracker.annotationEndCount++;
+		tracker.annotationEndClassName = className;
+		tracker.annotationEndElementName = elementName;
+		return tracker.annotationEndResult;
+	}
+
+	bool data(Variant &data) override
+	{
+		tracker.dataCount++;
+		tracker.dataData = data;
+		return tracker.dataResult;
+	}
+
+	static Handler *create(const HandlerData &handlerData)
+	{
+		return new TestHandler(handlerData);
+	}
+};
+}
+
+namespace States {
+static const State Document =
+    StateBuilder().parent(&None).elementHandler(TestHandler::create);
+static const State Body =
+    StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Empty =
+    StateBuilder().parent(&Document).elementHandler(TestHandler::create);
+static const State Special =
+    StateBuilder().parent(&All).elementHandler(TestHandler::create);
+static const State Arguments =
+    StateBuilder().parent(&None).elementHandler(TestHandler::create).arguments(
+        {Argument::Int("a"), Argument::String("b")});
+static const State BodyChildren =
+    StateBuilder().parent(&Body).elementHandler(TestHandler::create);
+static const State Any =
+    StateBuilder().parents({&None, &Any}).elementHandler(TestHandler::create);
+
+static const std::multimap<std::string, const State *> TestHandlers{
+    {"document", &Document},
+    {"body", &Body},
+    {"empty", &Empty},
+    {"special", &Special},
+    {"arguments", &Arguments},
+    {"*", &BodyChildren}};
+
+static const std::multimap<std::string, const State *> AnyHandlers{{"*", &Any}};
+}
+
+TEST(Stack, basicTest)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::TestHandlers};
+
+		EXPECT_EQ("", s.currentCommandName());
+		EXPECT_EQ(&States::None, &s.currentState());
+
+		s.command("document", {});
+		s.fieldStart(true);
+		s.data("test1");
+
+		EXPECT_EQ("document", s.currentCommandName());
+		EXPECT_EQ(&States::Document, &s.currentState());
+		tracker.expect(1, 0, 1, 0, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("body", {});
+		s.fieldStart(true);
+		s.data("test2");
+		EXPECT_EQ("body", s.currentCommandName());
+		EXPECT_EQ(&States::Body, &s.currentState());
+		tracker.expect(2, 0, 2, 0, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("inner", {});
+		s.fieldStart(true);
+		EXPECT_EQ("inner", s.currentCommandName());
+		EXPECT_EQ(&States::BodyChildren, &s.currentState());
+
+		s.fieldEnd();
+		tracker.expect(3, 1, 3, 1, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldEnd();
+		EXPECT_EQ("document", s.currentCommandName());
+		EXPECT_EQ(&States::Document, &s.currentState());
+		tracker.expect(3, 2, 3, 2, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("body", {});
+		s.fieldStart(true);
+		s.data("test3");
+		EXPECT_EQ("body", s.currentCommandName());
+		EXPECT_EQ(&States::Body, &s.currentState());
+		s.fieldEnd();
+		tracker.expect(4, 3, 4, 3, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+
+		EXPECT_EQ("document", s.currentCommandName());
+		EXPECT_EQ(&States::Document, &s.currentState());
+
+		s.fieldEnd();
+		tracker.expect(4, 4, 4, 4, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+
+		EXPECT_EQ("", s.currentCommandName());
+		EXPECT_EQ(&States::None, &s.currentState());
+	}
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidCommands)
+{
+	Stack s{env.context, States::TestHandlers};
+	tracker.reset();
+	EXPECT_THROW(s.command("body", {}), LoggableException);
+	s.command("document", {});
+	s.fieldStart(true);
+	EXPECT_THROW(s.command("document", {}), LoggableException);
+	s.command("empty", {});
+	s.fieldStart(true);
+	EXPECT_THROW(s.command("body", {}), LoggableException);
+	s.command("special", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	s.fieldEnd();
+	s.fieldEnd();
+	EXPECT_EQ(&States::None, &s.currentState());
+	ASSERT_THROW(s.fieldEnd(), LoggableException);
+	ASSERT_THROW(s.data("test"), LoggableException);
+}
+
+TEST(Stack, validation)
+{
+	Stack s{env.context, States::TestHandlers};
+	tracker.reset();
+	logger.reset();
+
+	s.command("arguments", {});
+	EXPECT_TRUE(logger.hasError());
+	s.fieldStart(true);
+	s.fieldEnd();
+
+	logger.reset();
+	s.command("arguments", {{"a", 5}});
+	EXPECT_TRUE(logger.hasError());
+	s.fieldStart(true);
+	s.fieldEnd();
+
+	logger.reset();
+	s.command("arguments", {{"a", 5}, {"b", "test"}});
+	EXPECT_FALSE(logger.hasError());
+	s.fieldStart(true);
+	s.fieldEnd();
+}
+
+TEST(Stack, invalidCommandName)
+{
+	Stack s{env.context, States::AnyHandlers};
+	tracker.reset();
+	logger.reset();
+
+	s.command("a", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+	s.command("a_", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	tracker.expect(2, 2, 2, 2, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+	s.command("a_:b", {});
+	s.fieldStart(true);
+	s.fieldEnd();
+	tracker.expect(3, 3, 3, 3, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+	ASSERT_THROW(s.command("_a", {}), LoggableException);
+	ASSERT_THROW(s.command("a:", {}), LoggableException);
+	ASSERT_THROW(s.command("a:_b", {}), LoggableException);
+	tracker.expect(3, 3, 3, 3, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, multipleFields)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {{"a", false}});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("a", s.currentCommandName());
+		EXPECT_EQ(Variant::mapType({{"a", false}}), tracker.startArgs);
+
+		s.fieldStart(false);
+		tracker.expect(1, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_FALSE(tracker.fieldStartIsDefault);
+		EXPECT_EQ(0U, tracker.fieldStartIdx);
+
+		s.data("test");
+		tracker.expect(1, 0, 1, 0, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("test", tracker.dataData);
+
+		s.fieldEnd();
+		tracker.expect(1, 0, 1, 1, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		tracker.expect(1, 0, 2, 1, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_FALSE(tracker.fieldStartIsDefault);
+		EXPECT_EQ(1U, tracker.fieldStartIdx);
+
+		s.data("test2");
+		tracker.expect(1, 0, 2, 1, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("test2", tracker.dataData);
+
+		s.fieldEnd();
+		tracker.expect(1, 0, 2, 2, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(true);
+		tracker.expect(1, 0, 3, 2, 0, 0, 2);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_TRUE(tracker.fieldStartIsDefault);
+		EXPECT_EQ(2U, tracker.fieldStartIdx);
+
+		s.data("test3");
+		tracker.expect(1, 0, 3, 2, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+		EXPECT_EQ("test3", tracker.dataData);
+
+		s.fieldEnd();
+		tracker.expect(1, 1, 3, 3, 0, 0, 3);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommand)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.command("b", {});
+		tracker.expect(2, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(2, 2, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnNewCommandWithExplicitDefaultField)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+
+		s.command("b", {});
+		tracker.expect(2, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+		s.fieldStart(true);
+		s.fieldEnd();
+		tracker.expect(2, 1, 2, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+	}
+	tracker.expect(2, 2, 2, 2, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldOnIncompatibleCommand)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+
+		tracker.fieldStartResult = false;
+		s.command("b", {});
+		tracker.expect(2, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+	}
+	tracker.expect(2, 2, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noImplicitDefaultFieldIfDefaultFieldGiven)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+		s.fieldStart(true);
+		tracker.expect(1, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+		s.fieldEnd();
+		tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("", s.currentCommandName());
+
+		s.command("b", {});
+		tracker.expect(2, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+	}
+	tracker.expect(2, 2, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, noEndIfStartFails)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("a", s.currentCommandName());
+
+		tracker.startResult = false;
+		s.command("b", {});
+		tracker.expect(3, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		ASSERT_EQ("b", s.currentCommandName());
+	}
+	tracker.expect(3, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_TRUE(logger.hasError());
+}
+
+TEST(Stack, implicitDefaultFieldOnData)
+{
+	tracker.reset();
+	logger.reset();
+	{
+		Stack s{env.context, States::AnyHandlers};
+
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.data("test");
+		tracker.expect(1, 0, 1, 0, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 1);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoFieldEnd)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, autoImplicitFieldEnd)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		s.command("b", {});
+		s.command("c", {});
+		s.command("d", {});
+		s.command("e", {});
+		s.fieldStart(true);
+		s.fieldEnd();
+		tracker.expect(5, 1, 5, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(5, 5, 5, 5, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, invalidDefaultField)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.fieldStartResult = false;
+		s.fieldStart(true);
+		s.fieldEnd();
+		tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	ASSERT_FALSE(logger.hasError());
+}
+
+TEST(Stack, errorInvalidDefaultFieldData)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.fieldStartResult = false;
+		s.fieldStart(true);
+		ASSERT_FALSE(logger.hasError());
+		s.data("test");
+		ASSERT_TRUE(logger.hasError());
+		s.fieldEnd();
+		tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorInvalidFieldData)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.fieldStartResult = false;
+		ASSERT_FALSE(logger.hasError());
+		s.fieldStart(false);
+		ASSERT_TRUE(logger.hasError());
+		s.data("test");
+		s.fieldEnd();
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorFieldStartNoCommand)
+{
+	tracker.reset();
+	logger.reset();
+
+	Stack s{env.context, States::AnyHandlers};
+	ASSERT_THROW(s.fieldStart(false), LoggableException);
+	ASSERT_THROW(s.fieldStart(true), LoggableException);
+	tracker.expect(0, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldStarts)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		ASSERT_FALSE(logger.hasError());
+		s.fieldStart(false);
+		ASSERT_TRUE(logger.hasError());
+		tracker.expect(1, 0, 1, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldEnd();
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorMutlipleFieldEnds)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		s.fieldEnd();
+		ASSERT_FALSE(logger.hasError());
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+		s.fieldEnd();
+		ASSERT_TRUE(logger.hasError());
+		tracker.expect(1, 0, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+	}
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+
+TEST(Stack, errorOpenField)
+{
+	tracker.reset();
+	logger.reset();
+
+	{
+		Stack s{env.context, States::AnyHandlers};
+		s.command("a", {});
+		tracker.expect(1, 0, 0, 0, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+
+		s.fieldStart(false);
+		ASSERT_FALSE(logger.hasError());
+	}
+	ASSERT_TRUE(logger.hasError());
+	tracker.expect(1, 1, 1, 1, 0, 0, 0);  // sc, ec, fsc, fse, asc, aec, dc
+}
+}
+}
+
diff --git a/test/core/parser/stack/StateTest.cpp b/test/core/parser/stack/StateTest.cpp
new file mode 100644
index 0000000..e503d30
--- /dev/null
+++ b/test/core/parser/stack/StateTest.cpp
@@ -0,0 +1,79 @@
+/*
+    Ousía
+    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/Rtti.hpp>
+#include <core/parser/stack/State.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+static const Rtti t1;
+static const Rtti t2;
+static const Rtti t3;
+static const Rtti t4;
+static const Rtti t5;
+
+static const State s1 = StateBuilder().createdNodeType(&t1);
+static const State s2a =
+    StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s2b =
+    StateBuilder().parent(&s1).createdNodeType(&t2);
+static const State s3 =
+    StateBuilder().parents({&s2a, &s1}).createdNodeType(&t3);
+static const State s4 =
+    StateBuilder().parent(&s3).createdNodeType(&t4);
+static const State s5 =
+    StateBuilder().parent(&s2b).createdNodeType(&t5);
+
+TEST(StateDeductor, deduce)
+{
+	using Result = std::vector<const State *>;
+	using Signature = std::vector<const Rtti *>;
+	std::vector<const State *> states{&s1, &s2a, &s2b, &s3, &s4, &s5};
+
+	// Should not crash on empty signature
+	ASSERT_EQ(Result{}, StateDeductor(Signature{}, states).deduce());
+
+	// Try repeating signature elements
+	ASSERT_EQ(Result({&s1}),
+	          StateDeductor(Signature({&t1}), states).deduce());
+	ASSERT_EQ(Result({&s1}),
+	          StateDeductor(Signature({&t1, &t1}), states).deduce());
+	ASSERT_EQ(Result({&s1}),
+	          StateDeductor(Signature({&t1, &t1, &t1}), states).deduce());
+
+	// Go to another state
+	ASSERT_EQ(Result({&s2a, &s2b}),
+	          StateDeductor(Signature({&t1, &t1, &t2}), states).deduce());
+	ASSERT_EQ(Result({&s4}),
+	          StateDeductor(Signature({&t1, &t3, &t4}), states).deduce());
+
+	// Skip one state
+	ASSERT_EQ(Result({&s4}),
+	          StateDeductor(Signature({&t2, &t4}), states).deduce());
+
+	// Impossible signature
+	ASSERT_EQ(Result({}),
+	          StateDeductor(Signature({&t4, &t5}), states).deduce());
+
+}
+}
+}
+
diff --git a/test/core/parser/utils/TokenTrieTest.cpp b/test/core/parser/utils/TokenTrieTest.cpp
new file mode 100644
index 0000000..087e6e6
--- /dev/null
+++ b/test/core/parser/utils/TokenTrieTest.cpp
@@ -0,0 +1,92 @@
+/*
+    Ousía
+    Copyright (C) 2014  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/parser/utils/TokenTrie.hpp>
+
+namespace ousia {
+
+static const TokenTypeId t1 = 0;
+static const TokenTypeId t2 = 1;
+static const TokenTypeId t3 = 2;
+static const TokenTypeId t4 = 3;
+
+TEST(TokenTrie, registerToken)
+{
+	TokenTrie tree;
+
+	ASSERT_TRUE(tree.registerToken("a", t1));
+	ASSERT_TRUE(tree.registerToken("ab", t2));
+	ASSERT_TRUE(tree.registerToken("b", t3));
+	ASSERT_TRUE(tree.registerToken("hello", t4));
+
+	ASSERT_FALSE(tree.registerToken("", t1));
+	ASSERT_FALSE(tree.registerToken("a", t4));
+	ASSERT_FALSE(tree.registerToken("ab", t4));
+	ASSERT_FALSE(tree.registerToken("b", t4));
+	ASSERT_FALSE(tree.registerToken("hello", t4));
+
+	ASSERT_EQ(t1, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(t3, tree.hasToken("b"));
+	ASSERT_EQ(t4, tree.hasToken("hello"));
+	ASSERT_EQ(EmptyToken, tree.hasToken(""));
+	ASSERT_EQ(EmptyToken, tree.hasToken("abc"));
+}
+
+TEST(TokenTrie, unregisterToken)
+{
+	TokenTrie tree;
+
+	ASSERT_TRUE(tree.registerToken("a", t1));
+	ASSERT_FALSE(tree.registerToken("a", t4));
+
+	ASSERT_TRUE(tree.registerToken("ab", t2));
+	ASSERT_FALSE(tree.registerToken("ab", t4));
+
+	ASSERT_TRUE(tree.registerToken("b", t3));
+	ASSERT_FALSE(tree.registerToken("b", t4));
+
+	ASSERT_EQ(t1, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(t3, tree.hasToken("b"));
+
+	ASSERT_TRUE(tree.unregisterToken("a"));
+	ASSERT_FALSE(tree.unregisterToken("a"));
+
+	ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(t3, tree.hasToken("b"));
+
+	ASSERT_TRUE(tree.unregisterToken("b"));
+	ASSERT_FALSE(tree.unregisterToken("b"));
+
+	ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+	ASSERT_EQ(t2, tree.hasToken("ab"));
+	ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+
+	ASSERT_TRUE(tree.unregisterToken("ab"));
+	ASSERT_FALSE(tree.unregisterToken("ab"));
+
+	ASSERT_EQ(EmptyToken, tree.hasToken("a"));
+	ASSERT_EQ(EmptyToken, tree.hasToken("ab"));
+	ASSERT_EQ(EmptyToken, tree.hasToken("b"));
+}
+}
+
diff --git a/test/core/parser/utils/TokenizerTest.cpp b/test/core/parser/utils/TokenizerTest.cpp
new file mode 100644
index 0000000..8565057
--- /dev/null
+++ b/test/core/parser/utils/TokenizerTest.cpp
@@ -0,0 +1,414 @@
+/*
+    Ousía
+    Copyright (C) 2014  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/common/CharReader.hpp>
+#include <core/parser/utils/Tokenizer.hpp>
+
+namespace ousia {
+
+TEST(Tokenizer, tokenRegistration)
+{
+	Tokenizer tokenizer;
+
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken(""));
+
+	ASSERT_EQ(0U, tokenizer.registerToken("a"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("a"));
+	ASSERT_EQ("a", tokenizer.getTokenString(0U));
+
+	ASSERT_EQ(1U, tokenizer.registerToken("b"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("b"));
+	ASSERT_EQ("b", tokenizer.getTokenString(1U));
+
+	ASSERT_EQ(2U, tokenizer.registerToken("c"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("c"));
+	ASSERT_EQ("c", tokenizer.getTokenString(2U));
+
+	ASSERT_TRUE(tokenizer.unregisterToken(1U));
+	ASSERT_FALSE(tokenizer.unregisterToken(1U));
+	ASSERT_EQ("", tokenizer.getTokenString(1U));
+
+	ASSERT_EQ(1U, tokenizer.registerToken("d"));
+	ASSERT_EQ(EmptyToken, tokenizer.registerToken("d"));
+	ASSERT_EQ("d", tokenizer.getTokenString(1U));
+}
+
+TEST(Tokenizer, textTokenPreserveWhitespace)
+{
+	{
+		CharReader reader{" this \t is only a  \n\n test   text   "};
+		//                 012345 6789012345678 9 0123456789012345
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ(" this \t is only a  \n\n test   text   ", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(36U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+
+	{
+		CharReader reader{"this \t is only a  \n\n test   text"};
+		//                 01234 5678901234567 8 9012345678901
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::PRESERVE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this \t is only a  \n\n test   text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(32U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+}
+
+TEST(Tokenizer, textTokenTrimWhitespace)
+{
+	{
+		CharReader reader{" this \t is only a  \n\n test   text   "};
+		//                 012345 6789012345678 9 0123456789012345
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this \t is only a  \n\n test   text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(33U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+
+	{
+		CharReader reader{"this \t is only a  \n\n test   text"};
+		//                 01234 5678901234567 8 9012345678901
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::TRIM};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this \t is only a  \n\n test   text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(32U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+}
+
+TEST(Tokenizer, textTokenCollapseWhitespace)
+{
+	{
+		CharReader reader{" this \t is only a  \n\n test   text   "};
+		//                 012345 6789012345678 9 0123456789012345
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this is only a test text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(33U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+
+	{
+		CharReader reader{"this \t is only a  \n\n test   text"};
+		//                 01234 5678901234567 8 9012345678901
+		//                 0          1           2         3
+		Tokenizer tokenizer{WhitespaceMode::COLLAPSE};
+
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("this is only a test text", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(32U, loc.getEnd());
+
+		ASSERT_FALSE(tokenizer.read(reader, token));
+	}
+}
+
+TEST(Tokenizer, simpleReadToken)
+{
+	CharReader reader{"test1:test2"};
+	Tokenizer tokenizer;
+
+	const TokenTypeId tid = tokenizer.registerToken(":");
+	ASSERT_EQ(0U, tid);
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test1", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
+
+		char c;
+		ASSERT_TRUE(reader.peek(c));
+		ASSERT_EQ(':', c);
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(tid, token.type);
+		ASSERT_EQ(":", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(5U, loc.getStart());
+		ASSERT_EQ(6U, loc.getEnd());
+
+		char c;
+		ASSERT_TRUE(reader.peek(c));
+		ASSERT_EQ('t', c);
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test2", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(6U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
+
+		char c;
+		ASSERT_FALSE(reader.peek(c));
+	}
+}
+
+TEST(Tokenizer, simplePeekToken)
+{
+	CharReader reader{"test1:test2"};
+	Tokenizer tokenizer;
+
+	const TokenTypeId tid = tokenizer.registerToken(":");
+	ASSERT_EQ(0U, tid);
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.peek(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test1", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
+		ASSERT_EQ(0U, reader.getOffset());
+		ASSERT_EQ(5U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.peek(reader, token));
+
+		ASSERT_EQ(tid, token.type);
+		ASSERT_EQ(":", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(5U, loc.getStart());
+		ASSERT_EQ(6U, loc.getEnd());
+		ASSERT_EQ(0U, reader.getOffset());
+		ASSERT_EQ(6U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.peek(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test2", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(6U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
+		ASSERT_EQ(0U, reader.getOffset());
+		ASSERT_EQ(11U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test1", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
+		ASSERT_EQ(5U, reader.getOffset());
+		ASSERT_EQ(5U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(tid, token.type);
+		ASSERT_EQ(":", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(5U, loc.getStart());
+		ASSERT_EQ(6U, loc.getEnd());
+		ASSERT_EQ(6U, reader.getOffset());
+		ASSERT_EQ(6U, reader.getPeekOffset());
+	}
+
+	{
+		Token token;
+		ASSERT_TRUE(tokenizer.read(reader, token));
+
+		ASSERT_EQ(TextToken, token.type);
+		ASSERT_EQ("test2", token.content);
+
+		SourceLocation loc = token.location;
+		ASSERT_EQ(6U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
+		ASSERT_EQ(11U, reader.getOffset());
+		ASSERT_EQ(11U, reader.getPeekOffset());
+	}
+}
+
+TEST(Tokenizer, ambiguousTokens)
+{
+	CharReader reader{"abc"};
+	Tokenizer tokenizer;
+
+	TokenTypeId t1 = tokenizer.registerToken("abd");
+	TokenTypeId t2 = tokenizer.registerToken("bc");
+
+	ASSERT_EQ(0U, t1);
+	ASSERT_EQ(1U, t2);
+
+	Token token;
+	ASSERT_TRUE(tokenizer.read(reader, token));
+
+	ASSERT_EQ(TextToken, token.type);
+	ASSERT_EQ("a", token.content);
+
+	SourceLocation loc = token.location;
+	ASSERT_EQ(0U, loc.getStart());
+	ASSERT_EQ(1U, loc.getEnd());
+
+	ASSERT_TRUE(tokenizer.read(reader, token));
+
+	ASSERT_EQ(t2, token.type);
+	ASSERT_EQ("bc", token.content);
+
+	loc = token.location;
+	ASSERT_EQ(1U, loc.getStart());
+	ASSERT_EQ(3U, loc.getEnd());
+
+	ASSERT_FALSE(tokenizer.read(reader, token));
+}
+
+TEST(Tokenizer, commentTestWhitespacePreserve)
+{
+	CharReader reader{"Test/Test /* Block Comment */", 0};
+	//                 012345678901234567890123456789
+	//                 0        1         2
+	Tokenizer tokenizer(WhitespaceMode::PRESERVE);
+
+	const TokenTypeId t1 = tokenizer.registerToken("/");
+	const TokenTypeId t2 = tokenizer.registerToken("/*");
+	const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+	std::vector<Token> expected = {
+	    {TextToken, "Test", SourceLocation{0, 0, 4}},
+	    {t1, "/", SourceLocation{0, 4, 5}},
+	    {TextToken, "Test ", SourceLocation{0, 5, 10}},
+	    {t2, "/*", SourceLocation{0, 10, 12}},
+	    {TextToken, " Block Comment ", SourceLocation{0, 12, 27}},
+	    {t3, "*/", SourceLocation{0, 27, 29}}};
+
+	Token t;
+	for (auto &te : expected) {
+		EXPECT_TRUE(tokenizer.read(reader, t));
+		EXPECT_EQ(te.type, t.type);
+		EXPECT_EQ(te.content, t.content);
+		EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+		EXPECT_EQ(te.location.getStart(), t.location.getStart());
+		EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+	}
+	ASSERT_FALSE(tokenizer.read(reader, t));
+}
+
+TEST(Tokenizer, commentTestWhitespaceCollapse)
+{
+	CharReader reader{"Test/Test /* Block Comment */", 0};
+	//                 012345678901234567890123456789
+	//                 0        1         2
+	Tokenizer tokenizer(WhitespaceMode::COLLAPSE);
+
+	const TokenTypeId t1 = tokenizer.registerToken("/");
+	const TokenTypeId t2 = tokenizer.registerToken("/*");
+	const TokenTypeId t3 = tokenizer.registerToken("*/");
+
+	std::vector<Token> expected = {
+	    {TextToken, "Test", SourceLocation{0, 0, 4}},
+	    {t1, "/", SourceLocation{0, 4, 5}},
+	    {TextToken, "Test", SourceLocation{0, 5, 9}},
+	    {t2, "/*", SourceLocation{0, 10, 12}},
+	    {TextToken, "Block Comment", SourceLocation{0, 13, 26}},
+	    {t3, "*/", SourceLocation{0, 27, 29}}};
+
+	Token t;
+	for (auto &te : expected) {
+		EXPECT_TRUE(tokenizer.read(reader, t));
+		EXPECT_EQ(te.type, t.type);
+		EXPECT_EQ(te.content, t.content);
+		EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId());
+		EXPECT_EQ(te.location.getStart(), t.location.getStart());
+		EXPECT_EQ(te.location.getEnd(), t.location.getEnd());
+	}
+	ASSERT_FALSE(tokenizer.read(reader, t));
+}
+}
+