Added user defined tokens test

1 files changed, 568 insertions, 536 deletions

diff --git a/test/formats/osml/OsmlStreamParserTest.cpp b/test/formats/osml/OsmlStreamParserTest.cpp
index 0ea087f..d47f529 100644
--- a/test/formats/osml/OsmlStreamParserTest.cpp
+++ b/test/formats/osml/OsmlStreamParserTest.cpp
@@ -24,17 +24,17 @@
 #include <core/common/Variant.hpp>
 #include <core/frontend/TerminalLogger.hpp>
 #include <core/parser/utils/TokenizedData.hpp>
-
 #include <formats/osml/OsmlStreamParser.hpp>
 
+#include <core/parser/utils/TokenizedDataTestUtils.hpp>
+
 namespace ousia {
 
 static TerminalLogger logger(std::cerr, true);
 // static ConcreteLogger logger;
 
 static void assertCommandStart(OsmlStreamParser &parser,
-                               const std::string &name,
-                               bool rangeCommand,
+                               const std::string &name, bool rangeCommand,
                                SourceOffset start = InvalidSourceOffset,
                                SourceOffset end = InvalidSourceOffset)
 {
@@ -52,8 +52,7 @@ static void assertCommandStart(OsmlStreamParser &parser,
 }
 
 static void assertCommandStart(OsmlStreamParser &parser,
-                               const std::string &name,
-                               bool rangeCommand,
+                               const std::string &name, bool rangeCommand,
                                const Variant::mapType &args,
                                SourceOffset start = InvalidSourceOffset,
                                SourceOffset end = InvalidSourceOffset)
@@ -62,17 +61,16 @@ static void assertCommandStart(OsmlStreamParser &parser,
 	EXPECT_EQ(args, parser.getCommandArguments());
 }
 
-static void assertCommand(OsmlStreamParser &parser,
-                               const std::string &name,
-                               SourceOffset start = InvalidSourceOffset,
-                               SourceOffset end = InvalidSourceOffset)
+static void assertCommand(OsmlStreamParser &parser, const std::string &name,
+                          SourceOffset start = InvalidSourceOffset,
+                          SourceOffset end = InvalidSourceOffset)
 {
 	assertCommandStart(parser, name, false, Variant::mapType{}, start, end);
 }
 
 static void assertRangeEnd(OsmlStreamParser &parser,
-                             SourceOffset start = InvalidSourceOffset,
-                             SourceOffset end = InvalidSourceOffset)
+                           SourceOffset start = InvalidSourceOffset,
+                           SourceOffset end = InvalidSourceOffset)
 {
 	ASSERT_EQ(OsmlStreamParser::State::RANGE_END, parser.parse());
 	if (start != InvalidSourceOffset) {
@@ -116,11 +114,12 @@ static void assertTextData(OsmlStreamParser &parser, const std::string &text,
 }
 
 static void assertData(OsmlStreamParser &parser, const std::string &text,
-                           SourceOffset textStart = InvalidSourceOffset,
-                           SourceOffset textEnd = InvalidSourceOffset,
-                           WhitespaceMode mode = WhitespaceMode::COLLAPSE)
+                       SourceOffset textStart = InvalidSourceOffset,
+                       SourceOffset textEnd = InvalidSourceOffset,
+                       WhitespaceMode mode = WhitespaceMode::COLLAPSE)
 {
-	assertTextData(parser, text, InvalidSourceOffset, InvalidSourceOffset, textStart, textEnd, mode);
+	assertTextData(parser, text, InvalidSourceOffset, InvalidSourceOffset,
+	               textStart, textEnd, mode);
 }
 
 static void assertEmptyData(OsmlStreamParser &parser)
@@ -134,7 +133,6 @@ static void assertEmptyData(OsmlStreamParser &parser)
 	EXPECT_FALSE(dataReader.read(token, TokenSet{}, WhitespaceMode::TRIM));
 }
 
-
 static void assertFieldStart(OsmlStreamParser &parser, bool defaultField,
                              SourceOffset start = InvalidSourceOffset,
                              SourceOffset end = InvalidSourceOffset)
@@ -585,363 +583,363 @@ TEST(OsmlStreamParser, dataOutsideField)
 
 TEST(OsmlStreamParser, nestedCommand)
 {
-    const char *testString = "\\test{a}{\\test2{b} c} d";
-    //                         012345678 90123456789012
-    //                         0          1         2
-    CharReader charReader(testString);
-    OsmlStreamParser parser(charReader, logger);
+	const char *testString = "\\test{a}{\\test2{b} c} d";
+	//                         012345678 90123456789012
+	//                         0          1         2
+	CharReader charReader(testString);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommand(parser, "test", 0, 5);
-    assertFieldStart(parser, false, 5, 6);
-    assertData(parser, "a", 6, 7);
-    assertFieldEnd(parser, 7, 8);
+	assertCommand(parser, "test", 0, 5);
+	assertFieldStart(parser, false, 5, 6);
+	assertData(parser, "a", 6, 7);
+	assertFieldEnd(parser, 7, 8);
 
-    assertFieldStart(parser, false, 8, 9);
-    assertCommand(parser, "test2", 9, 15);
-    assertFieldStart(parser, false, 15, 16);
-    assertData(parser, "b", 16, 17);
-    assertFieldEnd(parser, 17, 18);
-    assertData(parser, "c", 19, 20);
-    assertFieldEnd(parser, 20, 21);
-    assertData(parser, "d", 22, 23);
-    assertEnd(parser, 23, 23);
+	assertFieldStart(parser, false, 8, 9);
+	assertCommand(parser, "test2", 9, 15);
+	assertFieldStart(parser, false, 15, 16);
+	assertData(parser, "b", 16, 17);
+	assertFieldEnd(parser, 17, 18);
+	assertData(parser, "c", 19, 20);
+	assertFieldEnd(parser, 20, 21);
+	assertData(parser, "d", 22, 23);
+	assertEnd(parser, 23, 23);
 }
 
-
 TEST(OsmlStreamParser, nestedCommandImmediateEnd)
 {
-    const char *testString = "\\test{\\test2{b}} d";
-    //                         012345 678901234567
-    //                         0          1
-    CharReader charReader(testString);
-    OsmlStreamParser parser(charReader, logger);
-
-    assertCommand(parser, "test", 0, 5);
-    assertFieldStart(parser, false, 5, 6);
-    {
-        assertCommand(parser, "test2", 6, 12);
-        assertFieldStart(parser, false, 12, 13);
-        assertData(parser, "b", 13, 14);
-        assertFieldEnd(parser, 14, 15);
-    }
-    assertFieldEnd(parser, 15, 16);
-    assertData(parser, "d", 17, 18);
-    assertEnd(parser, 18, 18);
+	const char *testString = "\\test{\\test2{b}} d";
+	//                         012345 678901234567
+	//                         0          1
+	CharReader charReader(testString);
+	OsmlStreamParser parser(charReader, logger);
+
+	assertCommand(parser, "test", 0, 5);
+	assertFieldStart(parser, false, 5, 6);
+	{
+		assertCommand(parser, "test2", 6, 12);
+		assertFieldStart(parser, false, 12, 13);
+		assertData(parser, "b", 13, 14);
+		assertFieldEnd(parser, 14, 15);
+	}
+	assertFieldEnd(parser, 15, 16);
+	assertData(parser, "d", 17, 18);
+	assertEnd(parser, 18, 18);
 }
 
 TEST(OsmlStreamParser, nestedCommandNoData)
 {
-    const char *testString = "\\test{\\test2}";
-    //                         012345 6789012
-    CharReader charReader(testString);
-    OsmlStreamParser parser(charReader, logger);
+	const char *testString = "\\test{\\test2}";
+	//                         012345 6789012
+	CharReader charReader(testString);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommand(parser, "test", 0, 5);
-    assertFieldStart(parser, false, 5, 6);
-    assertCommand(parser, "test2", 6, 12);
-    assertFieldEnd(parser, 12, 13);
-    assertEnd(parser, 13, 13);
+	assertCommand(parser, "test", 0, 5);
+	assertFieldStart(parser, false, 5, 6);
+	assertCommand(parser, "test2", 6, 12);
+	assertFieldEnd(parser, 12, 13);
+	assertEnd(parser, 13, 13);
 }
 
 TEST(OsmlStreamParser, multipleCommands)
 {
-    const char *testString = "\\a \\b \\c \\d";
-    //                         012 345 678 90
-    //                         0            1
-    CharReader charReader(testString);
-    OsmlStreamParser parser(charReader, logger);
+	const char *testString = "\\a \\b \\c \\d";
+	//                         012 345 678 90
+	//                         0            1
+	CharReader charReader(testString);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommand(parser, "a", 0, 2);
-    assertEmptyData(parser);
-    assertCommand(parser, "b", 3, 5);
-    assertEmptyData(parser);
-    assertCommand(parser, "c", 6, 8);
-    assertEmptyData(parser);
-    assertCommand(parser, "d", 9, 11);
-    assertEnd(parser, 11, 11);
+	assertCommand(parser, "a", 0, 2);
+	assertEmptyData(parser);
+	assertCommand(parser, "b", 3, 5);
+	assertEmptyData(parser);
+	assertCommand(parser, "c", 6, 8);
+	assertEmptyData(parser);
+	assertCommand(parser, "d", 9, 11);
+	assertEnd(parser, 11, 11);
 }
 
 TEST(OsmlStreamParser, fieldsWithSpaces)
 {
-    const char *testString = "\\a {\\b \\c}   \n\n {\\d}";
-    //                         0123 456 789012 3 456 789
-    //                         0           1
-    CharReader charReader(testString);
-    OsmlStreamParser parser(charReader, logger);
-
-    assertCommand(parser, "a", 0, 2);
-    assertEmptyData(parser);
-    assertFieldStart(parser, false, 3, 4);
-    assertCommand(parser, "b", 4, 6);
-    assertEmptyData(parser);
-    assertCommand(parser, "c", 7, 9);
-    assertFieldEnd(parser, 9, 10);
-    assertEmptyData(parser);
-    assertFieldStart(parser, false, 16, 17);
-    assertCommand(parser, "d", 17, 19);
-    assertFieldEnd(parser, 19, 20);
-    assertEnd(parser, 20, 20);
+	const char *testString = "\\a {\\b \\c}   \n\n {\\d}";
+	//                         0123 456 789012 3 456 789
+	//                         0           1
+	CharReader charReader(testString);
+	OsmlStreamParser parser(charReader, logger);
+
+	assertCommand(parser, "a", 0, 2);
+	assertEmptyData(parser);
+	assertFieldStart(parser, false, 3, 4);
+	assertCommand(parser, "b", 4, 6);
+	assertEmptyData(parser);
+	assertCommand(parser, "c", 7, 9);
+	assertFieldEnd(parser, 9, 10);
+	assertEmptyData(parser);
+	assertFieldStart(parser, false, 16, 17);
+	assertCommand(parser, "d", 17, 19);
+	assertFieldEnd(parser, 19, 20);
+	assertEnd(parser, 20, 20);
 }
 
 TEST(OsmlStreamParser, errorEndButOpenField)
 {
-    const char *testString = "\\a b {";
-    //                         012345
-    //                         0
-    CharReader charReader(testString);
+	const char *testString = "\\a b {";
+	//                         012345
+	//                         0
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    assertCommand(parser, "a", 0, 2);
-    assertData(parser, "b", 3, 4);
-    assertFieldStart(parser, false, 5, 6);
-    ASSERT_FALSE(logger.hasError());
-    assertEnd(parser, 6, 6);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	assertCommand(parser, "a", 0, 2);
+	assertData(parser, "b", 3, 4);
+	assertFieldStart(parser, false, 5, 6);
+	ASSERT_FALSE(logger.hasError());
+	assertEnd(parser, 6, 6);
+	ASSERT_TRUE(logger.hasError());
 }
 
-
 TEST(OsmlStreamParser, errorNoFieldToEnd)
 {
-    const char *testString = "\\a b }";
-    //                         012345
-    //                         0
-    CharReader charReader(testString);
+	const char *testString = "\\a b }";
+	//                         012345
+	//                         0
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    assertCommand(parser, "a", 0, 2);
-    assertData(parser, "b", 3, 4);
-    ASSERT_FALSE(logger.hasError());
-    assertEnd(parser, 6, 6);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	assertCommand(parser, "a", 0, 2);
+	assertData(parser, "b", 3, 4);
+	ASSERT_FALSE(logger.hasError());
+	assertEnd(parser, 6, 6);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorNoFieldEndNested)
 {
-    const char *testString = "\\test{\\test2{}}}";
-    //                         012345 6789012345
-    //                         0          1
-    CharReader charReader(testString);
+	const char *testString = "\\test{\\test2{}}}";
+	//                         012345 6789012345
+	//                         0          1
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    assertCommand(parser, "test", 0, 5);
-    assertFieldStart(parser, false, 5, 6);
-    assertCommand(parser, "test2", 6, 12);
-    assertFieldStart(parser, false, 12, 13);
-    assertFieldEnd(parser, 13, 14);
-    assertFieldEnd(parser, 14, 15);
-    ASSERT_FALSE(logger.hasError());
-    assertEnd(parser, 16, 16);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	assertCommand(parser, "test", 0, 5);
+	assertFieldStart(parser, false, 5, 6);
+	assertCommand(parser, "test2", 6, 12);
+	assertFieldStart(parser, false, 12, 13);
+	assertFieldEnd(parser, 13, 14);
+	assertFieldEnd(parser, 14, 15);
+	ASSERT_FALSE(logger.hasError());
+	assertEnd(parser, 16, 16);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorNoFieldEndNestedData)
 {
-    const char *testString = "\\test{\\test2{}}a}";
-    //                         012345 67890123456
-    //                         0          1
-    CharReader charReader(testString);
+	const char *testString = "\\test{\\test2{}}a}";
+	//                         012345 67890123456
+	//                         0          1
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    assertCommand(parser, "test", 0, 5);
-    assertFieldStart(parser, false, 5, 6);
-    assertCommand(parser, "test2", 6, 12);
-    assertFieldStart(parser, false, 12, 13);
-    assertFieldEnd(parser, 13, 14);
-    assertFieldEnd(parser, 14, 15);
-    assertData(parser, "a", 15, 16);
-    ASSERT_FALSE(logger.hasError());
-    assertEnd(parser, 17, 17);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	assertCommand(parser, "test", 0, 5);
+	assertFieldStart(parser, false, 5, 6);
+	assertCommand(parser, "test2", 6, 12);
+	assertFieldStart(parser, false, 12, 13);
+	assertFieldEnd(parser, 13, 14);
+	assertFieldEnd(parser, 14, 15);
+	assertData(parser, "a", 15, 16);
+	ASSERT_FALSE(logger.hasError());
+	assertEnd(parser, 17, 17);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, beginEnd)
 {
-    const char *testString = "\\begin{book}\\end{book}";
-    //                         012345678901 2345678901
-    //                         0         1          2
-    CharReader charReader(testString);
+	const char *testString = "\\begin{book}\\end{book}";
+	//                         012345678901 2345678901
+	//                         0         1          2
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
-    assertRangeEnd(parser, 17, 21);
-    assertEnd(parser, 22, 22);
+	assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
+	assertRangeEnd(parser, 17, 21);
+	assertEnd(parser, 22, 22);
 }
 
 TEST(OsmlStreamParser, beginEndWithName)
 {
-    const char *testString = "\\begin{book#a}\\end{book}";
-    //                         01234567890123 4567890123
-    //                         0         1          2
-    CharReader charReader(testString);
+	const char *testString = "\\begin{book#a}\\end{book}";
+	//                         01234567890123 4567890123
+	//                         0         1          2
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommandStart(parser, "book", true, {{"name", "a"}}, 7, 11);
-    assertRangeEnd(parser, 19, 23);
-    assertEnd(parser, 24, 24);
+	assertCommandStart(parser, "book", true, {{"name", "a"}}, 7, 11);
+	assertRangeEnd(parser, 19, 23);
+	assertEnd(parser, 24, 24);
 }
 
 TEST(OsmlStreamParser, beginEndWithNameAndArgs)
 {
-    const char *testString = "\\begin{book#a}[a=1,b=2,c=\"test\"]\\end{book}";
-    //                         0123456789012345678901234 56789 01 2345678901
-    //                         0         1         2           3          4
-    CharReader charReader(testString);
+	const char *testString = "\\begin{book#a}[a=1,b=2,c=\"test\"]\\end{book}";
+	//                         0123456789012345678901234 56789 01 2345678901
+	//                         0         1         2           3          4
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommandStart(parser, "book", true,
-                  {{"name", "a"}, {"a", 1}, {"b", 2}, {"c", "test"}}, 7, 11);
-    assertRangeEnd(parser, 37, 41);
-    assertEnd(parser, 42, 42);
+	assertCommandStart(parser, "book", true,
+	                   {{"name", "a"}, {"a", 1}, {"b", 2}, {"c", "test"}}, 7,
+	                   11);
+	assertRangeEnd(parser, 37, 41);
+	assertEnd(parser, 42, 42);
 }
 
 TEST(OsmlStreamParser, beginEndWithNameAndArgsMultipleFields)
 {
-    const char *testString =
-        "\\begin{book#a}[a=1,b=2,c=\"test\"]{a \\test}{b \\test{}}\\end{book}";
-    //    0123456789012345678901234 56789 01234 567890123 45678901 2345678901
-    //    0         1         2           3          4          5          6
-    CharReader charReader(testString);
-
-    OsmlStreamParser parser(charReader, logger);
-
-    assertCommandStart(parser, "book", true,
-                  {{"name", "a"}, {"a", 1}, {"b", 2}, {"c", "test"}}, 7, 11);
-    assertFieldStart(parser, false, 32, 33);
-    assertData(parser, "a", 33, 34);
-    assertCommand(parser, "test", 35, 40);
-    assertFieldEnd(parser, 40, 41);
-    assertFieldStart(parser, false, 41, 42);
-    assertData(parser, "b", 42, 43);
-    assertCommand(parser, "test", 44, 49);
-    assertFieldStart(parser, false, 49, 50);
-    assertFieldEnd(parser, 50, 51);
-    assertFieldEnd(parser, 51, 52);
-    assertRangeEnd(parser, 57, 61);
-    assertEnd(parser, 62, 62);
+	const char *testString =
+	    "\\begin{book#a}[a=1,b=2,c=\"test\"]{a \\test}{b \\test{}}\\end{book}";
+	//    0123456789012345678901234 56789 01234 567890123 45678901 2345678901
+	//    0         1         2           3          4          5          6
+	CharReader charReader(testString);
+
+	OsmlStreamParser parser(charReader, logger);
+
+	assertCommandStart(parser, "book", true,
+	                   {{"name", "a"}, {"a", 1}, {"b", 2}, {"c", "test"}}, 7,
+	                   11);
+	assertFieldStart(parser, false, 32, 33);
+	assertData(parser, "a", 33, 34);
+	assertCommand(parser, "test", 35, 40);
+	assertFieldEnd(parser, 40, 41);
+	assertFieldStart(parser, false, 41, 42);
+	assertData(parser, "b", 42, 43);
+	assertCommand(parser, "test", 44, 49);
+	assertFieldStart(parser, false, 49, 50);
+	assertFieldEnd(parser, 50, 51);
+	assertFieldEnd(parser, 51, 52);
+	assertRangeEnd(parser, 57, 61);
+	assertEnd(parser, 62, 62);
 }
 
 TEST(OsmlStreamParser, beginEndWithData)
 {
-    const char *testString = "\\begin{book}a\\end{book}";
-    //                         0123456789012 3456789012
-    //                         0         1          2
-    CharReader charReader(testString);
+	const char *testString = "\\begin{book}a\\end{book}";
+	//                         0123456789012 3456789012
+	//                         0         1          2
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
-    assertData(parser, "a", 12, 13);
-    assertRangeEnd(parser, 18, 22);
-    assertEnd(parser, 23, 23);
+	assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
+	assertData(parser, "a", 12, 13);
+	assertRangeEnd(parser, 18, 22);
+	assertEnd(parser, 23, 23);
 }
 
 TEST(OsmlStreamParser, beginEndNested)
 {
-    const char *testString =
-        "\\begin{a}{b} c \\begin{d}{e}{f} \\g{h} \\end{d}\\end{a}";
-    //    012345678901234 5678901234567890 123456 7890123 4567890
-    //    0         1          2         3           4          5
-    CharReader charReader(testString);
+	const char *testString =
+	    "\\begin{a}{b} c \\begin{d}{e}{f} \\g{h} \\end{d}\\end{a}";
+	//    012345678901234 5678901234567890 123456 7890123 4567890
+	//    0         1          2         3           4          5
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommandStart(parser, "a", true, Variant::mapType{}, 7, 8);
-    assertFieldStart(parser, false, 9, 10);
-    assertData(parser, "b", 10, 11);
-    assertFieldEnd(parser, 11, 12);
+	assertCommandStart(parser, "a", true, Variant::mapType{}, 7, 8);
+	assertFieldStart(parser, false, 9, 10);
+	assertData(parser, "b", 10, 11);
+	assertFieldEnd(parser, 11, 12);
 
-    assertData(parser, "c", 13, 14);
+	assertData(parser, "c", 13, 14);
 
-    assertCommandStart(parser, "d", true, Variant::mapType{}, 22, 23);
-    assertFieldStart(parser, false, 24, 25);
-    assertData(parser, "e", 25, 26);
-    assertFieldEnd(parser, 26, 27);
-    assertFieldStart(parser, false, 27, 28);
-    assertData(parser, "f", 28, 29);
-    assertFieldEnd(parser, 29, 30);
+	assertCommandStart(parser, "d", true, Variant::mapType{}, 22, 23);
+	assertFieldStart(parser, false, 24, 25);
+	assertData(parser, "e", 25, 26);
+	assertFieldEnd(parser, 26, 27);
+	assertFieldStart(parser, false, 27, 28);
+	assertData(parser, "f", 28, 29);
+	assertFieldEnd(parser, 29, 30);
 
-    assertEmptyData(parser);
-    assertCommand(parser, "g", 31, 33);
-    assertFieldStart(parser, false, 33, 34);
-    assertData(parser, "h", 34, 35);
-    assertFieldEnd(parser, 35, 36);
-    assertEmptyData(parser);
-    assertRangeEnd(parser, 42, 43);
-    assertRangeEnd(parser, 49, 50);
-    assertEnd(parser, 51, 51);
+	assertEmptyData(parser);
+	assertCommand(parser, "g", 31, 33);
+	assertFieldStart(parser, false, 33, 34);
+	assertData(parser, "h", 34, 35);
+	assertFieldEnd(parser, 35, 36);
+	assertEmptyData(parser);
+	assertRangeEnd(parser, 42, 43);
+	assertRangeEnd(parser, 49, 50);
+	assertEnd(parser, 51, 51);
 }
 
 TEST(OsmlStreamParser, beginEndWithCommand)
 {
-    const char *testString = "\\begin{book}\\a{test}\\end{book}";
-    //                         012345678901 23456789 0123456789
-    //                         0         1           2
-    CharReader charReader(testString);
+	const char *testString = "\\begin{book}\\a{test}\\end{book}";
+	//                         012345678901 23456789 0123456789
+	//                         0         1           2
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
-    assertCommand(parser, "a", 12, 14);
-    assertFieldStart(parser, false, 14, 15);
-    assertData(parser, "test", 15, 19);
-    assertFieldEnd(parser, 19, 20);
-    assertRangeEnd(parser, 25, 29);
-    assertEnd(parser, 30, 30);
+	assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
+	assertCommand(parser, "a", 12, 14);
+	assertFieldStart(parser, false, 14, 15);
+	assertData(parser, "test", 15, 19);
+	assertFieldEnd(parser, 19, 20);
+	assertRangeEnd(parser, 25, 29);
+	assertEnd(parser, 30, 30);
 }
 
 TEST(OsmlStreamParser, beginEndNestedFields)
 {
-    const char *testString = "\\begin{book}a{{b{c}}}\\end{book}";
-    //                         012345678901234567890 1234567890
-    //                         0         1         2          3
-    CharReader charReader(testString);
-    OsmlStreamParser parser(charReader, logger);
-    logger.reset();
-
-    assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
-    assertData(parser, "a", 12, 13);
-    assertFieldStart(parser, false, 13, 14);
-    assertFieldStart(parser, false, 14, 15);
-    assertData(parser, "b", 15, 16);
-    assertFieldStart(parser, false, 16, 17);
-    assertData(parser, "c", 17, 18);
-    assertFieldEnd(parser, 18, 19);
-    assertFieldEnd(parser, 19, 20);
-    assertFieldEnd(parser, 20, 21);
-    assertRangeEnd(parser, 26, 30);
-    assertEnd(parser, 31, 31);
+	const char *testString = "\\begin{book}a{{b{c}}}\\end{book}";
+	//                         012345678901234567890 1234567890
+	//                         0         1         2          3
+	CharReader charReader(testString);
+	OsmlStreamParser parser(charReader, logger);
+	logger.reset();
+
+	assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
+	assertData(parser, "a", 12, 13);
+	assertFieldStart(parser, false, 13, 14);
+	assertFieldStart(parser, false, 14, 15);
+	assertData(parser, "b", 15, 16);
+	assertFieldStart(parser, false, 16, 17);
+	assertData(parser, "c", 17, 18);
+	assertFieldEnd(parser, 18, 19);
+	assertFieldEnd(parser, 19, 20);
+	assertFieldEnd(parser, 20, 21);
+	assertRangeEnd(parser, 26, 30);
+	assertEnd(parser, 31, 31);
 }
 
 TEST(OsmlStreamParser, errorBeginEndUnbalancedNestedFields)
 {
-    const char *testString = "\\begin{book}a{{b{c}}\\end{book}";
-    //                         012345678901234567890 123456789
-    //                         0         1         2
-    CharReader charReader(testString);
-    OsmlStreamParser parser(charReader, logger);
-    logger.reset();
-
-    assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
-    assertData(parser, "a", 12, 13);
-    assertFieldStart(parser, false, 13, 14);
-    assertFieldStart(parser, false, 14, 15);
-    assertData(parser, "b", 15, 16);
-    assertFieldStart(parser, false, 16, 17);
-    assertData(parser, "c", 17, 18);
-    assertFieldEnd(parser, 18, 19);
-    assertFieldEnd(parser, 19, 20);
-    ASSERT_THROW(assertRangeEnd(parser, 25, 29), LoggableException);
+	const char *testString = "\\begin{book}a{{b{c}}\\end{book}";
+	//                         012345678901234567890 123456789
+	//                         0         1         2
+	CharReader charReader(testString);
+	OsmlStreamParser parser(charReader, logger);
+	logger.reset();
+
+	assertCommandStart(parser, "book", true, Variant::mapType{}, 7, 11);
+	assertData(parser, "a", 12, 13);
+	assertFieldStart(parser, false, 13, 14);
+	assertFieldStart(parser, false, 14, 15);
+	assertData(parser, "b", 15, 16);
+	assertFieldStart(parser, false, 16, 17);
+	assertData(parser, "c", 17, 18);
+	assertFieldEnd(parser, 18, 19);
+	assertFieldEnd(parser, 19, 20);
+	ASSERT_THROW(assertRangeEnd(parser, 25, 29), LoggableException);
 }
 
 TEST(OsmlStreamParser, errorBeginEndUnbalancedFields)
@@ -962,446 +960,480 @@ TEST(OsmlStreamParser, errorBeginEndUnbalancedFields)
 
 TEST(OsmlStreamParser, errorBeginNoBraceOpen)
 {
-    const char *testString = "\\begin a";
-    //                         01234567
-    CharReader charReader(testString);
+	const char *testString = "\\begin a";
+	//                         01234567
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertData(parser, "a", 7, 8);
-    ASSERT_TRUE(logger.hasError());
-    assertEnd(parser, 8, 8);
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertData(parser, "a", 7, 8);
+	ASSERT_TRUE(logger.hasError());
+	assertEnd(parser, 8, 8);
 }
 
 TEST(OsmlStreamParser, errorBeginNoIdentifier)
 {
-    const char *testString = "\\begin{!";
-    CharReader charReader(testString);
+	const char *testString = "\\begin{!";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    ASSERT_THROW(parser.parse(), LoggableException);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	ASSERT_THROW(parser.parse(), LoggableException);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorBeginNoBraceClose)
 {
-    const char *testString = "\\begin{a";
-    CharReader charReader(testString);
+	const char *testString = "\\begin{a";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    ASSERT_THROW(parser.parse(), LoggableException);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	ASSERT_THROW(parser.parse(), LoggableException);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorBeginNoName)
 {
-    const char *testString = "\\begin{a#}";
-    CharReader charReader(testString);
+	const char *testString = "\\begin{a#}";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertCommandStart(parser, "a", true);
-    ASSERT_TRUE(logger.hasError());
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertEnd(parser);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertCommandStart(parser, "a", true);
+	ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertEnd(parser);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorEndNoBraceOpen)
 {
-    const char *testString = "\\end a";
-    //                         012345
-    CharReader charReader(testString);
+	const char *testString = "\\end a";
+	//                         012345
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertData(parser, "a", 5, 6);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertData(parser, "a", 5, 6);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorEndNoIdentifier)
 {
-    const char *testString = "\\end{!";
-    CharReader charReader(testString);
+	const char *testString = "\\end{!";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    ASSERT_THROW(parser.parse(), LoggableException);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	ASSERT_THROW(parser.parse(), LoggableException);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorEndNoBraceClose)
 {
-    const char *testString = "\\end{a";
-    CharReader charReader(testString);
+	const char *testString = "\\end{a";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    ASSERT_THROW(parser.parse(), LoggableException);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	ASSERT_THROW(parser.parse(), LoggableException);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorEndNoBegin)
 {
-    const char *testString = "\\end{a}";
-    CharReader charReader(testString);
+	const char *testString = "\\end{a}";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    ASSERT_THROW(parser.parse(), LoggableException);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	ASSERT_THROW(parser.parse(), LoggableException);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, errorBeginEndMismatch)
 {
-    const char *testString = "\\begin{a} \\begin{b} test \\end{a}";
-    //                         0123456789 012345678901234 5678901
-    //                         0          1         2          3
-    CharReader charReader(testString);
+	const char *testString = "\\begin{a} \\begin{b} test \\end{a}";
+	//                         0123456789 012345678901234 5678901
+	//                         0          1         2          3
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    assertCommandStart(parser, "a", true, Variant::mapType{}, 7, 8);
-    assertEmptyData(parser);
-    assertCommandStart(parser, "b", true, Variant::mapType{}, 17, 18);
-    assertData(parser, "test", 20, 24);
-    ASSERT_FALSE(logger.hasError());
-    ASSERT_THROW(parser.parse(), LoggableException);
-    ASSERT_TRUE(logger.hasError());
+	logger.reset();
+	assertCommandStart(parser, "a", true, Variant::mapType{}, 7, 8);
+	assertEmptyData(parser);
+	assertCommandStart(parser, "b", true, Variant::mapType{}, 17, 18);
+	assertData(parser, "test", 20, 24);
+	ASSERT_FALSE(logger.hasError());
+	ASSERT_THROW(parser.parse(), LoggableException);
+	ASSERT_TRUE(logger.hasError());
 }
 
 TEST(OsmlStreamParser, commandWithNSSep)
 {
-    const char *testString = "\\test1:test2";
-    //                         012345678901
-    CharReader charReader(testString);
+	const char *testString = "\\test1:test2";
+	//                         012345678901
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommand(parser, "test1:test2", 0, 12);
-    assertEnd(parser, 12, 12);
+	assertCommand(parser, "test1:test2", 0, 12);
+	assertEnd(parser, 12, 12);
 }
 
 TEST(OsmlStreamParser, beginEndWithNSSep)
 {
-    const char *testString = "\\begin{test1:test2}\\end{test1:test2}";
-    //                         0123456789012345678 90123456789012345
-    //                         0         1          2         3
-    CharReader charReader(testString);
+	const char *testString = "\\begin{test1:test2}\\end{test1:test2}";
+	//                         0123456789012345678 90123456789012345
+	//                         0         1          2         3
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommandStart(parser, "test1:test2", true, Variant::mapType{}, 7, 18);
-    assertRangeEnd(parser, 24, 35);
-    assertEnd(parser, 36, 36);
+	assertCommandStart(parser, "test1:test2", true, Variant::mapType{}, 7, 18);
+	assertRangeEnd(parser, 24, 35);
+	assertEnd(parser, 36, 36);
 }
 
 TEST(OsmlStreamParser, errorBeginNSSep)
 {
-    const char *testString = "\\begin:test{blub}\\end{blub}";
-    CharReader charReader(testString);
+	const char *testString = "\\begin:test{blub}\\end{blub}";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertCommandStart(parser, "blub", true, Variant::mapType{});
-    ASSERT_TRUE(logger.hasError());
-    assertRangeEnd(parser);
-    assertEnd(parser);
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertCommandStart(parser, "blub", true, Variant::mapType{});
+	ASSERT_TRUE(logger.hasError());
+	assertRangeEnd(parser);
+	assertEnd(parser);
 }
 
 TEST(OsmlStreamParser, errorEndNSSep)
 {
-    const char *testString = "\\begin{blub}\\end:test{blub}";
-    CharReader charReader(testString);
+	const char *testString = "\\begin{blub}\\end:test{blub}";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    assertCommandStart(parser, "blub", true, Variant::mapType{});
-    ASSERT_FALSE(logger.hasError());
-    assertRangeEnd(parser);
-    ASSERT_TRUE(logger.hasError());
-    assertEnd(parser);
+	logger.reset();
+	assertCommandStart(parser, "blub", true, Variant::mapType{});
+	ASSERT_FALSE(logger.hasError());
+	assertRangeEnd(parser);
+	ASSERT_TRUE(logger.hasError());
+	assertEnd(parser);
 }
 
 TEST(OsmlStreamParser, errorEmptyNs)
 {
-    const char *testString = "\\test:";
-    CharReader charReader(testString);
+	const char *testString = "\\test:";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertCommand(parser, "test");
-    ASSERT_TRUE(logger.hasError());
-    assertData(parser, ":");
-    assertEnd(parser);
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertCommand(parser, "test");
+	ASSERT_TRUE(logger.hasError());
+	assertData(parser, ":");
+	assertEnd(parser);
 }
 
 TEST(OsmlStreamParser, errorRepeatedNs)
 {
-    const char *testString = "\\test::";
-    CharReader charReader(testString);
+	const char *testString = "\\test::";
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertCommand(parser, "test");
-    ASSERT_TRUE(logger.hasError());
-    assertData(parser, "::");
-    assertEnd(parser);
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertCommand(parser, "test");
+	ASSERT_TRUE(logger.hasError());
+	assertData(parser, "::");
+	assertEnd(parser);
 }
 
 TEST(OsmlStreamParser, explicitDefaultField)
 {
-    const char *testString = "\\a{!b}c";
-    //                         01234567
-    CharReader charReader(testString);
+	const char *testString = "\\a{!b}c";
+	//                         01234567
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommand(parser, "a", 0, 2);
-    assertFieldStart(parser, true, 2, 4);
-    assertData(parser, "b", 4, 5);
-    assertFieldEnd(parser, 5, 6);
-    assertData(parser, "c", 6, 7);
-    assertEnd(parser, 7, 7);
+	assertCommand(parser, "a", 0, 2);
+	assertFieldStart(parser, true, 2, 4);
+	assertData(parser, "b", 4, 5);
+	assertFieldEnd(parser, 5, 6);
+	assertData(parser, "c", 6, 7);
+	assertEnd(parser, 7, 7);
 }
 
 TEST(OsmlStreamParser, explicitDefaultFieldWithCommand)
 {
-    const char *testString = "\\a{!\\b}c";
-    //                         0123 4567
-    CharReader charReader(testString);
+	const char *testString = "\\a{!\\b}c";
+	//                         0123 4567
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertCommand(parser, "a", 0, 2);
-    assertFieldStart(parser, true, 2, 4);
-    assertCommand(parser, "b", 4, 6);
-    assertFieldEnd(parser, 6, 7);
-    assertData(parser, "c", 7, 8);
-    assertEnd(parser, 8, 8);
+	assertCommand(parser, "a", 0, 2);
+	assertFieldStart(parser, true, 2, 4);
+	assertCommand(parser, "b", 4, 6);
+	assertFieldEnd(parser, 6, 7);
+	assertData(parser, "c", 7, 8);
+	assertEnd(parser, 8, 8);
 }
 
 TEST(OsmlStreamParser, fieldAfterExplicitDefaultField)
 {
-    const char *testString = "\\a{!\\b}{c}";
-    //                         0123 456789
-    CharReader charReader(testString);
+	const char *testString = "\\a{!\\b}{c}";
+	//                         0123 456789
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    assertCommand(parser, "a", 0, 2);
-    assertFieldStart(parser, true, 2, 4);
-    assertCommand(parser, "b", 4, 6);
-    assertFieldEnd(parser, 6, 7);
-    assertFieldStart(parser, false, 7, 8);
-    assertData(parser, "c", 8, 9);
-    assertFieldEnd(parser, 9, 10);
-    assertEnd(parser, 10, 10);
+	logger.reset();
+	assertCommand(parser, "a", 0, 2);
+	assertFieldStart(parser, true, 2, 4);
+	assertCommand(parser, "b", 4, 6);
+	assertFieldEnd(parser, 6, 7);
+	assertFieldStart(parser, false, 7, 8);
+	assertData(parser, "c", 8, 9);
+	assertFieldEnd(parser, 9, 10);
+	assertEnd(parser, 10, 10);
 }
 
 TEST(OsmlStreamParser, annotationStart)
 {
-    const char *testString = "<\\a";
-    //                        0 12
+	const char *testString = "<\\a";
+	//                        0 12
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationStart(parser, "a", Variant::mapType{}, 0, 3);
-    assertEnd(parser, 3, 3);
+	assertAnnotationStart(parser, "a", Variant::mapType{}, 0, 3);
+	assertEnd(parser, 3, 3);
 }
 
 TEST(OsmlStreamParser, annotationStartWithName)
 {
-    const char *testString = "<\\annotationWithName#aName";
-    //                        0 1234567890123456789012345
-    //                        0          1         2
+	const char *testString = "<\\annotationWithName#aName";
+	//                        0 1234567890123456789012345
+	//                        0          1         2
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationStart(parser, "annotationWithName",
-                          Variant::mapType{{"name", "aName"}}, 0, 20);
-    assertEnd(parser, 26, 26);
+	assertAnnotationStart(parser, "annotationWithName",
+	                      Variant::mapType{{"name", "aName"}}, 0, 20);
+	assertEnd(parser, 26, 26);
 }
 
 TEST(OsmlStreamParser, annotationStartWithArguments)
 {
-    const char *testString = "<\\annotationWithName#aName[a=1,b=2]";
-    //                        0 1234567890123456789012345678901234
-    //                        0          1         2         3
+	const char *testString = "<\\annotationWithName#aName[a=1,b=2]";
+	//                        0 1234567890123456789012345678901234
+	//                        0          1         2         3
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationStart(
-        parser, "annotationWithName",
-        Variant::mapType{{"name", "aName"}, {"a", 1}, {"b", 2}}, 0, 20);
-    assertEnd(parser, 35, 35);
+	assertAnnotationStart(
+	    parser, "annotationWithName",
+	    Variant::mapType{{"name", "aName"}, {"a", 1}, {"b", 2}}, 0, 20);
+	assertEnd(parser, 35, 35);
 }
 
 TEST(OsmlStreamParser, simpleAnnotationStartBeginEnd)
 {
-    const char *testString = "<\\begin{ab#name}[a=1,b=2] a \\end{ab}\\>";
-    //                        0 123456789012345678901234567 89012345 67
-    //                        0          1         2          3
+	const char *testString = "<\\begin{ab#name}[a=1,b=2] a \\end{ab}\\>";
+	//                        0 123456789012345678901234567 89012345 67
+	//                        0          1         2          3
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationStart(
-        parser, "ab", Variant::mapType{{"name", "name"}, {"a", 1}, {"b", 2}}, 8,
-        10);
-    ASSERT_TRUE(parser.inRangeCommand());
-    assertData(parser, "a", 26, 27);
-    assertRangeEnd(parser, 33, 35);
-    assertAnnotationEnd(parser, "", "", 36, 38);
-    assertEnd(parser, 38, 38);
+	assertAnnotationStart(
+	    parser, "ab", Variant::mapType{{"name", "name"}, {"a", 1}, {"b", 2}}, 8,
+	    10);
+	ASSERT_TRUE(parser.inRangeCommand());
+	assertData(parser, "a", 26, 27);
+	assertRangeEnd(parser, 33, 35);
+	assertAnnotationEnd(parser, "", "", 36, 38);
+	assertEnd(parser, 38, 38);
 }
 
 TEST(OsmlStreamParser, annotationEnd)
 {
-    const char *testString = "\\a>";
-    //                         012
+	const char *testString = "\\a>";
+	//                         012
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationEnd(parser, "a", "", 0, 2);
-    assertEnd(parser, 3, 3);
+	assertAnnotationEnd(parser, "a", "", 0, 2);
+	assertEnd(parser, 3, 3);
 }
 
 TEST(OsmlStreamParser, annotationEndWithName)
 {
-    const char *testString = "\\a#name>";
-    //                         01234567
+	const char *testString = "\\a#name>";
+	//                         01234567
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationEnd(parser, "a", "name", 0, 2);
-    assertEnd(parser, 8, 8);
+	assertAnnotationEnd(parser, "a", "name", 0, 2);
+	assertEnd(parser, 8, 8);
 }
 
 TEST(OsmlStreamParser, annotationEndWithNameAsArgs)
 {
-    const char *testString = "\\a[name=name]>";
-    //                         01234567890123
+	const char *testString = "\\a[name=name]>";
+	//                         01234567890123
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationEnd(parser, "a", "name", 0, 2);
-    assertEnd(parser, 14, 14);
+	assertAnnotationEnd(parser, "a", "name", 0, 2);
+	assertEnd(parser, 14, 14);
 }
 
 TEST(OsmlStreamParser, errorAnnotationEndWithArguments)
 {
-    const char *testString = "\\a[foo=bar]>";
-    //                         012345678901
-    //                         0         1
+	const char *testString = "\\a[foo=bar]>";
+	//                         012345678901
+	//                         0         1
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    logger.reset();
-    ASSERT_FALSE(logger.hasError());
-    assertCommandStart(parser, "a", false, Variant::mapType{{"foo", "bar"}}, 0, 2);
-    ASSERT_TRUE(logger.hasError());
-    assertData(parser, ">", 11, 12);
-    assertEnd(parser, 12, 12);
+	logger.reset();
+	ASSERT_FALSE(logger.hasError());
+	assertCommandStart(parser, "a", false, Variant::mapType{{"foo", "bar"}}, 0,
+	                   2);
+	ASSERT_TRUE(logger.hasError());
+	assertData(parser, ">", 11, 12);
+	assertEnd(parser, 12, 12);
 }
 
 TEST(OsmlStreamParser, closingAnnotation)
 {
-    const char *testString = "<\\a>";
-    //                        0 123
+	const char *testString = "<\\a>";
+	//                        0 123
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertAnnotationStart(parser, "a", Variant::mapType{}, 0, 3);
-    assertData(parser, ">", 3, 4);
-    assertEnd(parser, 4, 4);
+	assertAnnotationStart(parser, "a", Variant::mapType{}, 0, 3);
+	assertData(parser, ">", 3, 4);
+	assertEnd(parser, 4, 4);
 }
 
 TEST(OsmlStreamParser, annotationWithFields)
 {
-    const char *testString = "a <\\b{c}{d}{!e} f \\> g";
-    //                        012 345678901234567 8901
-    //                        0          1          2
+	const char *testString = "a <\\b{c}{d}{!e} f \\> g";
+	//                        012 345678901234567 8901
+	//                        0          1          2
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertData(parser, "a", 0, 1);
-    assertAnnotationStart(parser, "b", Variant::mapType{}, 2, 5);
-    assertFieldStart(parser, false, 5, 6);
-    assertData(parser, "c", 6, 7);
-    assertFieldEnd(parser, 7, 8);
-    assertFieldStart(parser, false, 8, 9);
-    assertData(parser, "d", 9, 10);
-    assertFieldEnd(parser, 10, 11);
-    assertFieldStart(parser, true, 11, 13);
-    assertData(parser, "e", 13, 14);
-    assertFieldEnd(parser, 14, 15);
-    assertData(parser, "f", 16, 17);
-    assertAnnotationEnd(parser, "", "", 18, 20);
-    assertData(parser, "g", 21, 22);
-    assertEnd(parser, 22, 22);
+	assertData(parser, "a", 0, 1);
+	assertAnnotationStart(parser, "b", Variant::mapType{}, 2, 5);
+	assertFieldStart(parser, false, 5, 6);
+	assertData(parser, "c", 6, 7);
+	assertFieldEnd(parser, 7, 8);
+	assertFieldStart(parser, false, 8, 9);
+	assertData(parser, "d", 9, 10);
+	assertFieldEnd(parser, 10, 11);
+	assertFieldStart(parser, true, 11, 13);
+	assertData(parser, "e", 13, 14);
+	assertFieldEnd(parser, 14, 15);
+	assertData(parser, "f", 16, 17);
+	assertAnnotationEnd(parser, "", "", 18, 20);
+	assertData(parser, "g", 21, 22);
+	assertEnd(parser, 22, 22);
 }
 
 TEST(OsmlStreamParser, annotationStartEscape)
 {
-    const char *testString = "<\\%test";
-    //                        0 123456
-    //                        0
+	const char *testString = "<\\%test";
+	//                        0 123456
+	//                        0
 
-    CharReader charReader(testString);
+	CharReader charReader(testString);
 
-    OsmlStreamParser parser(charReader, logger);
+	OsmlStreamParser parser(charReader, logger);
 
-    assertData(parser, "<%test", 0, 7);
-    assertEnd(parser, 7, 7);
+	assertData(parser, "<%test", 0, 7);
+	assertEnd(parser, 7, 7);
 }
 
+TEST(OsmlStreamParser, userDefinedTokens)
+{
+	const char *testString = "<<My dear fellows>>, the *old man* said.";
+	//                        0123456789012345678901234567890123456789
+	//                        0         1         2         3
+
+	CharReader charReader(testString);
+
+	OsmlStreamParser parser(charReader, logger);
+
+	TokenId tSpeechStart = parser.registerToken("<<");
+	TokenId tSpeechEnd = parser.registerToken(">>");
+	TokenId tStar = parser.registerToken("*");
+
+	ASSERT_TRUE(tSpeechStart != Tokens::Empty);
+	ASSERT_TRUE(tSpeechEnd != Tokens::Empty);
+	ASSERT_TRUE(tStar != Tokens::Empty);
+
+	TokenSet tokens{tSpeechStart, tSpeechEnd, tStar};
+
+	ASSERT_EQ(OsmlStreamParser::State::DATA, parser.parse());
+	TokenizedDataReader reader = parser.getData().reader();
+
+	assertToken(reader, tSpeechStart, "<<", tokens, WhitespaceMode::PRESERVE, 0, 2);
+	assertText(reader, "My dear fellows", tokens, WhitespaceMode::PRESERVE, 2, 17);
+	assertToken(reader, tSpeechEnd, ">>", tokens, WhitespaceMode::PRESERVE, 17, 19);
+	assertText(reader, ", the ", tokens, WhitespaceMode::PRESERVE, 19, 25);
+	assertToken(reader, tStar, "*", tokens, WhitespaceMode::PRESERVE, 25, 26);
+	assertText(reader, "old man", tokens, WhitespaceMode::PRESERVE, 26, 33);
+	assertToken(reader, tStar, "*", tokens, WhitespaceMode::PRESERVE, 33, 34);
+	assertText(reader, " said.", tokens, WhitespaceMode::PRESERVE, 34, 40);
+	assertEnd(reader);
+}
 }