1 files changed, 516 insertions, 284 deletions

diff --git a/src/formats/osml/OsmlStreamParser.cpp b/src/formats/osml/OsmlStreamParser.cpp
index f61ac7d..64a489d 100644
--- a/src/formats/osml/OsmlStreamParser.cpp
+++ b/src/formats/osml/OsmlStreamParser.cpp
@@ -16,179 +16,421 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <cassert>
+#include <stack>
+#include <vector>
+
 #include <core/common/CharReader.hpp>
 #include <core/common/Logger.hpp>
 #include <core/common/Utils.hpp>
+#include <core/common/Variant.hpp>
 #include <core/common/VariantReader.hpp>
 
+#include <core/parser/utils/Tokenizer.hpp>
+#include <core/parser/utils/TokenizedData.hpp>
+
 #include "OsmlStreamParser.hpp"
 
 namespace ousia {
 
+namespace {
 /**
- * Plain format default tokenizer.
+ * Osml format default tokenizer. Registers the primary tokens in its
+ * constructor. A single, static instance of this class is created as
+ * "OsmlTokens", which is copied to the Tokenizer instance of
+ * OsmlStreamParserImpl.
  */
-class PlainFormatTokens : public Tokenizer {
+class OsmlFormatTokens : public Tokenizer {
 public:
+	TokenId Backslash;
+	TokenId LineComment;
+	TokenId BlockCommentStart;
+	TokenId BlockCommentEnd;
+	TokenId FieldStart;
+	TokenId FieldEnd;
+	TokenId DefaultFieldStart;
+	TokenId AnnotationStart;
+	TokenId AnnotationEnd;
+
 	/**
-	 * Id of the backslash token.
+	 * Registers the plain format tokens in the internal tokenizer.
 	 */
-	TokenId Backslash;
+	OsmlFormatTokens()
+	{
+		Backslash = registerToken("\\");
+		LineComment = registerToken("%");
+		BlockCommentStart = registerToken("%{");
+		BlockCommentEnd = registerToken("}%");
+		FieldStart = registerToken("{");
+		FieldEnd = registerToken("}");
+		DefaultFieldStart = registerToken("{!");
+		AnnotationStart = registerToken("<\\");
+		AnnotationEnd = registerToken("\\>");
+	}
+};
+
+/**
+ * Instance of OsmlFormatTokens used to initialize the internal tokenizer
+ * instance of OsmlStreamParserImpl.
+ */
+static const OsmlFormatTokens OsmlTokens;
 
+/**
+ * Structure representing a field.
+ */
+struct Field {
 	/**
-	 * Id of the line comment token.
+	 * Specifies whether this field was marked as default field.
 	 */
-	TokenId LineComment;
+	bool defaultField;
 
 	/**
-	 * Id of the block comment start token.
+	 * Location at which the field was started.
 	 */
-	TokenId BlockCommentStart;
+	SourceLocation location;
 
 	/**
-	 * Id of the block comment end token.
+	 * Constructor of the Field structure, initializes all member variables with
+	 * the given values.
+	 *
+	 * @param defaultField is a flag specifying whether this field is a default
+	 * field.
+	 * @param location specifies the location at which the field was started.
 	 */
-	TokenId BlockCommentEnd;
+	Field(bool defaultField = false,
+	      const SourceLocation &location = SourceLocation{})
+	    : defaultField(defaultField), location(location)
+	{
+	}
+};
 
+/**
+ * Entry used for the command stack.
+ */
+class Command {
+private:
 	/**
-	 * Id of the field start token.
+	 * Name and location of the current command.
 	 */
-	TokenId FieldStart;
+	Variant name;
 
 	/**
-	 * Id of the field end token.
+	 * Arguments that were passed to the command.
 	 */
-	TokenId FieldEnd;
+	Variant arguments;
 
 	/**
-	 * Id of the default field start token.
+	 * Vector used as stack for holding the number of opening/closing braces
+	 * and the corresponding "isDefaultField" flag.
 	 */
-	TokenId DefaultFieldStart;
+	std::vector<Field> fields;
 
 	/**
-	 * Id of the annotation start token.
+	 * Set to true if this is a command with clear begin and end.
 	 */
-	TokenId AnnotationStart;
+	bool hasRange;
 
+public:
 	/**
-	 * Id of the annotation end token.
+	 * Default constructor, marks this command as normal, non-range command.
 	 */
-	TokenId AnnotationEnd;
+	Command() : hasRange(false) {}
 
 	/**
-	 * Registers the plain format tokens in the internal tokenizer.
+	 * Constructor of the Command class.
+	 *
+	 * @param name is a string variant with name and location of the
+	 * command.
+	 * @param arguments is a map variant with the arguments given to the
+	 * command.
+	 * @param hasRange should be set to true if this is a command with
+	 * explicit range.
 	 */
-	PlainFormatTokens()
+	Command(Variant name, Variant arguments, bool hasRange)
+	    : name(std::move(name)),
+	      arguments(std::move(arguments)),
+	      hasRange(hasRange)
 	{
-		Backslash = registerToken("\\");
-		LineComment = registerToken("%");
-		BlockCommentStart = registerToken("%{");
-		BlockCommentEnd = registerToken("}%");
-		FieldStart = registerToken("{");
-		FieldEnd = registerToken("}");
-		DefaultFieldStart = registerToken("{!");
-		AnnotationStart = registerToken("<\\");
-		AnnotationEnd = registerToken("\\>");
 	}
-};
 
-static const PlainFormatTokens OsmlTokens;
+	/**
+	 * Returns a reference at the variant representing name and location of the
+	 * command.
+	 *
+	 * @return a variant containing name and location of the command.
+	 */
+	const Variant &getName() const { return name; }
 
-/**
- * Class used internally to collect data issued via "DATA" event.
- */
-class DataHandler {
-private:
 	/**
-	 * Internal character buffer.
+	 * Returns a reference at the variant containing name, value and location of
+	 * the arguments.
+	 *
+	 * @return the arguments stored for the command.
 	 */
-	std::vector<char> buf;
+	const Variant &getArguments() const { return arguments; }
 
 	/**
-	 * Start location of the character data.
+	 * Returns a reference at the internal field list. This list should be used
+	 * for printing error messages when fields are still open although the outer
+	 * range field closes.
+	 *
+	 * @return a const reference at the internal field vector.
 	 */
-	SourceOffset start;
+	const std::vector<Field> &getFields() const { return fields; }
 
 	/**
-	 * End location of the character data.
+	 * Returns true if this command is currently in a default field.
+	 *
+	 * @return true if the current field on the field stack was explicitly
+	 * marked as default field. If the field stack is empty, true is returned
+	 * if this is a range command.
 	 */
-	SourceOffset end;
+	bool inDefaultField() const
+	{
+		return (!fields.empty() && fields.back().defaultField) ||
+		       (fields.empty() && hasRange);
+	}
 
-public:
 	/**
-	 * Default constructor, initializes start and end with zeros.
+	 * Returns true if this command currently is in any field.
+	 *
+	 * @return true if a field is on the stack or this is a range commands.
+	 * Range commands always are in a field.
 	 */
-	DataHandler() : start(0), end(0) {}
+	bool inField() const { return !fields.empty() || hasRange; }
 
 	/**
-	 * Returns true if the internal buffer is empty.
+	 * Returns true if this command currently is in a range field.
 	 *
-	 * @return true if no characters were added to the internal buffer, false
-	 * otherwise.
+	 * @return true if the command has a range and no other ranges are on the
+	 * stack.
 	 */
-	bool isEmpty() { return buf.empty(); }
+	bool inRangeField() const { return fields.empty() && hasRange; }
 
 	/**
-	 * Appends a single character to the internal buffer.
+	 * Returns true if this command currently is in a non-range field.
 	 *
-	 * @param c is the character that should be added to the internal buffer.
-	 * @param charStart is the start position of the character.
-	 * @param charEnd is the end position of the character.
+	 * @return true if the command is in a field, but the field is not the field
+	 * constructed by the "range"
 	 */
-	void append(char c, SourceOffset charStart, SourceOffset charEnd)
+	bool inNonRangeField() const { return !fields.empty(); }
+
+	/**
+	 * Pushes another field onto the field stack of this command.
+	 *
+	 * @param defaultField if true, explicitly marks this field as default
+	 * field.
+	 * @param location is the source location at which the field was started.
+	 * Used for error messages in which the user is notified about an error with
+	 * too few closing fields.
+	 */
+	void pushField(bool defaultField = false,
+	               const SourceLocation &location = SourceLocation{})
 	{
-		if (isEmpty()) {
-			start = charStart;
-		}
-		buf.push_back(c);
-		end = charEnd;
+		fields.emplace_back(defaultField, location);
 	}
 
 	/**
-	 * Appends a string to the internal buffer.
+	 * Removes another field from the field stack of this command, returns true
+	 * if the operation was successful.
 	 *
-	 * @param s is the string that should be added to the internal buffer.
-	 * @param stringStart is the start position of the string.
-	 * @param stringEnd is the end position of the string.
+	 * @return true if there was a field to pop on the stack, false otherwise.
 	 */
-	void append(const std::string &s, SourceOffset stringStart,
-	            SourceOffset stringEnd)
+	bool popField()
 	{
-		if (isEmpty()) {
-			start = stringStart;
+		if (!fields.empty()) {
+			fields.pop_back();
+			return true;
 		}
-		std::copy(s.c_str(), s.c_str() + s.size(), back_inserter(buf));
-		end = stringEnd;
+		return false;
 	}
+};
+}
+
+/* Class OsmlStreamParserImpl */
+
+/**
+ * Internal implementation of OsmlStreamParser.
+ */
+class OsmlStreamParserImpl {
+public:
+	/**
+	 * State enum compatible with OsmlStreamParserState but extended by two more
+	 * entries (END and NONE).
+	 */
+	enum class State : uint8_t {
+		COMMAND_START = 0,
+		RANGE_END = 1,
+		FIELD_START = 2,
+		FIELD_END = 3,
+		ANNOTATION_START = 4,
+		ANNOTATION_END = 5,
+		DATA = 6,
+		END = 7,
+		RECOVERABLE_ERROR = 8,
+		IRRECOVERABLE_ERROR = 9
+	};
+
+private:
+	/**
+	 * Reference to the CharReader instance from which the incomming bytes are
+	 * read.
+	 */
+	CharReader &reader;
 
 	/**
-	 * Converts the internal buffer to a variant with attached location
-	 * information.
+	 * Reference at the logger instance to which all error messages are sent.
+	 */
+	Logger &logger;
+
+	/**
+	 * Tokenizer instance used to read individual tokens from the text.
+	 */
+	Tokenizer tokenizer;
+
+	/**
+	 * Stack containing the current commands.
+	 */
+	std::stack<Command> commands;
+
+	/**
+	 * Variant containing the tokenized data that was returned from the
+	 * tokenizer as data.
+	 */
+	TokenizedData data;
+
+	/**
+	 * Variable containing the current location of the parser.
+	 */
+	SourceLocation location;
+
+	/**
+	 * Function used internally to parse an identifier.
 	 *
-	 * @param sourceId is the source id which is needed for building the
-	 * location information.
-	 * @return a Variant with the internal buffer content as string and
-	 * the correct start and end location.
+	 * @param start is the start byte offset of the identifier (including the
+	 * backslash).
+	 * @param allowNSSep should be set to true if the namespace separator is
+	 * allowed in the identifier name. Issues error if the namespace separator
+	 * is placed incorrectly.
 	 */
-	Variant toVariant(SourceId sourceId)
-	{
-		Variant res = Variant::fromString(std::string(buf.data(), buf.size()));
-		res.setLocation({sourceId, start, end});
-		return res;
-	}
+	Variant parseIdentifier(size_t start, bool allowNSSep = false);
+
+	/**
+	 * Function used internally to handle the special "\begin" command.
+	 *
+	 * @return an internal State specifying whether an error occured (return
+	 * values State::REOVERABLE_ERROR or State::IRRECOVERABLE_ERROR) or a
+	 * command was actually started (return value State::COMMAND_START).
+	 */
+	State parseBeginCommand();
+
+	/**
+	 * Function used internally to handle the special "\end" command.
+	 *
+	 * @return an internal State specifying whether an error occured (return
+	 * values State::REOVERABLE_ERROR or State::IRRECOVERABLE_ERROR) or a
+	 * command was actually ended (return value State::RANGE_END).
+	 */
+	State parseEndCommand();
+
+	/**
+	 * Parses the command arguments. Handles errors if the name of the command
+	 * was given using the hash notation and as a name field.
+	 *
+	 * @param commandArgName is the name argument that was given using the hash
+	 * notation.
+	 * @return a map variant containing the arguments.
+	 */
+	Variant parseCommandArguments(Variant commandArgName);
+
+	/**
+	 * Function used internally to parse a command.
+	 *
+	 * @param start is the start byte offset of the command (including the
+	 * backslash)
+	 * @param isAnnotation if true, the command is not returned as command, but
+	 * as annotation start.
+	 * @return true if a command was actuall parsed, false otherwise.
+	 */
+	State parseCommand(size_t start, bool isAnnotation);
+
+	/**
+	 * Function used internally to parse a block comment.
+	 */
+	void parseBlockComment();
+
+	/**
+	 * Function used internally to parse a generic comment.
+	 */
+	void parseLineComment();
+
+	/**
+	 * Pushes the parsed command onto the command stack.
+	 */
+	void pushCommand(Variant commandName, Variant commandArguments,
+	                 bool hasRange);
+
+	/**
+	 * Checks whether there is any data pending to be issued, if yes, resets the
+	 * currently peeked characters and returns true.
+	 *
+	 * @return true if there was any data and DATA should be returned by the
+	 * parse function, false otherwise.
+	 */
+	bool checkIssueData();
+
+	/**
+	 * Returns a reference at the current command at the top of the command
+	 * stack.
+	 *
+	 * @return a reference at the top command in the command stack.
+	 */
+	Command &cmd() { return commands.top(); }
+
+	/**
+	 * Returns a reference at the current command at the top of the command
+	 * stack.
+	 *
+	 * @return a reference at the top command in the command stack.
+	 */
+	const Command &cmd() const { return commands.top(); }
+
+public:
+	/**
+	 * Constructor of the OsmlStreamParserImpl class. Attaches the new
+	 * OsmlStreamParserImpl to the given CharReader and Logger instances.
+	 *
+	 * @param reader is the reader instance from which incomming characters
+	 * should be read.
+	 * @param logger is the logger instance to which errors should be written.
+	 */
+	OsmlStreamParserImpl(CharReader &reader, Logger &logger);
+
+	State parse();
+
+	TokenId registerToken(const std::string &token);
+	void unregisterToken(TokenId id);
+
+	const TokenizedData &getData() const { return data; }
+	const Variant &getCommandName() const { return cmd().getName(); }
+	const Variant &getCommandArguments() const { return cmd().getArguments(); }
+	const SourceLocation &getLocation() const { return location; }
+	bool inRangeCommand() const { return cmd().inRangeField(); };
+	bool inDefaultField() const { return cmd().inDefaultField(); }
 };
 
-OsmlStreamParser::OsmlStreamParser(CharReader &reader, Logger &logger)
+/* Class OsmlStreamParserImpl */
+
+OsmlStreamParserImpl::OsmlStreamParserImpl(CharReader &reader, Logger &logger)
     : reader(reader), logger(logger), tokenizer(OsmlTokens)
 {
-	// Place an intial command representing the complete file on the stack
-	commands.push(Command{"", Variant::mapType{}, true, true, true, false});
+	commands.emplace("", Variant::mapType{}, true);
 }
 
-Variant OsmlStreamParser::parseIdentifier(size_t start, bool allowNSSep)
+Variant OsmlStreamParserImpl::parseIdentifier(size_t start, bool allowNSSep)
 {
 	bool first = true;
-	bool hasCharSiceNSSep = false;
+	bool hasCharSinceNSSep = false;
 	std::vector<char> identifier;
 	size_t end = reader.getPeekOffset();
 	char c, c2;
@@ -197,7 +439,7 @@ Variant OsmlStreamParser::parseIdentifier(size_t start, bool allowNSSep)
 		if ((first && Utils::isIdentifierStartCharacter(c)) ||
 		    (!first && Utils::isIdentifierCharacter(c))) {
 			identifier.push_back(c);
-		} else if (c == ':' && hasCharSiceNSSep && reader.fetchPeek(c2) &&
+		} else if (c == ':' && hasCharSinceNSSep && reader.fetchPeek(c2) &&
 		           Utils::isIdentifierStartCharacter(c2)) {
 			identifier.push_back(c);
 		} else {
@@ -214,8 +456,8 @@ Variant OsmlStreamParser::parseIdentifier(size_t start, bool allowNSSep)
 		// This is no longer the first character
 		first = false;
 
-		// Advance the hasCharSiceNSSep flag
-		hasCharSiceNSSep = allowNSSep && (c != ':');
+		// Advance the hasCharSinceNSSep flag
+		hasCharSinceNSSep = allowNSSep && (c != ':');
 
 		end = reader.getPeekOffset();
 		reader.consumePeek();
@@ -228,20 +470,20 @@ Variant OsmlStreamParser::parseIdentifier(size_t start, bool allowNSSep)
 	return res;
 }
 
-OsmlStreamParser::State OsmlStreamParser::parseBeginCommand()
+OsmlStreamParserImpl::State OsmlStreamParserImpl::parseBeginCommand()
 {
 	// Expect a '{' after the command
 	reader.consumeWhitespace();
 	if (!reader.expect('{')) {
 		logger.error("Expected \"{\" after \\begin", reader);
-		return State::NONE;
+		return State::RECOVERABLE_ERROR;
 	}
 
 	// Parse the name of the command that should be opened
 	Variant commandName = parseIdentifier(reader.getOffset(), true);
 	if (commandName.asString().empty()) {
 		logger.error("Expected identifier", commandName);
-		return State::ERROR;
+		return State::IRRECOVERABLE_ERROR;
 	}
 
 	// Check whether the next character is a '#', indicating the start of the
@@ -257,7 +499,7 @@ OsmlStreamParser::State OsmlStreamParser::parseBeginCommand()
 
 	if (!reader.expect('}')) {
 		logger.error("Expected \"}\"", reader);
-		return State::ERROR;
+		return State::IRRECOVERABLE_ERROR;
 	}
 
 	// Parse the arguments
@@ -266,28 +508,15 @@ OsmlStreamParser::State OsmlStreamParser::parseBeginCommand()
 	// Push the command onto the command stack
 	pushCommand(std::move(commandName), std::move(commandArguments), true);
 
-	return State::COMMAND;
-}
-
-static bool checkStillInField(const OsmlStreamParser::Command &cmd,
-                              const Variant &endName, Logger &logger)
-{
-	if (cmd.inField && !cmd.inRangeField) {
-		logger.error(std::string("\\end in open field of command \"") +
-		                 cmd.name.asString() + std::string("\""),
-		             endName);
-		logger.note(std::string("Open command started here:"), cmd.name);
-		return true;
-	}
-	return false;
+	return State::COMMAND_START;
 }
 
-OsmlStreamParser::State OsmlStreamParser::parseEndCommand()
+OsmlStreamParserImpl::State OsmlStreamParserImpl::parseEndCommand()
 {
 	// Expect a '{' after the command
 	if (!reader.expect('{')) {
 		logger.error("Expected \"{\" after \\end", reader);
-		return State::NONE;
+		return State::RECOVERABLE_ERROR;
 	}
 
 	// Fetch the name of the command that should be ended here
@@ -296,56 +525,58 @@ OsmlStreamParser::State OsmlStreamParser::parseEndCommand()
 	// Make sure the given command name is not empty
 	if (name.asString().empty()) {
 		logger.error("Expected identifier", name);
-		return State::ERROR;
+		return State::IRRECOVERABLE_ERROR;
 	}
 
 	// Make sure the command name is terminated with a '}'
 	if (!reader.expect('}')) {
 		logger.error("Expected \"}\"", reader);
-		return State::ERROR;
-	}
-
-	// Unroll the command stack up to the last range command
-	while (!commands.top().hasRange) {
-		if (checkStillInField(commands.top(), name, logger)) {
-			return State::ERROR;
+		return State::IRRECOVERABLE_ERROR;
+	}
+
+	// Unroll the command stack up to the last range command, make sure we do
+	// not intersect with any open field
+	while (!cmd().inRangeField()) {
+		if (cmd().inField()) {
+			logger.error(std::string("\\end in open field of command \"") +
+			                 cmd().getName().asString() + std::string("\""),
+			             name);
+			const std::vector<Field> &fields = cmd().getFields();
+			for (const Field &field : fields) {
+				logger.note(std::string("Still open field started here: "),
+				            field.location);
+			}
+			return State::IRRECOVERABLE_ERROR;
 		}
 		commands.pop();
 	}
 
-	// Make sure we're not in an open field of this command
-	if (checkStillInField(commands.top(), name, logger)) {
-		return State::ERROR;
-	}
-
 	// Special error message if the top-level command is reached
 	if (commands.size() == 1) {
 		logger.error(std::string("Cannot end command \"") + name.asString() +
 		                 std::string("\" here, no command open"),
 		             name);
-		return State::ERROR;
+		return State::IRRECOVERABLE_ERROR;
 	}
 
-	// Inform the about command mismatches
-	const Command &cmd = commands.top();
-	if (commands.top().name.asString() != name.asString()) {
-		logger.error(std::string("Trying to end command \"") +
-		                 cmd.name.asString() +
+	// Inform the user about command mismatches, copy the current command
+	// descriptor before popping it from the stack
+	if (getCommandName().asString() != name.asString()) {
+		logger.error(std::string("Trying to end command \"") + name.asString() +
 		                 std::string("\", but open command is \"") +
-		                 name.asString() + std::string("\""),
+		                 getCommandName().asString() + std::string("\""),
 		             name);
-		logger.note("Last command was opened here:", cmd.name);
-		return State::ERROR;
+		logger.note("Open command started here:", getCommandName());
+		return State::IRRECOVERABLE_ERROR;
 	}
 
-	// Set the location to the location of the command that was ended, then end
-	// the current command
+	// End the current command
 	location = name.getLocation();
 	commands.pop();
-	return cmd.inRangeField ? State::FIELD_END : State::NONE;
+	return State::RANGE_END;
 }
 
-Variant OsmlStreamParser::parseCommandArguments(Variant commandArgName)
+Variant OsmlStreamParserImpl::parseCommandArguments(Variant commandArgName)
 {
 	// Parse the arguments using the universal VariantReader
 	Variant commandArguments;
@@ -371,29 +602,14 @@ Variant OsmlStreamParser::parseCommandArguments(Variant commandArgName)
 	return commandArguments;
 }
 
-void OsmlStreamParser::pushCommand(Variant commandName,
-                                   Variant commandArguments, bool hasRange)
-{
-	// Store the location on the stack
-	location = commandName.getLocation();
-
-	// Place the command on the command stack, remove the last commands if we're
-	// not currently inside a field of these commands
-	while (!commands.top().inField) {
-		commands.pop();
-	}
-	commands.push(Command{std::move(commandName), std::move(commandArguments),
-	                      hasRange, false, false, false});
-}
-
-OsmlStreamParser::State OsmlStreamParser::parseCommand(size_t start,
-                                                       bool isAnnotation)
+OsmlStreamParserImpl::State OsmlStreamParserImpl::parseCommand(
+    size_t start, bool isAnnotation)
 {
 	// Parse the commandName as a first identifier
 	Variant commandName = parseIdentifier(start, true);
 	if (commandName.asString().empty()) {
 		logger.error("Empty command name", reader);
-		return State::NONE;
+		return State::RECOVERABLE_ERROR;
 	}
 
 	// Handle the special "begin" and "end" commands
@@ -403,7 +619,7 @@ OsmlStreamParser::State OsmlStreamParser::parseCommand(size_t start,
 	const bool isEnd = commandNameComponents[0] == "end";
 
 	// Parse the begin or end command
-	State res = State::COMMAND;
+	State res = State::COMMAND_START;
 	if (isBegin || isEnd) {
 		if (commandNameComponents.size() > 1) {
 			logger.error(
@@ -459,12 +675,13 @@ OsmlStreamParser::State OsmlStreamParser::parseCommand(size_t start,
 		} else {
 			// Make sure no arguments apart from the "name" argument are given
 			// to an annotation end
-			Variant::mapType &map = commands.top().arguments.asMap();
+			const Variant::mapType &map = getCommandArguments().asMap();
 			if (!map.empty()) {
 				if (map.count("name") == 0 || map.size() > 1U) {
 					logger.error(
 					    "An annotation end command may not have any arguments "
-					    "other than \"name\"");
+					    "other than \"name\"",
+					    reader);
 					return res;
 				}
 			}
@@ -478,17 +695,21 @@ OsmlStreamParser::State OsmlStreamParser::parseCommand(size_t start,
 
 	// If we're starting an annotation, return the command as annotation start
 	// instead of command
-	if (isAnnotation && res == State::COMMAND) {
+	if (isAnnotation && res == State::COMMAND_START) {
 		return State::ANNOTATION_START;
 	}
 	return res;
 }
 
-void OsmlStreamParser::parseBlockComment()
+void OsmlStreamParserImpl::parseBlockComment()
 {
 	Token token;
+	TokenizedData commentData;
 	size_t depth = 1;
-	while (tokenizer.read(reader, token)) {
+	while (tokenizer.read(reader, token, commentData)) {
+		// Throw the comment data away
+		commentData.clear();
+
 		if (token.id == OsmlTokens.BlockCommentEnd) {
 			depth--;
 			if (depth == 0) {
@@ -504,7 +725,7 @@ void OsmlStreamParser::parseBlockComment()
 	logger.error("File ended while being in a block comment", reader);
 }
 
-void OsmlStreamParser::parseLineComment()
+void OsmlStreamParserImpl::parseLineComment()
 {
 	char c;
 	while (reader.read(c)) {
@@ -514,86 +735,46 @@ void OsmlStreamParser::parseLineComment()
 	}
 }
 
-bool OsmlStreamParser::checkIssueData(DataHandler &handler)
+void OsmlStreamParserImpl::pushCommand(Variant commandName,
+                                       Variant commandArguments, bool hasRange)
 {
-	if (!handler.isEmpty()) {
-		data = handler.toVariant(reader.getSourceId());
-		location = data.getLocation();
-		reader.resetPeek();
-		return true;
-	}
-	return false;
-}
-
-bool OsmlStreamParser::checkIssueFieldStart()
-{
-	// Fetch the current command, and check whether we're currently inside a
-	// field of this command
-	Command &cmd = commands.top();
-	if (!cmd.inField) {
-		// If this is a range command, we're now implicitly inside the field of
-		// this command -- we'll have to issue a field start command!
-		if (cmd.hasRange) {
-			cmd.inField = true;
-			cmd.inRangeField = true;
-			reader.resetPeek();
-			return true;
-		}
+	// Store the location of the command
+	location = commandName.getLocation();
 
-		// This was not a range command, so obviously we're now inside within
-		// a field of some command -- so unroll the commands stack until a
-		// command with open field is reached
-		while (!commands.top().inField) {
-			commands.pop();
-		}
+	// Place the command on the command stack, remove the last commands if we're
+	// not currently inside a field of these commands
+	while (!cmd().inField()) {
+		commands.pop();
 	}
-	return false;
+
+	// Push the new command onto the command stack
+	commands.emplace(std::move(commandName), std::move(commandArguments),
+	                 hasRange);
 }
 
-bool OsmlStreamParser::closeField()
+bool OsmlStreamParserImpl::checkIssueData()
 {
-	// Try to end an open field of the current command -- if the current command
-	// is not inside an open field, end this command and try to close the next
-	// one
-	for (int i = 0; i < 2 && commands.size() > 1; i++) {
-		Command &cmd = commands.top();
-		if (!cmd.inRangeField) {
-			if (cmd.inField) {
-				cmd.inField = false;
-				if (cmd.inDefaultField) {
-					commands.pop();
-				}
-				return true;
-			}
-			commands.pop();
-		} else {
-			return false;
-		}
+	if (!data.empty()) {
+		location = data.getLocation();
+		reader.resetPeek();
+		return true;
 	}
 	return false;
 }
 
-OsmlStreamParser::State OsmlStreamParser::parse()
+OsmlStreamParserImpl::State OsmlStreamParserImpl::parse()
 {
-	// Handler for incomming data
-	DataHandler handler;
+	// Reset the data handler
+	data.clear();
 
 	// Read tokens until the outer loop should be left
 	Token token;
-	while (tokenizer.peek(reader, token)) {
+	while (tokenizer.peek(reader, token, data)) {
 		const TokenId type = token.id;
 
 		// Special handling for Backslash and Text
 		if (type == OsmlTokens.Backslash ||
 		    type == OsmlTokens.AnnotationStart) {
-			// Before appending anything to the output data or starting a new
-			// command, check whether FIELD_START has to be issued, as the
-			// current command is a command with range
-			if (checkIssueFieldStart()) {
-				location = token.location;
-				return State::FIELD_START;
-			}
-
 			// Check whether a command starts now, without advancing the peek
 			// cursor
 			char c;
@@ -606,7 +787,7 @@ OsmlStreamParser::State OsmlStreamParser::parse()
 			// Try to parse a command
 			if (Utils::isIdentifierStartCharacter(c)) {
 				// Make sure to issue any data before it is to late
-				if (checkIssueData(handler)) {
+				if (checkIssueData()) {
 					return State::DATA;
 				}
 
@@ -614,11 +795,11 @@ OsmlStreamParser::State OsmlStreamParser::parse()
 				State res = parseCommand(token.location.getStart(),
 				                         type == OsmlTokens.AnnotationStart);
 				switch (res) {
-					case State::ERROR:
+					case State::IRRECOVERABLE_ERROR:
 						throw LoggableException(
 						    "Last error was irrecoverable, ending parsing "
 						    "process");
-					case State::NONE:
+					case State::RECOVERABLE_ERROR:
 						continue;
 					default:
 						return res;
@@ -632,78 +813,64 @@ OsmlStreamParser::State OsmlStreamParser::parse()
 
 			// If this was an annotation start token, add the parsed < to the
 			// output
+			SourceOffset charStart = token.location.getStart();
+			SourceOffset charEnd = reader.getPeekOffset();
 			if (type == OsmlTokens.AnnotationStart) {
-				handler.append('<', token.location.getStart(),
-				               token.location.getStart() + 1);
+				data.append('<', charStart, charStart + 1);
+				charStart = charStart + 1;
 			}
 
-			handler.append(c, token.location.getStart(),
-			               reader.getPeekOffset());
+			// Append the character to the output data, mark it as protected
+			data.append(c, charStart, charEnd, true);
 			reader.consumePeek();
 			continue;
 		} else if (type == Tokens::Data) {
-			// Check whether FIELD_START has to be issued before appending text
-			if (checkIssueFieldStart()) {
-				location = token.location;
-				return State::FIELD_START;
-			}
-
-			// Append the text to the data handler
-			handler.append(token.content, token.location.getStart(),
-			               token.location.getEnd());
-
 			reader.consumePeek();
 			continue;
+		} else if (type == OsmlTokens.LineComment) {
+			reader.consumePeek();
+			parseLineComment();
+			continue;
+		} else if (type == OsmlTokens.BlockCommentStart) {
+			reader.consumePeek();
+			parseBlockComment();
+			continue;
 		}
 
 		// A non-text token was reached, make sure all pending data commands
 		// have been issued
-		if (checkIssueData(handler)) {
+		if (checkIssueData()) {
 			return State::DATA;
 		}
 
 		// We will handle the token now, consume the peeked characters
 		reader.consumePeek();
 
-		// Update the location to the current token location
+		// Synchronize the location with the current token location
 		location = token.location;
 
-		if (token.id == OsmlTokens.LineComment) {
-			parseLineComment();
-		} else if (token.id == OsmlTokens.BlockCommentStart) {
-			parseBlockComment();
-		} else if (token.id == OsmlTokens.FieldStart) {
-			Command &cmd = commands.top();
-			if (!cmd.inField) {
-				cmd.inField = true;
-				return State::FIELD_START;
-			}
-			logger.error(
-			    "Got field start token \"{\", but no command for which to "
-			    "start the field. Write \"\\{\" to insert this sequence as "
-			    "text.",
-			    token);
+		if (token.id == OsmlTokens.FieldStart) {
+			cmd().pushField(false, token.location);
+			return State::FIELD_START;
 		} else if (token.id == OsmlTokens.FieldEnd) {
-			if (closeField()) {
+			// Remove all commands from the list that currently are not in any
+			// field
+			while (!cmd().inField()) {
+				commands.pop();
+			}
+
+			// If the remaining command is not in a range field, remove this
+			// command
+			if (cmd().inNonRangeField()) {
+				cmd().popField();
 				return State::FIELD_END;
 			}
 			logger.error(
-			    "Got field end token \"}\", but there is no field to end. "
-			    "Write \"\\}\" to insert this sequence as text.",
+			    "Got field end token \"}\", but there is no field to end.",
 			    token);
 		} else if (token.id == OsmlTokens.DefaultFieldStart) {
-			// Try to start a default field the first time the token is reached
-			Command &topCmd = commands.top();
-			if (!topCmd.inField) {
-				topCmd.inField = true;
-				topCmd.inDefaultField = true;
-				return State::FIELD_START;
-			}
-			logger.error(
-			    "Got default field start token \"{!\", but no command for "
-			    "which to start the field. Write \"\\{!\" to insert this "
-			    "sequence as text",
-			    token);
+			cmd().pushField(true, token.location);
+			return State::FIELD_START;
 		} else if (token.id == OsmlTokens.AnnotationEnd) {
 			// We got a single annotation end token "\>" -- simply issue the
 			// ANNOTATION_END event
@@ -717,38 +884,103 @@ OsmlStreamParser::State OsmlStreamParser::parse()
 	}
 
 	// Issue available data
-	if (checkIssueData(handler)) {
+	if (checkIssueData()) {
 		return State::DATA;
 	}
 
 	// Make sure all open commands and fields have been ended at the end of the
 	// stream
-	while (commands.size() > 1) {
-		Command &cmd = commands.top();
-		if (cmd.inField || cmd.hasRange) {
-			logger.error("Reached end of stream, but command \"" +
-			                 cmd.name.asString() + "\" has not been ended",
-			             cmd.name);
+	while (true) {
+		bool topLevelCommand = commands.size() == 1U;
+		if (cmd().inField()) {
+			// If the stream ended with an open range field, issue information
+			// about the range field
+			if (cmd().inRangeField() && !topLevelCommand) {
+				// Inform about the still open command itself
+				logger.error("Reached end of stream, but command \"" +
+				                 getCommandName().asString() +
+				                 "\" has not been ended",
+				             getCommandName());
+			} else {
+				// Issue information about still open fields
+				const std::vector<Field> &fields = cmd().getFields();
+				if (!fields.empty()) {
+					logger.error(
+					    std::string(
+					        "Reached end of stream, but field is still open."),
+					    fields.back().location);
+				}
+			}
+		}
+		if (!topLevelCommand) {
+			commands.pop();
+		} else {
+			break;
 		}
-		commands.pop();
 	}
 
 	location = SourceLocation{reader.getSourceId(), reader.getOffset()};
 	return State::END;
 }
 
+TokenId OsmlStreamParserImpl::registerToken(const std::string &token)
+{
+	return tokenizer.registerToken(token, false);
+}
+
+void OsmlStreamParserImpl::unregisterToken(TokenId id)
+{
+	assert(tokenizer.unregisterToken(id));
+}
+
+/* Class OsmlStreamParser */
+
+OsmlStreamParser::OsmlStreamParser(CharReader &reader, Logger &logger)
+    : impl(new OsmlStreamParserImpl(reader, logger))
+{
+}
+
+OsmlStreamParser::~OsmlStreamParser()
+{
+	// Stub needed because OsmlStreamParserImpl is incomplete in header
+}
+
+OsmlStreamParser::State OsmlStreamParser::parse()
+{
+	return static_cast<State>(impl->parse());
+}
+
+const TokenizedData &OsmlStreamParser::getData() const
+{
+	return impl->getData();
+}
+
 const Variant &OsmlStreamParser::getCommandName() const
 {
-	return commands.top().name;
+	return impl->getCommandName();
 }
 
 const Variant &OsmlStreamParser::getCommandArguments() const
 {
-	return commands.top().arguments;
+	return impl->getCommandArguments();
+}
+
+const SourceLocation &OsmlStreamParser::getLocation() const
+{
+	return impl->getLocation();
+}
+
+bool OsmlStreamParser::inDefaultField() const { return impl->inDefaultField(); }
+
+bool OsmlStreamParser::inRangeCommand() const { return impl->inRangeCommand(); }
+
+TokenId OsmlStreamParser::registerToken(const std::string &token)
+{
+	return impl->registerToken(token);
 }
 
-bool OsmlStreamParser::inDefaultField() const
+void OsmlStreamParser::unregisterToken(TokenId id)
 {
-	return commands.top().inRangeField || commands.top().inDefaultField;
+	impl->unregisterToken(id);
 }
 }