First implementation of user defined syntax, many features still missing and probably many bugs

* Implement startToken and endToken in DocumentChildHandler
* Implement pushScopeToken, which pushes tokens for the element that is currently on top of the Scope stack onto the token stack
* Implement rollbackPath() method (was really needed once in the development process, but only used in one place for now)
* Push and pop tokens from stack whenever a new explicit field or command is created/ended. Take advantage of the fact, that the tokens for transparent structures are always included in the token list
* Remember pending close tokens in the HandlerInfo structure in StackImpl
* Implement handleToken() in StackImpl
* Implement readToken() method used by readData() and data() in StackImpl
* Check whether there still is data available in handleData()
* Plus many more changes in the affected files...

3 files changed, 535 insertions, 74 deletions

diff --git a/src/core/parser/stack/DocumentHandler.cpp b/src/core/parser/stack/DocumentHandler.cpp
index a322028..3e85f72 100644
--- a/src/core/parser/stack/DocumentHandler.cpp
+++ b/src/core/parser/stack/DocumentHandler.cpp
@@ -128,6 +128,42 @@ void DocumentChildHandler::preamble(Rooted<Node> &parentNode, size_t &fieldIdx,
 	}
 }
 
+void DocumentChildHandler::pushScopeTokens()
+{
+	// List containing the unfiltered syntax descriptors
+	std::vector<SyntaxDescriptor> descrs;
+
+	// Fetch the current scope stack and search the first non-transparent field
+	// or structure
+	const NodeVector<Node> &stack = scope().getStack();
+	for (auto sit = stack.crbegin(); sit != stack.crend(); sit++) {
+		Rooted<Node> nd = *sit;
+
+		// TODO: Why can't this functionality be in a common base class?
+
+		// Check whether the field is transparent, if not, fetch the tokens
+		if (nd->isa(&RttiTypes::DocumentField)) {
+			Rooted<DocumentField> field = nd.cast<DocumentField>();
+			if (!field->transparent) {
+				descrs = field->getDescriptor()->getPermittedTokens();
+				break;
+			}
+		}
+
+		// Check whether the sturcture is transparent, if not, fetch the tokens
+		if (nd->isa(&RttiTypes::StructuredEntity)) {
+			Rooted<StructuredEntity> entity = nd.cast<StructuredEntity>();
+			if (!entity->isTransparent()) {
+				descrs = entity->getDescriptor()->getPermittedTokens();
+				break;
+			}
+		}
+	}
+
+	// Push the filtered tokens onto the stack
+	pushTokens(descrs);
+}
+
 void DocumentChildHandler::pushDocumentField(Handle<Node> parent,
                                              Handle<FieldDescriptor> fieldDescr,
                                              size_t fieldIdx, bool transparent)
@@ -196,6 +232,22 @@ void DocumentChildHandler::createPath(const size_t &firstFieldIdx,
 	scope().setFlag(ParserFlag::POST_EXPLICIT_FIELDS, false);
 }
 
+void DocumentChildHandler::rollbackPath()
+{
+	// Remove the topmost field
+	popDocumentField();
+
+	// Pop all remaining transparent elements.
+	while (scope().getLeaf()->isa(&RttiTypes::StructuredEntity) &&
+	       scope().getLeaf().cast<StructuredEntity>()->isTransparent()) {
+		// Pop the transparent element.
+		scope().pop(logger());
+
+		// Pop the transparent field.
+		popDocumentField();
+	}
+}
+
 static std::string extractNameAttribute(Variant::mapType &args)
 {
 	// Extract the special "name" attribute from the input arguments.
@@ -262,10 +314,11 @@ bool DocumentChildHandler::startCommand(Variant::mapType &args)
 						    std::string(
 						        "Data or structure commands have already been "
 						        "given, command \"") +
-						        name() + std::string(
-						                     "\" is not interpreted explicit "
-						                     "field. Move explicit field "
-						                     "references to the beginning."),
+						        name() +
+						        std::string(
+						            "\" is not interpreted as explicit "
+						            "field. Move explicit field "
+						            "references to the beginning."),
 						    location());
 					} else {
 						pushDocumentField(
@@ -273,6 +326,7 @@ bool DocumentChildHandler::startCommand(Variant::mapType &args)
 						    parent->getDescriptor()->getFieldDescriptor(
 						        newFieldIdx),
 						    newFieldIdx, false);
+						pushScopeTokens();
 						isExplicitField = true;
 						return true;
 					}
@@ -329,6 +383,8 @@ bool DocumentChildHandler::startCommand(Variant::mapType &args)
 		// Push the entity onto the stack
 		entity->setLocation(location());
 		scope().push(entity);
+		pushScopeTokens();
+
 		return true;
 	}
 }
@@ -389,7 +445,7 @@ bool DocumentChildHandler::startAnnotation(Variant::mapType &args)
 	Rooted<Anchor> anchor = parent->createChildAnchor(fieldIdx);
 	anchor->setLocation(location());
 
-	// resolve the AnnotationClass
+	// Resolve the AnnotationClass
 	Rooted<AnnotationClass> annoClass;
 	if (!name().empty()) {
 		annoClass = scope().resolve<AnnotationClass>(Utils::split(name(), ':'),
@@ -443,22 +499,142 @@ bool DocumentChildHandler::startAnnotation(Variant::mapType &args)
 
 bool DocumentChildHandler::startToken(Handle<Node> node)
 {
-	// TODO: Handle token start
-	return false;
+	bool isStruct = node->isa(&RttiTypes::StructuredClass);
+//	bool isField = node->isa(&RttiTypes::FieldDescriptor);
+//	bool isAnnotation = node->isa(&RttiTypes::AnnotationClass);
+
+	if (!isStruct) {
+		// TODO: Implement
+		return false;
+	}
+
+	Rooted<StructuredClass> strct = node.cast<StructuredClass>();
+
+	scope().setFlag(ParserFlag::POST_HEAD, true);
+	while (true) {
+		// Make sure the parent node is not the document
+		Rooted<Node> parentNode = scope().getLeaf();
+		if (parentNode->isa(&RttiTypes::Document)) {
+			logger().error("Tokens are not allowed on the root document level.");
+			return false;
+		}
+		assert(parentNode->isa(&RttiTypes::DocumentField));
+
+		// TODO: Move this to more generic method
+		// Fetch the parent document entity and the parent field index
+		size_t fieldIdx;
+		DocumentEntity *parent;
+		preamble(parentNode, fieldIdx, parent);
+
+		// Calculate a path if transparent entities are needed in between.
+		Rooted<FieldDescriptor> field = parent->getDescriptor()->getFieldDescriptor(fieldIdx);
+		size_t lastFieldIdx = fieldIdx;
+		auto pathRes = field->pathTo(strct, logger());
+		if (!pathRes.second) {
+			// If we have transparent elements above us in the structure tree,
+			// try to unwind them before we give up.
+			if (scope().getLeaf().cast<DocumentField>()->transparent) {
+				// Pop the implicit field.
+				popDocumentField();
+
+				// Pop the implicit element.
+				scope().pop(logger());
+				continue;
+			}
+			throw LoggableException(
+			    std::string("An instance of \"") + strct->getName() +
+			        "\" is not allowed as child of field \"" +
+			        field->getNameOrDefaultName() + "\" of descriptor \"" +
+			        parent->getDescriptor()->getName() + "\"",
+			    location());
+		}
+
+		// Create the path (if one is available)
+		if (!pathRes.first.empty()) {
+			createPath(lastFieldIdx, pathRes.first, parent);
+			lastFieldIdx =
+			    parent->getDescriptor()->getFieldDescriptorIndex();
+		}
+
+		// Create the entity for the new element at last.
+		Rooted<StructuredEntity> entity = parent->createChildStructuredEntity(
+			strct, lastFieldIdx, Variant::mapType{}, "");
+
+		// We're past the region in which explicit fields can be defined in the
+		// parent structure element
+		scope().setFlag(ParserFlag::POST_EXPLICIT_FIELDS, true);
+
+		// Push the entity onto the stack
+		entity->setLocation(location());
+		scope().push(entity);
+		pushScopeTokens();
+
+		return true;
+	}
 }
 
 DocumentChildHandler::EndTokenResult DocumentChildHandler::endToken(
     const Token &token, Handle<Node> node)
 {
-	// TODO: Handle token end
-	return EndTokenResult::ENDED_NONE;
+	// Iterate over the transparent elements in the scope stack
+	const NodeVector<Node> &stack = scope().getStack();
+	ssize_t depth = -1;
+	for (auto sit = stack.crbegin(); sit != stack.crend(); sit++, depth++) {
+		Rooted<Node> leaf = *sit;
+		if (leaf->isa(&RttiTypes::DocumentField)) {
+			Rooted<DocumentField> field = leaf.cast<DocumentField>();
+			if (field->getDescriptor() == node) {
+				// If the field is transparent, end it by incrementing the depth
+				// counter -- both the field itself and the consecutive element
+				// need to be removed
+				if (field->transparent) {
+					depth += 2;
+					break;
+				}
+				return EndTokenResult::ENDED_THIS;
+			}
+
+			// Abort if the field is explicit
+			if (!field->transparent) {
+				return EndTokenResult::ENDED_NONE;
+			}
+		}
+
+		if (leaf->isa(&RttiTypes::StructuredEntity)) {
+			Rooted<StructuredEntity> entity = leaf.cast<StructuredEntity>();
+			if (entity->getDescriptor() == node) {
+				// If the entity is transparent, end it by incrementing the
+				// depth counter and aborting
+				if (entity->isTransparent()) {
+					depth++;
+					break;
+				}
+				return EndTokenResult::ENDED_THIS;
+			}
+
+			// Abort if this entity is explicit
+			if (!entity->isTransparent()) {
+				return EndTokenResult::ENDED_NONE;
+			}
+		}
+
+		// TODO: End annotations!
+	}
+
+	// End all elements that were marked for being closed
+	for (ssize_t i = 0; i <= depth; i++) {
+		scope().pop(logger());
+	}
+	return (depth >= 0) ? EndTokenResult::ENDED_HIDDEN : EndTokenResult::ENDED_NONE;
 }
 
 void DocumentChildHandler::end()
 {
+	// Distinguish the handler type
 	switch (type()) {
 		case HandlerType::COMMAND:
 		case HandlerType::ANNOTATION_START:
+		case HandlerType::TOKEN:
 			// In case of explicit fields we do not want to pop something from
 			// the stack.
 			if (!isExplicitField) {
@@ -469,9 +645,6 @@ void DocumentChildHandler::end()
 		case HandlerType::ANNOTATION_END:
 			// We have nothing to pop from the stack
 			break;
-		case HandlerType::TOKEN:
-			// TODO
-			break;
 	}
 }
 
@@ -505,8 +678,10 @@ bool DocumentChildHandler::fieldStart(bool &isDefault, size_t fieldIdx)
 		}
 		isDefault = fieldIdx == fields.size() - 1;
 	}
+
 	// push the field on the stack.
 	pushDocumentField(parentNode, fields[fieldIdx], fieldIdx, false);
+	pushScopeTokens();
 
 	// Generally allow explicit fields in the new field
 	scope().setFlag(ParserFlag::POST_EXPLICIT_FIELDS, false);
@@ -516,19 +691,10 @@ bool DocumentChildHandler::fieldStart(bool &isDefault, size_t fieldIdx)
 
 void DocumentChildHandler::fieldEnd()
 {
-	assert(scope().getLeaf()->isa(&RttiTypes::DocumentField));
-
-	// Pop the field from the stack.
-	popDocumentField();
-
-	// Pop all remaining transparent elements.
-	while (scope().getLeaf()->isa(&RttiTypes::StructuredEntity) &&
-	       scope().getLeaf().cast<StructuredEntity>()->isTransparent()) {
-		// Pop the transparent element.
-		scope().pop(logger());
-		// Pop the transparent field.
-		popDocumentField();
+	if (!isExplicitField) {
+		popTokens();
 	}
+	rollbackPath();
 }
 
 bool DocumentChildHandler::convertData(Handle<FieldDescriptor> field,
@@ -580,7 +746,7 @@ bool DocumentChildHandler::data()
 	// If it is a primitive field directly, try to parse the content.
 	if (field->isPrimitive()) {
 		// Add it as primitive content.
-		Variant text = readData();
+		Variant text = readData(); // TODO: Eliminate readData method
 		if (!convertData(field, text, logger())) {
 			return false;
 		}
@@ -593,6 +759,7 @@ bool DocumentChildHandler::data()
 	// allow primitive content at this point and could be constructed via
 	// transparent intermediate entities.
 	NodeVector<FieldDescriptor> defaultFields = field->getDefaultFields();
+
 	// Try to parse the data using the type specified by the respective field.
 	// If that does not work we proceed to the next possible field.
 	std::vector<LoggerFork> forks;
@@ -600,9 +767,8 @@ bool DocumentChildHandler::data()
 		// Then try to parse the content using the type specification.
 		forks.emplace_back(logger().fork());
 
-		// TODO: Actually the data has to be read after the path has been
-		// created (as createPath may push more tokens onto the stack)
-		Variant text = readData();
+		// Try to parse the data
+		Variant text = readData(); // TODO: Eliminate readData method
 		if (!convertData(primitiveField, text, forks.back())) {
 			continue;
 		}
diff --git a/src/core/parser/stack/DocumentHandler.hpp b/src/core/parser/stack/DocumentHandler.hpp
index 5157709..0a32267 100644
--- a/src/core/parser/stack/DocumentHandler.hpp
+++ b/src/core/parser/stack/DocumentHandler.hpp
@@ -159,6 +159,12 @@ private:
 	                DocumentEntity *&parent);
 
 	/**
+	 * Removes the transparent elements created by the createPath() method from
+	 * the Scope.
+	 */
+	void rollbackPath();
+
+	/**
 	 * Tries to convert the given data to the type that is specified in the
 	 * given primitive field.
 	 *
@@ -174,6 +180,17 @@ private:
 	                 Logger &logger);
 
 	/**
+	 * Fetches the top-most non-transparent descriptor from the scope, gets the
+	 * permitted tokens of this descriptor and filters them to those tokens
+	 * which are actually possible, according to the current content of the
+	 * ParserScope.
+	 *
+	 * @param fieldDescr is the field descriptor of which the tokens should be
+	 * registered.
+	 */
+	void pushScopeTokens();
+
+	/**
 	 * Pushes a new DocumentField onto the scope stack and registers all
 	 * premitted tokens in the parser.
 	 */
diff --git a/src/core/parser/stack/Stack.cpp b/src/core/parser/stack/Stack.cpp
index e027271..54be719 100644
--- a/src/core/parser/stack/Stack.cpp
+++ b/src/core/parser/stack/Stack.cpp
@@ -62,6 +62,17 @@ public:
 	size_t fieldIdx;
 
 	/**
+	 * TokenId of the close token.
+	 */
+	TokenId closeToken;
+
+	/**
+	 * Descriptor corresponding to the associated "close" token, set to nullptr
+	 * if not needed.
+	 */
+	Rooted<Node> tokenDesciptor;
+
+	/**
 	 * Set to true if the handler is valid (which is the case if the "start"
 	 * method has returned true). If the handler is invalid, no more calls are
 	 * directed at it until it can be removed from the stack.
@@ -169,6 +180,7 @@ HandlerInfo::HandlerInfo() : HandlerInfo(nullptr) {}
 HandlerInfo::HandlerInfo(std::shared_ptr<Handler> handler)
     : handler(handler),
       fieldIdx(0),
+      closeToken(Tokens::Empty),
       valid(true),
       implicit(false),
       range(false),
@@ -184,6 +196,7 @@ HandlerInfo::HandlerInfo(bool implicit, bool inField, bool inDefaultField,
                          bool inImplicitDefaultField)
     : handler(nullptr),
       fieldIdx(0),
+      closeToken(Tokens::Empty),
       valid(true),
       implicit(implicit),
       range(false),
@@ -382,6 +395,15 @@ private:
 	HandlerInfo &lastInfo();
 
 	/**
+	 * Returns a list containing the currently pending close tokens.
+	 *
+	 * @param token is a TokenId for which the result list should be filtered.
+	 * If set to Tokens::Empty, all tokens are returned.
+	 */
+	std::vector<SyntaxDescriptor> pendingCloseTokens(
+	    TokenId token = Tokens::Empty) const;
+
+	/**
 	 * Returns a set containing the tokens that should currently be processed
 	 * by the TokenizedData instance.
 	 *
@@ -396,6 +418,11 @@ private:
 	WhitespaceMode currentWhitespaceMode() const;
 
 	/**
+	 * Ends a currently open field of the current handler.
+	 */
+	void endCurrentField();
+
+	/**
 	 * Ends the current handler and removes the corresponding element from the
 	 * stack.
 	 *
@@ -411,9 +438,16 @@ private:
 	 * startToken(), startCommand(), annotationStart() or annotationEnd() events
 	 * are reached.
 	 *
+	 * @param startImplicitDefaultField if set to true, starts a new default
+	 * field.
+	 * @param endHandlersWithoutDefaultField important if
+	 * startImplicitDefaultField is set to false. If false, prevents this method
+	 * from ending a handler if it potentially can have a default field, but did
+	 * not have one yet.
 	 * @return true if the current command is in a valid field.
 	 */
-	bool prepareCurrentHandler(bool startImplicitDefaultField = true);
+	bool prepareCurrentHandler(bool startImplicitDefaultField = true,
+	                           bool endHandlersWithoutDefaultField = true);
 
 	/**
 	 * Returns true if all handlers on the stack are currently valid, or false
@@ -427,7 +461,7 @@ private:
 	 * Called whenever there is an actual data pending on the current
 	 * TokenizedDataReader. Tries to feed this data to the current handler.
 	 */
-	void handleData();
+	bool handleData();
 
 	/**
 	 * Called whenever the annotationStart or annotationEnd methods are called.
@@ -441,6 +475,31 @@ private:
 	                              bool range, HandlerType type);
 
 	/**
+	 * Called by handleToken in order to process a list of potential "close"
+	 * tokens.
+	 *
+	 * @param token is the matching token.
+	 * @param descrs is the list with SyntaxDescriptors that should be used.
+	 * @return true if something was closed, false otherwise.
+	 */
+	bool handleCloseTokens(const Token &token,
+	                       const std::vector<SyntaxDescriptor> &descrs);
+
+	/**
+	 * Called by handleToken in order to process a list of potential "open"
+	 * tokens.
+	 *
+	 * @param logger responsible for receiving error messages.
+	 * @param token is the matching token.
+	 * @param shortForm is set to true, if the descriptors describe a short form
+	 * token instead of an open token.
+	 * @param descrs is the list with SyntaxDescriptors that should be used.
+	 * @return true if something was opened, false otherwise.
+	 */
+	bool handleOpenTokens(Logger &logger, const Token &token, bool shortForm,
+	                      const std::vector<SyntaxDescriptor> &descrs);
+
+	/**
 	 * Called whenever there is a token waiting to be processed. If possible
 	 * tries to end a current handler with this token or to start a new handler
 	 * with the token.
@@ -479,6 +538,7 @@ public:
 
 	TokenId registerToken(const std::string &token) override;
 	void unregisterToken(TokenId id) override;
+	bool readToken(Token &token);
 	Variant readData() override;
 	void pushTokens(const std::vector<SyntaxDescriptor> &tokens) override;
 	void popTokens() override;
@@ -603,12 +663,36 @@ std::string StackImpl::currentCommandName() const
 	return stack.empty() ? std::string{} : stack.back().name();
 }
 
+std::vector<SyntaxDescriptor> StackImpl::pendingCloseTokens(TokenId token) const
+{
+	// TODO: Are there cases in which the returned vector will contain more
+	// than one element?
+	std::vector<SyntaxDescriptor> res;
+	for (auto it = stack.crbegin(); it != stack.crend(); it++) {
+		if (it->closeToken != Tokens::Empty &&
+		    (token == Tokens::Empty || token == it->closeToken)) {
+			res.push_back(SyntaxDescriptor(Tokens::Empty, it->closeToken,
+			                               Tokens::Empty, it->tokenDesciptor,
+			                               -1));
+		}
+		if (it->range || it->inField) {
+			break;
+		}
+	}
+	return res;
+}
+
 TokenSet StackImpl::currentTokens() const
 {
+	TokenSet res;
 	if (currentInfo().state().supportsTokens) {
-		return tokenStack.tokens();
+		res = tokenStack.tokens();
+		std::vector<SyntaxDescriptor> pending = pendingCloseTokens();
+		for (const auto &descr : pending) {
+			res.insert(descr.close);
+		}
 	}
-	return TokenSet{};
+	return res;
 }
 
 WhitespaceMode StackImpl::currentWhitespaceMode() const
@@ -634,6 +718,19 @@ HandlerInfo &StackImpl::lastInfo()
 
 /* Stack helper functions */
 
+void StackImpl::endCurrentField()
+{
+	if (!stack.empty()) {
+		HandlerInfo &info = stack.back();
+		if (!info.implicit && handlersValid() && info.inField) {
+			if (info.inValidField) {
+				info.handler->fieldEnd();
+			}
+			info.fieldEnd();
+		}
+	}
+}
+
 bool StackImpl::endCurrentHandler()
 {
 	if (!stack.empty()) {
@@ -643,14 +740,8 @@ bool StackImpl::endCurrentHandler()
 		// Do not call any callback functions while the stack is marked as
 		// invalid or this is an elment marked as "implicit"
 		if (!info.implicit && handlersValid()) {
-			// Make sure the fieldEnd handler is called if the element still
-			// is in a field
-			if (info.inField) {
-				if (info.inValidField) {
-					info.handler->fieldEnd();
-				}
-				info.fieldEnd();
-			}
+			// End all currently open fields
+			endCurrentField();
 
 			// Call the "end" function of the corresponding Handler instance
 			info.handler->end();
@@ -663,7 +754,8 @@ bool StackImpl::endCurrentHandler()
 	return false;
 }
 
-bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField)
+bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField,
+                                      bool endHandlersWithoutDefaultField)
 {
 	// Repeat until a valid handler is found on the stack
 	while (!stack.empty()) {
@@ -682,8 +774,9 @@ bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField)
 		    info.type() == HandlerType::COMMAND ||
 		    info.type() == HandlerType::TOKEN ||
 		    (info.type() == HandlerType::ANNOTATION_START && info.range);
-		if (info.hadDefaultField || !startImplicitDefaultField || !info.valid ||
-		    !canHaveImplicitDefaultField) {
+		if (info.hadDefaultField ||
+		    (!startImplicitDefaultField && endHandlersWithoutDefaultField) ||
+		    !info.valid || !canHaveImplicitDefaultField) {
 			// We cannot end the command if it is marked as "range" command
 			if (info.range) {
 				return false;
@@ -695,16 +788,18 @@ bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField)
 		}
 
 		// Try to start a new default field, abort if this did not work
-		bool isDefault = true;
-		if (!info.handler->fieldStart(isDefault, info.fieldIdx)) {
-			endCurrentHandler();
-			continue;
-		}
+		if (startImplicitDefaultField) {
+			bool isDefault = true;
+			if (!info.handler->fieldStart(isDefault, info.fieldIdx)) {
+				endCurrentHandler();
+				continue;
+			}
 
-		// Mark the field as started and return -- the field should be marked
-		// is implicit if this is not a field with range
-		info.fieldStart(true, !info.range, true);
-		return true;
+			// Mark the field as started and return
+			info.fieldStart(true, !info.range, true);
+			return true;
+		}
+		return false;
 	}
 	return false;
 }
@@ -719,7 +814,7 @@ bool StackImpl::handlersValid()
 	return true;
 }
 
-void StackImpl::handleData()
+bool StackImpl::handleData()
 {
 	// Repeat until we found some handle willingly consuming the data
 	while (true) {
@@ -738,12 +833,18 @@ void StackImpl::handleData()
 			if (!info.hadDefaultField) {
 				logger().error("Did not expect any data here");
 			}
-			return;
+			return true;
 		}
 
 		// If we're currently in an invalid subtree, just eat the data and abort
 		if (!handlersValid()) {
-			return;
+			return true;
+		}
+
+		// Check whether "readData" still returns data and not an empty string
+		// (because a token has now become valid)
+		if (!readData().isString()) {
+			return false;
 		}
 
 		// Fork the logger and set it as temporary logger for the "data"
@@ -774,15 +875,175 @@ void StackImpl::handleData()
 
 		// Commit the content of the logger fork. Do not change the valid flag.
 		loggerFork.commit();
-		return;
+		return true;
 	}
 }
 
+/* Token management */
+
+static void strayTokenNote(const std::string &preamble,
+                           const std::vector<SyntaxDescriptor> &descrs,
+                           Logger &logger)
+{
+	for (const auto &d : descrs) {
+		std::string type = "";
+		if (d.isAnnotation()) {
+			type = " annotation";
+		} else if (d.isFieldDescriptor()) {
+			type = " field";
+		} else if (d.isStruct()) {
+			type = " structure";
+		}
+		logger.note(preamble + " \"" + d.descriptor->getName() + "\"" + type +
+		                ", as specified here",
+		            *(d.descriptor));
+	}
+}
+
+static void strayTokenError(const Token &token, TokenDescriptor &descr,
+                            Logger &logger)
+{
+	logger.error("Stray token", token);
+	logger.note("This token must be used in one of the following contexts:",
+	            token, MessageMode::NO_CONTEXT);
+	strayTokenNote("To close a", descr.close, logger);
+	strayTokenNote("To open a", descr.open, logger);
+	strayTokenNote("As a short form of", descr.shortForm, logger);
+
+	return;
+}
+
+bool StackImpl::handleCloseTokens(const Token &token,
+                                  const std::vector<SyntaxDescriptor> &descrs)
+{
+	// Fetch the current information
+	HandlerInfo &info = currentInfo();
+
+	// Iterate over all possible SyntaxDescriptors and try to end the token
+	for (const SyntaxDescriptor &descr : descrs) {
+		Handler::EndTokenResult res =
+		    info.handler->endToken(token.id, descr.descriptor);
+		switch (res) {
+			case Handler::EndTokenResult::ENDED_THIS:
+				endCurrentHandler();
+				return true;
+			case Handler::EndTokenResult::ENDED_HIDDEN:
+				return true;
+			case Handler::EndTokenResult::ENDED_NONE:
+				break;
+		}
+	}
+	return false;
+}
+
+bool StackImpl::handleOpenTokens(Logger &logger, const Token &token,
+                                 bool shortForm,
+                                 const std::vector<SyntaxDescriptor> &descrs)
+{
+	// Make sure we currently are in a field
+	if (!currentInfo().inField) {
+		throw LoggableException("Cannot start a command here", token);
+	}
+
+	// Iterate over all given descriptors
+	for (const SyntaxDescriptor &descr : descrs) {
+		// Find the special target state TODO: Is there some better solution?
+		const State *state = findTargetState("*");
+		if (state == nullptr) {
+			throw LoggableException("Cannot handle start tokens here", token);
+		}
+
+		// Instantiate the handler and push it onto the stack
+		HandlerConstructor ctor = state->elementHandler ? state->elementHandler
+		                                                : EmptyHandler::create;
+		std::shared_ptr<Handler> handler{
+		    ctor({ctx, *this, *state, token, HandlerType::TOKEN})};
+		stack.emplace_back(handler);
+
+		// Call the startAnnotation method of the newly created handler, store
+		// the valid flag
+		HandlerInfo &info = currentInfo();
+		info.valid = false;
+		try {
+			info.valid = handler->startToken(descr.descriptor);
+		}
+		catch (LoggableException ex) {
+			logger.log(ex);
+		}
+
+		// End the handler again if an error occured
+		if (!info.valid) {
+			endCurrentHandler();
+		}
+
+		// If this is not a short form token and the "close" descriptor is
+		// given, mark the current handler as "range" handler
+		if (!shortForm && descr.close != Tokens::Empty) {
+			info.closeToken = descr.close;
+			info.tokenDesciptor = descr.descriptor;
+			info.range = true;
+		}
+		return true;
+	}
+	return false;
+}
+
 void StackImpl::handleToken(const Token &token)
 {
-	std::cout << "Got token " << token.id << std::endl;
-	// TODO: Implement
-	// Just eat them for now
+	// Fetch the TokenDescriptor and the "pendingClose" list
+	TokenDescriptor descr = tokenStack.lookup(token.id);
+	std::vector<SyntaxDescriptor> pendingClose = pendingCloseTokens(token.id);
+
+	// Iterate until the stack can no longer be unwound
+	while (true) {
+		LoggerFork loggerFork = logger().fork();
+
+		bool hadError = false;
+		try {
+			// Try to close the current handlers
+			if (handleCloseTokens(token, pendingClose) ||
+			    (pendingClose.empty() &&
+			     handleCloseTokens(token, descr.close))) {
+				return;
+			}
+
+			// Try to open an implicit default field and try to start the token
+			// as short form or as start token
+			prepareCurrentHandler(pendingClose.empty());
+			if (pendingClose.empty()) {
+				if (handleOpenTokens(loggerFork, token, true,
+				                     descr.shortForm) ||
+				    handleOpenTokens(loggerFork, token, false, descr.open)) {
+					return;
+				}
+			}
+		}
+		catch (LoggableException ex) {
+			hadError = true;
+			loggerFork.log(ex);
+		}
+
+		// Neither of the above worked, try to unroll the stack
+		HandlerInfo &info = currentInfo();
+		if (info.inImplicitDefaultField && !stack.empty()) {
+			endCurrentHandler();
+		} else if (info.inDefaultField && info.closeToken == token.id) {
+			endCurrentField();
+		} else {
+			// Ignore close-only special (whitespace) rules, in all other cases
+			// issue an error
+			if (!Token::isSpecial(token.id) || descr.close.empty() ||
+			    !descr.open.empty() || !descr.shortForm.empty()) {
+				loggerFork.commit();
+
+				// If there was no other error, issue a "stray token" error
+				if (!hadError) {
+					strayTokenError(token, descr, logger());
+				}
+			}
+			return;
+		}
+	}
 }
 
 void StackImpl::handleFieldEnd(bool endRange)
@@ -872,7 +1133,8 @@ void StackImpl::handleAnnotationStartEnd(const Variant &name,
 	info.valid = false;
 	try {
 		info.valid = handler->startAnnotation(args);
-	} catch (LoggableException ex) {
+	}
+	catch (LoggableException ex) {
 		logger().log(ex);
 	}
 	info.range = range;
@@ -1007,18 +1269,27 @@ void StackImpl::data(const TokenizedData &data)
 	// dataReader is resetted to nullptr once this scope is left.
 	GuardedTemporaryPointer<TokenizedDataReader> ptr(&reader, &dataReader);
 
+	// Close all handlers that did already had or cannot have a default field
+	// and are not currently inside a field (repeat this after each chunk of
+	// data/text)
+	prepareCurrentHandler(false, false);
+
 	// Peek a token from the reader, repeat until all tokens have been read
 	Token token;
-	while (reader.peek(token, currentTokens(), currentWhitespaceMode())) {
+
+	while (readToken(token)) {
 		// Handle the token as text data or as actual token
 		if (token.id == Tokens::Data) {
-			handleData();
+			// Only consume the data if reading was sucessful -- sometimes (as
+			// it turns out) -- there is no data
+			if (handleData()) {
+				reader.consumePeek();
+			}
 		} else {
 			handleToken(token);
+			reader.consumePeek();
 		}
-
-		// Consume the peeked token
-		reader.consumePeek();
+		prepareCurrentHandler(false, false);
 	}
 }
 
@@ -1100,17 +1371,24 @@ void StackImpl::popTokens()
 	tokenStack.popTokens();
 }
 
-Variant StackImpl::readData()
+bool StackImpl::readToken(Token &token)
 {
 	if (dataReader != nullptr) {
-		TokenizedDataReaderFork dataReaderFork = dataReader->fork();
-		Token token;
-		dataReaderFork.read(token, currentTokens(), currentWhitespaceMode());
-		if (token.id == Tokens::Data) {
-			Variant res = Variant::fromString(token.content);
-			res.setLocation(token.getLocation());
-			return res;
-		}
+		dataReader->resetPeek();
+		return dataReader->peek(token, currentTokens(),
+		                        currentWhitespaceMode());
+	}
+	return false;
+}
+
+Variant StackImpl::readData()
+{
+	Token token;
+	if (readToken(token) && token.id == Tokens::Data) {
+		// TODO: Introduce function for string variant with location
+		Variant res = Variant::fromString(token.content);
+		res.setLocation(token.getLocation());
+		return res;
 	}
 	return Variant{};
 }