First implementation of user defined syntax, many features still missing and probably many bugs

* Implement startToken and endToken in DocumentChildHandler
* Implement pushScopeToken, which pushes tokens for the element that is currently on top of the Scope stack onto the token stack
* Implement rollbackPath() method (was really needed once in the development process, but only used in one place for now)
* Push and pop tokens from stack whenever a new explicit field or command is created/ended. Take advantage of the fact, that the tokens for transparent structures are always included in the token list
* Remember pending close tokens in the HandlerInfo structure in StackImpl
* Implement handleToken() in StackImpl
* Implement readToken() method used by readData() and data() in StackImpl
* Check whether there still is data available in handleData()
* Plus many more changes in the affected files...

1 files changed, 324 insertions, 46 deletions

diff --git a/src/core/parser/stack/Stack.cpp b/src/core/parser/stack/Stack.cpp
index e027271..54be719 100644
--- a/src/core/parser/stack/Stack.cpp
+++ b/src/core/parser/stack/Stack.cpp
@@ -62,6 +62,17 @@ public:
 	size_t fieldIdx;
 
 	/**
+	 * TokenId of the close token.
+	 */
+	TokenId closeToken;
+
+	/**
+	 * Descriptor corresponding to the associated "close" token, set to nullptr
+	 * if not needed.
+	 */
+	Rooted<Node> tokenDesciptor;
+
+	/**
 	 * Set to true if the handler is valid (which is the case if the "start"
 	 * method has returned true). If the handler is invalid, no more calls are
 	 * directed at it until it can be removed from the stack.
@@ -169,6 +180,7 @@ HandlerInfo::HandlerInfo() : HandlerInfo(nullptr) {}
 HandlerInfo::HandlerInfo(std::shared_ptr<Handler> handler)
     : handler(handler),
       fieldIdx(0),
+      closeToken(Tokens::Empty),
       valid(true),
       implicit(false),
       range(false),
@@ -184,6 +196,7 @@ HandlerInfo::HandlerInfo(bool implicit, bool inField, bool inDefaultField,
                          bool inImplicitDefaultField)
     : handler(nullptr),
       fieldIdx(0),
+      closeToken(Tokens::Empty),
       valid(true),
       implicit(implicit),
       range(false),
@@ -382,6 +395,15 @@ private:
 	HandlerInfo &lastInfo();
 
 	/**
+	 * Returns a list containing the currently pending close tokens.
+	 *
+	 * @param token is a TokenId for which the result list should be filtered.
+	 * If set to Tokens::Empty, all tokens are returned.
+	 */
+	std::vector<SyntaxDescriptor> pendingCloseTokens(
+	    TokenId token = Tokens::Empty) const;
+
+	/**
 	 * Returns a set containing the tokens that should currently be processed
 	 * by the TokenizedData instance.
 	 *
@@ -396,6 +418,11 @@ private:
 	WhitespaceMode currentWhitespaceMode() const;
 
 	/**
+	 * Ends a currently open field of the current handler.
+	 */
+	void endCurrentField();
+
+	/**
 	 * Ends the current handler and removes the corresponding element from the
 	 * stack.
 	 *
@@ -411,9 +438,16 @@ private:
 	 * startToken(), startCommand(), annotationStart() or annotationEnd() events
 	 * are reached.
 	 *
+	 * @param startImplicitDefaultField if set to true, starts a new default
+	 * field.
+	 * @param endHandlersWithoutDefaultField important if
+	 * startImplicitDefaultField is set to false. If false, prevents this method
+	 * from ending a handler if it potentially can have a default field, but did
+	 * not have one yet.
 	 * @return true if the current command is in a valid field.
 	 */
-	bool prepareCurrentHandler(bool startImplicitDefaultField = true);
+	bool prepareCurrentHandler(bool startImplicitDefaultField = true,
+	                           bool endHandlersWithoutDefaultField = true);
 
 	/**
 	 * Returns true if all handlers on the stack are currently valid, or false
@@ -427,7 +461,7 @@ private:
 	 * Called whenever there is an actual data pending on the current
 	 * TokenizedDataReader. Tries to feed this data to the current handler.
 	 */
-	void handleData();
+	bool handleData();
 
 	/**
 	 * Called whenever the annotationStart or annotationEnd methods are called.
@@ -441,6 +475,31 @@ private:
 	                              bool range, HandlerType type);
 
 	/**
+	 * Called by handleToken in order to process a list of potential "close"
+	 * tokens.
+	 *
+	 * @param token is the matching token.
+	 * @param descrs is the list with SyntaxDescriptors that should be used.
+	 * @return true if something was closed, false otherwise.
+	 */
+	bool handleCloseTokens(const Token &token,
+	                       const std::vector<SyntaxDescriptor> &descrs);
+
+	/**
+	 * Called by handleToken in order to process a list of potential "open"
+	 * tokens.
+	 *
+	 * @param logger responsible for receiving error messages.
+	 * @param token is the matching token.
+	 * @param shortForm is set to true, if the descriptors describe a short form
+	 * token instead of an open token.
+	 * @param descrs is the list with SyntaxDescriptors that should be used.
+	 * @return true if something was opened, false otherwise.
+	 */
+	bool handleOpenTokens(Logger &logger, const Token &token, bool shortForm,
+	                      const std::vector<SyntaxDescriptor> &descrs);
+
+	/**
 	 * Called whenever there is a token waiting to be processed. If possible
 	 * tries to end a current handler with this token or to start a new handler
 	 * with the token.
@@ -479,6 +538,7 @@ public:
 
 	TokenId registerToken(const std::string &token) override;
 	void unregisterToken(TokenId id) override;
+	bool readToken(Token &token);
 	Variant readData() override;
 	void pushTokens(const std::vector<SyntaxDescriptor> &tokens) override;
 	void popTokens() override;
@@ -603,12 +663,36 @@ std::string StackImpl::currentCommandName() const
 	return stack.empty() ? std::string{} : stack.back().name();
 }
 
+std::vector<SyntaxDescriptor> StackImpl::pendingCloseTokens(TokenId token) const
+{
+	// TODO: Are there cases in which the returned vector will contain more
+	// than one element?
+	std::vector<SyntaxDescriptor> res;
+	for (auto it = stack.crbegin(); it != stack.crend(); it++) {
+		if (it->closeToken != Tokens::Empty &&
+		    (token == Tokens::Empty || token == it->closeToken)) {
+			res.push_back(SyntaxDescriptor(Tokens::Empty, it->closeToken,
+			                               Tokens::Empty, it->tokenDesciptor,
+			                               -1));
+		}
+		if (it->range || it->inField) {
+			break;
+		}
+	}
+	return res;
+}
+
 TokenSet StackImpl::currentTokens() const
 {
+	TokenSet res;
 	if (currentInfo().state().supportsTokens) {
-		return tokenStack.tokens();
+		res = tokenStack.tokens();
+		std::vector<SyntaxDescriptor> pending = pendingCloseTokens();
+		for (const auto &descr : pending) {
+			res.insert(descr.close);
+		}
 	}
-	return TokenSet{};
+	return res;
 }
 
 WhitespaceMode StackImpl::currentWhitespaceMode() const
@@ -634,6 +718,19 @@ HandlerInfo &StackImpl::lastInfo()
 
 /* Stack helper functions */
 
+void StackImpl::endCurrentField()
+{
+	if (!stack.empty()) {
+		HandlerInfo &info = stack.back();
+		if (!info.implicit && handlersValid() && info.inField) {
+			if (info.inValidField) {
+				info.handler->fieldEnd();
+			}
+			info.fieldEnd();
+		}
+	}
+}
+
 bool StackImpl::endCurrentHandler()
 {
 	if (!stack.empty()) {
@@ -643,14 +740,8 @@ bool StackImpl::endCurrentHandler()
 		// Do not call any callback functions while the stack is marked as
 		// invalid or this is an elment marked as "implicit"
 		if (!info.implicit && handlersValid()) {
-			// Make sure the fieldEnd handler is called if the element still
-			// is in a field
-			if (info.inField) {
-				if (info.inValidField) {
-					info.handler->fieldEnd();
-				}
-				info.fieldEnd();
-			}
+			// End all currently open fields
+			endCurrentField();
 
 			// Call the "end" function of the corresponding Handler instance
 			info.handler->end();
@@ -663,7 +754,8 @@ bool StackImpl::endCurrentHandler()
 	return false;
 }
 
-bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField)
+bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField,
+                                      bool endHandlersWithoutDefaultField)
 {
 	// Repeat until a valid handler is found on the stack
 	while (!stack.empty()) {
@@ -682,8 +774,9 @@ bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField)
 		    info.type() == HandlerType::COMMAND ||
 		    info.type() == HandlerType::TOKEN ||
 		    (info.type() == HandlerType::ANNOTATION_START && info.range);
-		if (info.hadDefaultField || !startImplicitDefaultField || !info.valid ||
-		    !canHaveImplicitDefaultField) {
+		if (info.hadDefaultField ||
+		    (!startImplicitDefaultField && endHandlersWithoutDefaultField) ||
+		    !info.valid || !canHaveImplicitDefaultField) {
 			// We cannot end the command if it is marked as "range" command
 			if (info.range) {
 				return false;
@@ -695,16 +788,18 @@ bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField)
 		}
 
 		// Try to start a new default field, abort if this did not work
-		bool isDefault = true;
-		if (!info.handler->fieldStart(isDefault, info.fieldIdx)) {
-			endCurrentHandler();
-			continue;
-		}
+		if (startImplicitDefaultField) {
+			bool isDefault = true;
+			if (!info.handler->fieldStart(isDefault, info.fieldIdx)) {
+				endCurrentHandler();
+				continue;
+			}
 
-		// Mark the field as started and return -- the field should be marked
-		// is implicit if this is not a field with range
-		info.fieldStart(true, !info.range, true);
-		return true;
+			// Mark the field as started and return
+			info.fieldStart(true, !info.range, true);
+			return true;
+		}
+		return false;
 	}
 	return false;
 }
@@ -719,7 +814,7 @@ bool StackImpl::handlersValid()
 	return true;
 }
 
-void StackImpl::handleData()
+bool StackImpl::handleData()
 {
 	// Repeat until we found some handle willingly consuming the data
 	while (true) {
@@ -738,12 +833,18 @@ void StackImpl::handleData()
 			if (!info.hadDefaultField) {
 				logger().error("Did not expect any data here");
 			}
-			return;
+			return true;
 		}
 
 		// If we're currently in an invalid subtree, just eat the data and abort
 		if (!handlersValid()) {
-			return;
+			return true;
+		}
+
+		// Check whether "readData" still returns data and not an empty string
+		// (because a token has now become valid)
+		if (!readData().isString()) {
+			return false;
 		}
 
 		// Fork the logger and set it as temporary logger for the "data"
@@ -774,15 +875,175 @@ void StackImpl::handleData()
 
 		// Commit the content of the logger fork. Do not change the valid flag.
 		loggerFork.commit();
-		return;
+		return true;
 	}
 }
 
+/* Token management */
+
+static void strayTokenNote(const std::string &preamble,
+                           const std::vector<SyntaxDescriptor> &descrs,
+                           Logger &logger)
+{
+	for (const auto &d : descrs) {
+		std::string type = "";
+		if (d.isAnnotation()) {
+			type = " annotation";
+		} else if (d.isFieldDescriptor()) {
+			type = " field";
+		} else if (d.isStruct()) {
+			type = " structure";
+		}
+		logger.note(preamble + " \"" + d.descriptor->getName() + "\"" + type +
+		                ", as specified here",
+		            *(d.descriptor));
+	}
+}
+
+static void strayTokenError(const Token &token, TokenDescriptor &descr,
+                            Logger &logger)
+{
+	logger.error("Stray token", token);
+	logger.note("This token must be used in one of the following contexts:",
+	            token, MessageMode::NO_CONTEXT);
+	strayTokenNote("To close a", descr.close, logger);
+	strayTokenNote("To open a", descr.open, logger);
+	strayTokenNote("As a short form of", descr.shortForm, logger);
+
+	return;
+}
+
+bool StackImpl::handleCloseTokens(const Token &token,
+                                  const std::vector<SyntaxDescriptor> &descrs)
+{
+	// Fetch the current information
+	HandlerInfo &info = currentInfo();
+
+	// Iterate over all possible SyntaxDescriptors and try to end the token
+	for (const SyntaxDescriptor &descr : descrs) {
+		Handler::EndTokenResult res =
+		    info.handler->endToken(token.id, descr.descriptor);
+		switch (res) {
+			case Handler::EndTokenResult::ENDED_THIS:
+				endCurrentHandler();
+				return true;
+			case Handler::EndTokenResult::ENDED_HIDDEN:
+				return true;
+			case Handler::EndTokenResult::ENDED_NONE:
+				break;
+		}
+	}
+	return false;
+}
+
+bool StackImpl::handleOpenTokens(Logger &logger, const Token &token,
+                                 bool shortForm,
+                                 const std::vector<SyntaxDescriptor> &descrs)
+{
+	// Make sure we currently are in a field
+	if (!currentInfo().inField) {
+		throw LoggableException("Cannot start a command here", token);
+	}
+
+	// Iterate over all given descriptors
+	for (const SyntaxDescriptor &descr : descrs) {
+		// Find the special target state TODO: Is there some better solution?
+		const State *state = findTargetState("*");
+		if (state == nullptr) {
+			throw LoggableException("Cannot handle start tokens here", token);
+		}
+
+		// Instantiate the handler and push it onto the stack
+		HandlerConstructor ctor = state->elementHandler ? state->elementHandler
+		                                                : EmptyHandler::create;
+		std::shared_ptr<Handler> handler{
+		    ctor({ctx, *this, *state, token, HandlerType::TOKEN})};
+		stack.emplace_back(handler);
+
+		// Call the startAnnotation method of the newly created handler, store
+		// the valid flag
+		HandlerInfo &info = currentInfo();
+		info.valid = false;
+		try {
+			info.valid = handler->startToken(descr.descriptor);
+		}
+		catch (LoggableException ex) {
+			logger.log(ex);
+		}
+
+		// End the handler again if an error occured
+		if (!info.valid) {
+			endCurrentHandler();
+		}
+
+		// If this is not a short form token and the "close" descriptor is
+		// given, mark the current handler as "range" handler
+		if (!shortForm && descr.close != Tokens::Empty) {
+			info.closeToken = descr.close;
+			info.tokenDesciptor = descr.descriptor;
+			info.range = true;
+		}
+		return true;
+	}
+	return false;
+}
+
 void StackImpl::handleToken(const Token &token)
 {
-	std::cout << "Got token " << token.id << std::endl;
-	// TODO: Implement
-	// Just eat them for now
+	// Fetch the TokenDescriptor and the "pendingClose" list
+	TokenDescriptor descr = tokenStack.lookup(token.id);
+	std::vector<SyntaxDescriptor> pendingClose = pendingCloseTokens(token.id);
+
+	// Iterate until the stack can no longer be unwound
+	while (true) {
+		LoggerFork loggerFork = logger().fork();
+
+		bool hadError = false;
+		try {
+			// Try to close the current handlers
+			if (handleCloseTokens(token, pendingClose) ||
+			    (pendingClose.empty() &&
+			     handleCloseTokens(token, descr.close))) {
+				return;
+			}
+
+			// Try to open an implicit default field and try to start the token
+			// as short form or as start token
+			prepareCurrentHandler(pendingClose.empty());
+			if (pendingClose.empty()) {
+				if (handleOpenTokens(loggerFork, token, true,
+				                     descr.shortForm) ||
+				    handleOpenTokens(loggerFork, token, false, descr.open)) {
+					return;
+				}
+			}
+		}
+		catch (LoggableException ex) {
+			hadError = true;
+			loggerFork.log(ex);
+		}
+
+		// Neither of the above worked, try to unroll the stack
+		HandlerInfo &info = currentInfo();
+		if (info.inImplicitDefaultField && !stack.empty()) {
+			endCurrentHandler();
+		} else if (info.inDefaultField && info.closeToken == token.id) {
+			endCurrentField();
+		} else {
+			// Ignore close-only special (whitespace) rules, in all other cases
+			// issue an error
+			if (!Token::isSpecial(token.id) || descr.close.empty() ||
+			    !descr.open.empty() || !descr.shortForm.empty()) {
+				loggerFork.commit();
+
+				// If there was no other error, issue a "stray token" error
+				if (!hadError) {
+					strayTokenError(token, descr, logger());
+				}
+			}
+			return;
+		}
+	}
 }
 
 void StackImpl::handleFieldEnd(bool endRange)
@@ -872,7 +1133,8 @@ void StackImpl::handleAnnotationStartEnd(const Variant &name,
 	info.valid = false;
 	try {
 		info.valid = handler->startAnnotation(args);
-	} catch (LoggableException ex) {
+	}
+	catch (LoggableException ex) {
 		logger().log(ex);
 	}
 	info.range = range;
@@ -1007,18 +1269,27 @@ void StackImpl::data(const TokenizedData &data)
 	// dataReader is resetted to nullptr once this scope is left.
 	GuardedTemporaryPointer<TokenizedDataReader> ptr(&reader, &dataReader);
 
+	// Close all handlers that did already had or cannot have a default field
+	// and are not currently inside a field (repeat this after each chunk of
+	// data/text)
+	prepareCurrentHandler(false, false);
+
 	// Peek a token from the reader, repeat until all tokens have been read
 	Token token;
-	while (reader.peek(token, currentTokens(), currentWhitespaceMode())) {
+
+	while (readToken(token)) {
 		// Handle the token as text data or as actual token
 		if (token.id == Tokens::Data) {
-			handleData();
+			// Only consume the data if reading was sucessful -- sometimes (as
+			// it turns out) -- there is no data
+			if (handleData()) {
+				reader.consumePeek();
+			}
 		} else {
 			handleToken(token);
+			reader.consumePeek();
 		}
-
-		// Consume the peeked token
-		reader.consumePeek();
+		prepareCurrentHandler(false, false);
 	}
 }
 
@@ -1100,17 +1371,24 @@ void StackImpl::popTokens()
 	tokenStack.popTokens();
 }
 
-Variant StackImpl::readData()
+bool StackImpl::readToken(Token &token)
 {
 	if (dataReader != nullptr) {
-		TokenizedDataReaderFork dataReaderFork = dataReader->fork();
-		Token token;
-		dataReaderFork.read(token, currentTokens(), currentWhitespaceMode());
-		if (token.id == Tokens::Data) {
-			Variant res = Variant::fromString(token.content);
-			res.setLocation(token.getLocation());
-			return res;
-		}
+		dataReader->resetPeek();
+		return dataReader->peek(token, currentTokens(),
+		                        currentWhitespaceMode());
+	}
+	return false;
+}
+
+Variant StackImpl::readData()
+{
+	Token token;
+	if (readToken(token) && token.id == Tokens::Data) {
+		// TODO: Introduce function for string variant with location
+		Variant res = Variant::fromString(token.content);
+		res.setLocation(token.getLocation());
+		return res;
 	}
 	return Variant{};
 }