Prepared implementation of user defined tokens

6 files changed, 199 insertions, 70 deletions

diff --git a/src/core/parser/stack/DocumentHandler.cpp b/src/core/parser/stack/DocumentHandler.cpp
index c239583..78629ac 100644
--- a/src/core/parser/stack/DocumentHandler.cpp
+++ b/src/core/parser/stack/DocumentHandler.cpp
@@ -42,8 +42,10 @@ bool DocumentHandler::startCommand(Variant::mapType &args)
 	Rooted<Document> document =
 	    context().getProject()->createDocument(args["name"].asString());
 	document->setLocation(location());
+
 	scope().push(document);
 	scope().setFlag(ParserFlag::POST_HEAD, false);
+	scope().setFlag(ParserFlag::POST_USER_DEFINED_TOKEN_REGISTRATION, false);
 
 	return true;
 }
@@ -55,6 +57,30 @@ void DocumentHandler::end() { scope().pop(logger()); }
 DocumentChildHandler::DocumentChildHandler(const HandlerData &handlerData)
     : Handler(handlerData), isExplicitField(false)
 {
+	// Register all user defined tokens if this has not yet been done
+	if (!scope().getFlag(ParserFlag::POST_USER_DEFINED_TOKEN_REGISTRATION)) {
+		registerUserDefinedTokens();
+	}
+}
+
+void DocumentChildHandler::registerUserDefinedTokens()
+{
+	// Set the POST_USER_DEFINED_TOKEN_REGISTRATION flag, to prevent this method
+	// from being called again
+	scope().setFlag(ParserFlag::POST_USER_DEFINED_TOKEN_REGISTRATION, true);
+
+	// Fetch the underlying document and all ontologies registered in the
+	// document and register all user defined tokens in the parser
+	Rooted<Document> doc = scope().selectOrThrow<Document>();
+	for (Rooted<Ontology> ontology : doc->getOntologies()) {
+		std::vector<TokenDescriptor *> tokens =
+		    ontology->getAllTokenDescriptors();
+		for (TokenDescriptor *token : tokens) {
+			if (!token->special) {
+				token->id = registerToken(token->token);
+			}
+		}
+	}
 }
 
 void DocumentChildHandler::preamble(Rooted<Node> &parentNode, size_t &fieldIdx,
@@ -79,16 +105,42 @@ void DocumentChildHandler::preamble(Rooted<Node> &parentNode, size_t &fieldIdx,
 	}
 }
 
+void DocumentChildHandler::pushDocumentField(Handle<Node> parent,
+                                             Handle<FieldDescriptor> fieldDescr,
+                                             size_t fieldIdx, bool transparent)
+{
+	// Push the field onto the scope
+	Rooted<DocumentField> field =
+	    new DocumentField(manager(), parent, fieldIdx, transparent);
+	field->setLocation(location());
+	scope().push(field);
+
+	// Push all possible tokens onto the stack
+	//pushTokens(fieldDescr->getPermittedTokens());
+}
+
+void DocumentChildHandler::popDocumentField()
+{
+	// Pop the field from the scope, make sure it actually is a DocumentField
+	assert(scope().getLeaf()->isa(&RttiTypes::DocumentField));
+	scope().pop(logger());
+
+	// Pop the registered tokens from the stack
+	//popTokens();
+}
+
 void DocumentChildHandler::createPath(const NodeVector<Node> &path,
                                       DocumentEntity *&parent, size_t p0)
 {
 	size_t S = path.size();
 	for (size_t p = p0; p < S; p = p + 2) {
 		// add the field.
-		Rooted<DocumentField> field{new DocumentField(
-		    manager(), scope().getLeaf(),
-		    parent->getDescriptor()->getFieldDescriptorIndex(), true)};
-		scope().push(field);
+		const ssize_t fieldIdx =
+		    parent->getDescriptor()->getFieldDescriptorIndex();
+		const Rooted<FieldDescriptor> fieldDescr =
+		    parent->getDescriptor()->getFieldDescriptor(fieldIdx);
+		pushDocumentField(scope().getLeaf(), fieldDescr, fieldIdx, true);
+
 		// add the transparent/implicit structure element.
 		Rooted<StructuredEntity> transparent =
 		    parent->createChildStructuredEntity(path[p].cast<StructuredClass>(),
@@ -99,11 +151,11 @@ void DocumentChildHandler::createPath(const NodeVector<Node> &path,
 		scope().push(transparent);
 		parent = static_cast<DocumentEntity *>(transparent.get());
 	}
-	// add the last field.
-	Rooted<DocumentField> field{new DocumentField(
-	    manager(), scope().getLeaf(),
-	    parent->getDescriptor()->getFieldDescriptorIndex(), true)};
-	scope().push(field);
+	// add the field.
+	const ssize_t fieldIdx = parent->getDescriptor()->getFieldDescriptorIndex();
+	const Rooted<FieldDescriptor> fieldDescr =
+	    parent->getDescriptor()->getFieldDescriptor(fieldIdx);
+	pushDocumentField(scope().getLeaf(), fieldDescr, fieldIdx, true);
 
 	// Generally allow explicit fields in the new field
 	scope().setFlag(ParserFlag::POST_EXPLICIT_FIELDS, false);
@@ -199,10 +251,11 @@ bool DocumentChildHandler::startCommand(Variant::mapType &args)
 						                     "references to the beginning."),
 						    location());
 					} else {
-						Rooted<DocumentField> field{new DocumentField(
-						    manager(), parentNode, newFieldIdx, false)};
-						field->setLocation(location());
-						scope().push(field);
+						pushDocumentField(
+						    parentNode,
+						    parent->getDescriptor()->getFieldDescriptor(
+						        newFieldIdx),
+						    newFieldIdx, false);
 						isExplicitField = true;
 						return true;
 					}
@@ -230,7 +283,7 @@ bool DocumentChildHandler::startCommand(Variant::mapType &args)
 					// if we have transparent elements above us in the structure
 					// tree we try to unwind them before we give up.
 					// pop the implicit field.
-					scope().pop(logger());
+					popDocumentField();
 					// pop the implicit element.
 					scope().pop(logger());
 					continue;
@@ -298,7 +351,7 @@ bool DocumentChildHandler::startAnnotation(Variant::mapType &args)
 			// if we have transparent elements above us in the structure
 			// tree we try to unwind them before we give up.
 			// pop the implicit field.
-			scope().pop(logger());
+			popDocumentField();
 			// pop the implicit element.
 			scope().pop(logger());
 			continue;
@@ -390,8 +443,7 @@ void DocumentChildHandler::end()
 		case HandlerType::COMMAND:
 		case HandlerType::ANNOTATION_START:
 			// In case of explicit fields we do not want to pop something from
-			// the
-			// stack.
+			// the stack.
 			if (!isExplicitField) {
 				// pop the "main" element.
 				scope().pop(logger());
@@ -437,10 +489,7 @@ bool DocumentChildHandler::fieldStart(bool &isDefault, size_t fieldIdx)
 		isDefault = fieldIdx == fields.size() - 1;
 	}
 	// push the field on the stack.
-	Rooted<DocumentField> field{
-	    new DocumentField(manager(), parentNode, fieldIdx, false)};
-	field->setLocation(location());
-	scope().push(field);
+	pushDocumentField(parentNode, fields[fieldIdx], fieldIdx, false);
 
 	// Generally allow explicit fields in the new field
 	scope().setFlag(ParserFlag::POST_EXPLICIT_FIELDS, false);
@@ -453,7 +502,7 @@ void DocumentChildHandler::fieldEnd()
 	assert(scope().getLeaf()->isa(&RttiTypes::DocumentField));
 
 	// Pop the field from the stack.
-	scope().pop(logger());
+	popDocumentField();
 
 	// Pop all remaining transparent elements.
 	while (scope().getLeaf()->isa(&RttiTypes::StructuredEntity) &&
@@ -461,7 +510,7 @@ void DocumentChildHandler::fieldEnd()
 		// Pop the transparent element.
 		scope().pop(logger());
 		// Pop the transparent field.
-		scope().pop(logger());
+		popDocumentField();
 	}
 }
 
@@ -598,4 +647,4 @@ namespace RttiTypes {
 const Rtti DocumentField = RttiBuilder<ousia::parser_stack::DocumentField>(
                                "DocumentField").parent(&Node);
 }
-}
\ No newline at end of file
+}
diff --git a/src/core/parser/stack/DocumentHandler.hpp b/src/core/parser/stack/DocumentHandler.hpp
index 6f9f4a2..47534fb 100644
--- a/src/core/parser/stack/DocumentHandler.hpp
+++ b/src/core/parser/stack/DocumentHandler.hpp
@@ -98,9 +98,14 @@ private:
 	bool isExplicitField;
 
 	/**
-	 * Code shared by both the start(), fieldStart() and the data() method.
-	 * Checks whether the parser currently is in a field and returns the name
-	 * of this field.
+	 * Registers all user defined tokens in the parser.
+	 */
+	void registerUserDefinedTokens();
+
+	/**
+	 * Code shared by the startCommand(), startAnnotation(), fieldStart() and
+	 * the data() method. Checks whether the parser currently is in a field and
+	 * returns the name of this field.
 	 *
 	 * @param parentNode is the next possible parent node (a document,
 	 * a structured entity, an annotation entity or a field).
@@ -163,6 +168,20 @@ private:
 	bool convertData(Handle<FieldDescriptor> field, Variant &data,
 	                 Logger &logger);
 
+	/**
+	 * Pushes a new DocumentField onto the scope stack and registers all
+	 * premitted tokens in the parser.
+	 */
+	void pushDocumentField(Handle<Node> parent,
+	                       Handle<FieldDescriptor> fieldDescr, size_t fieldIdx,
+	                       bool transparent);
+
+	/**
+	 * Pops a DocumentField from the scope stack and retracts the permitted
+	 * tokens from the parser.
+	 */
+	void popDocumentField();
+
 public:
 	DocumentChildHandler(const HandlerData &handlerData);
 
diff --git a/src/core/parser/stack/Stack.cpp b/src/core/parser/stack/Stack.cpp
index bd16b43..436d3a5 100644
--- a/src/core/parser/stack/Stack.cpp
+++ b/src/core/parser/stack/Stack.cpp
@@ -35,6 +35,9 @@
 #include <iostream>
 #endif
 
+// TODO: Remove
+#include <iostream>
+
 namespace ousia {
 namespace parser_stack {
 namespace {
@@ -303,6 +306,11 @@ private:
 	TokenRegistry tokenRegistry;
 
 	/**
+	 * Collection of all currently enabled tokens.
+	 */
+	TokenStack tokenStack;
+
+	/**
 	 * Pointer at a TokenizedDataReader instance from which the data should
 	 * currently be read.
 	 */
@@ -362,6 +370,12 @@ private:
 	HandlerInfo &currentInfo();
 
 	/**
+	 * Returns a reference at the current HandlerInfo instance (or a stub
+	 * HandlerInfo instance if the stack is empty).
+	 */
+	const HandlerInfo &currentInfo() const;
+
+	/**
 	 * Returns a reference at the last HandlerInfo instance (or a stub
 	 * HandlerInfo instance if the stack has only one element).
 	 */
@@ -591,7 +605,9 @@ std::string StackImpl::currentCommandName() const
 
 TokenSet StackImpl::currentTokens() const
 {
-	// TODO: Implement
+	if (currentInfo().state().supportsTokens) {
+		return tokenStack.tokens();
+	}
 	return TokenSet{};
 }
 
@@ -605,6 +621,12 @@ HandlerInfo &StackImpl::currentInfo()
 {
 	return stack.empty() ? EmptyHandlerInfo : stack.back();
 }
+
+const HandlerInfo &StackImpl::currentInfo() const
+{
+	return stack.empty() ? EmptyHandlerInfo : stack.back();
+}
+
 HandlerInfo &StackImpl::lastInfo()
 {
 	return stack.size() < 2U ? EmptyHandlerInfo : stack[stack.size() - 2];
@@ -658,7 +680,8 @@ bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField)
 		// end it and repeat
 		bool canHaveImplicitDefaultField =
 		    info.type() == HandlerType::COMMAND ||
-		    info.type() == HandlerType::TOKEN || (info.type() == HandlerType::ANNOTATION_START && info.range);
+		    info.type() == HandlerType::TOKEN ||
+		    (info.type() == HandlerType::ANNOTATION_START && info.range);
 		if (info.hadDefaultField || !startImplicitDefaultField || !info.valid ||
 		    !canHaveImplicitDefaultField) {
 			// We cannot end the command if it is marked as "range" command
@@ -757,6 +780,7 @@ void StackImpl::handleData()
 
 void StackImpl::handleToken(const Token &token)
 {
+	std::cout << "Got token " << token.id << std::endl;
 	// TODO: Implement
 	// Just eat them for now
 }
@@ -1060,12 +1084,19 @@ void StackImpl::unregisterToken(TokenId id)
 
 void StackImpl::pushTokens(const std::vector<SyntaxDescriptor> &tokens)
 {
-	// TODO
+	// Push the tokens onto the token stack
+	for (const SyntaxDescriptor &token: tokens) {
+		std::cout << token.open << std::endl;
+		std::cout << token.close << std::endl;
+		std::cout << token.shortForm << std::endl;
+	}
+	tokenStack.pushTokens(tokens);
 }
 
 void StackImpl::popTokens()
 {
-	// TODO
+	// Pop the last set of tokens from the token stack.
+	tokenStack.popTokens();
 }
 
 Variant StackImpl::readData()
@@ -1123,7 +1154,8 @@ void Stack::annotationStart(const Variant &className,
 	impl->annotationStart(className, args, range);
 }
 
-void Stack::annotationEnd(const Variant &className, const Variant::mapType &args)
+void Stack::annotationEnd(const Variant &className,
+                          const Variant::mapType &args)
 {
 #if STACK_DEBUG_OUTPUT
 	std::cout << "STACK: annotationEnd " << className << " " << args
diff --git a/src/core/parser/stack/TokenStack.cpp b/src/core/parser/stack/TokenStack.cpp
index ac1d94e..b6fc6e1 100644
--- a/src/core/parser/stack/TokenStack.cpp
+++ b/src/core/parser/stack/TokenStack.cpp
@@ -18,6 +18,8 @@
 
 #include "TokenStack.hpp"
 
+#include <iostream>
+
 namespace ousia {
 namespace parser_stack {
 
@@ -26,17 +28,17 @@ void TokenStack::pushTokens(const std::vector<SyntaxDescriptor> &tokens)
 	stack.push_back(tokens);
 }
 
-void TokenStack::popTokens() { stack.pop_back(); }
+void TokenStack::popTokens() {
+	stack.pop_back();
+}
 
 TokenSet TokenStack::tokens() const
 {
-	if (stack.empty() && parentStack != nullptr) {
-		return parentStack->tokens();
-	}
-
 	TokenSet res;
-	for (const SyntaxDescriptor &descr : stack.back()) {
-		descr.insertIntoTokenSet(res);
+	if (!stack.empty()) {
+		for (const SyntaxDescriptor &descr : stack.back()) {
+			descr.insertIntoTokenSet(res);
+		}
 	}
 	return res;
 }
diff --git a/src/core/parser/stack/TokenStack.hpp b/src/core/parser/stack/TokenStack.hpp
index f2e7edc..667b976 100644
--- a/src/core/parser/stack/TokenStack.hpp
+++ b/src/core/parser/stack/TokenStack.hpp
@@ -44,44 +44,13 @@ namespace parser_stack {
 class TokenStack {
 private:
 	/**
-	 * Shared pointer at the parent TokenStack instance. May be nullptr, in
-	 * which case no parent TokenStack instance exists.
-	 */
-	const TokenStack *parentStack;
-
-	/**
 	 * Stack containing vectors of TokenSyntaxDescriptor instances as given by
 	 * the user.
 	 */
 	std::vector<std::vector<SyntaxDescriptor>> stack;
 
-	/**
-	 * Constructor of the TokenStack class.
-	 *
-	 * @param parentStack is a pointer at the underlying parentStack instance
-	 * to which calls should be forwarded if no data has been pushed onto this
-	 * stack instance.
-	 */
-	TokenStack(const TokenStack *parentStack) : parentStack(parentStack) {}
-
 public:
 	/**
-	 * Default constructor of the TokenStack class with no reference at a parent
-	 * stack.
-	 */
-	TokenStack() : TokenStack(nullptr) {}
-
-	/**
-	 * Constructor of the TokenStack class with a reference at a parent
-	 * TokenStack instance.
-	 *
-	 * @param parentStack is a reference at a parent TokenStack instance. If no
-	 * data has yet been pushed onto this instance, calls will be forwarded to
-	 * the parent stack.
-	 */
-	TokenStack(const TokenStack &parentStack) : TokenStack(&parentStack) {}
-
-	/**
 	 * Pushes a list of SyntaxDescriptor instances onto the internal stack.
 	 *
 	 * @param tokens is a list of SyntaxDescriptor instances that should be
diff --git a/testdata/osmlparser/user_defined_syntax.osml b/testdata/osmlparser/user_defined_syntax.osml
new file mode 100644
index 0000000..3d44c29
--- /dev/null
+++ b/testdata/osmlparser/user_defined_syntax.osml
@@ -0,0 +1,58 @@
+\begin{document}
+
+\ontology#article{
+	\struct#article[root=true]
+		\field
+			\childRef[ref=section]
+			\childRef[ref=paragraph]
+	\struct#section
+		\primitive#title[type=string,subtree=true]
+			\syntax
+				\open{==}
+				\close{==}
+		\field
+			\childRef[ref=paragraph]
+	\struct#paragraph[transparent=true]
+		\syntax
+			\close{\paragraph}
+		\field
+			\childRef[ref=text]
+			\childRef[ref=code]
+	\struct#code
+		\syntax
+			\open{`}
+			\close{`}
+		\field
+			\childRef[ref=text]
+	\struct#speech
+		\syntax
+			\open{<<}
+			\close{>>}
+		\fieldRef[ref=paragraph.$default]
+	\struct#text[transparent=true]
+		\primitive[type=string]
+}
+
+\begin{article}
+== This tests somewhat plays with user defined syntax ==
+
+As you can see, just the tiny `\\syntax` definitions above make writing documents
+much easier.
+
+For instance, this is a new paragraph.
+
+And here goes another one.
+
+All with the power of just adding `\\close\{\\paragraph\}` to the `\\syntax`
+definition of the paragraph structure.
+
+<<Well, that's insane!>>, I hear you say. Right so! This indeed is insane.
+
+== One last thing ==
+
+Did you notice how we started sections here?
+
+Sections can be allowed to use Wiki-like syntax with only three lines of code.
+\end{article}
+\end{document}
+