Remove "inherit" flag in TokenStack, add tiny unit test for TokenStack, temporarily remove pushTokens and popTokens calls from DocumentChildHandler

7 files changed, 21 insertions, 35 deletions

diff --git a/src/core/parser/stack/Callbacks.hpp b/src/core/parser/stack/Callbacks.hpp
index 12796ed..dfe41fc 100644
--- a/src/core/parser/stack/Callbacks.hpp
+++ b/src/core/parser/stack/Callbacks.hpp
@@ -86,10 +86,8 @@ public:
 	 *
 	 * @param tokens is a list of TokenSyntaxDescriptor instances that should be
 	 * stored on the stack.
-	 * @param inherit if true, the previous tokens descriptors are extended.
 	 */
-	virtual void pushTokens(const std::vector<SyntaxDescriptor> &tokens,
-	                        bool inherit) = 0;
+	virtual void pushTokens(const std::vector<SyntaxDescriptor> &tokens) = 0;
 
 	/**
 	 * Removes the previously pushed list of tokens from the stack.
diff --git a/src/core/parser/stack/DocumentHandler.cpp b/src/core/parser/stack/DocumentHandler.cpp
index 7a81186..a7fbaa8 100644
--- a/src/core/parser/stack/DocumentHandler.cpp
+++ b/src/core/parser/stack/DocumentHandler.cpp
@@ -137,9 +137,6 @@ void DocumentChildHandler::pushDocumentField(Handle<Node> parent,
 	    new DocumentField(manager(), parent, fieldIdx, transparent);
 	field->setLocation(location());
 	scope().push(field);
-
-	// Push all possible tokens onto the stack
-	pushTokens(fieldDescr->getPermittedTokens(), true);
 }
 
 void DocumentChildHandler::popDocumentField()
@@ -147,9 +144,6 @@ void DocumentChildHandler::popDocumentField()
 	// Pop the field from the scope, make sure it actually is a DocumentField
 	assert(scope().getLeaf()->isa(&RttiTypes::DocumentField));
 	scope().pop(logger());
-
-	// Pop the registered tokens from the stack
-	popTokens();
 }
 
 void DocumentChildHandler::createPath(const NodeVector<Node> &path,
diff --git a/src/core/parser/stack/Handler.cpp b/src/core/parser/stack/Handler.cpp
index 019ae5a..0a6e67c 100644
--- a/src/core/parser/stack/Handler.cpp
+++ b/src/core/parser/stack/Handler.cpp
@@ -78,10 +78,9 @@ const State &Handler::state() const { return handlerData.state; }
 
 Variant Handler::readData() { return handlerData.callbacks.readData(); }
 
-void Handler::pushTokens(const std::vector<SyntaxDescriptor> &tokens,
-                         bool inherit)
+void Handler::pushTokens(const std::vector<SyntaxDescriptor> &tokens)
 {
-	handlerData.callbacks.pushTokens(tokens, inherit);
+	handlerData.callbacks.pushTokens(tokens);
 }
 
 void Handler::popTokens() { handlerData.callbacks.popTokens(); }
diff --git a/src/core/parser/stack/Handler.hpp b/src/core/parser/stack/Handler.hpp
index 1149ed2..d85848a 100644
--- a/src/core/parser/stack/Handler.hpp
+++ b/src/core/parser/stack/Handler.hpp
@@ -166,10 +166,8 @@ protected:
 	 *
 	 * @param tokens is a list of TokenSyntaxDescriptor instances that should be
 	 * stored on the stack.
-	 * @param inherit if true, the last tokens on the stack are extended and not
-	 * overriden.
 	 */
-	void pushTokens(const std::vector<SyntaxDescriptor> &tokens, bool inherit);
+	void pushTokens(const std::vector<SyntaxDescriptor> &tokens);
 
 	/**
 	 * Calls the corresponding function in the HandlerCallbacks instance.
diff --git a/src/core/parser/stack/Stack.cpp b/src/core/parser/stack/Stack.cpp
index 6fa8659..e027271 100644
--- a/src/core/parser/stack/Stack.cpp
+++ b/src/core/parser/stack/Stack.cpp
@@ -480,7 +480,7 @@ public:
 	TokenId registerToken(const std::string &token) override;
 	void unregisterToken(TokenId id) override;
 	Variant readData() override;
-	void pushTokens(const std::vector<SyntaxDescriptor> &tokens, bool inherit) override;
+	void pushTokens(const std::vector<SyntaxDescriptor> &tokens) override;
 	void popTokens() override;
 };
 
@@ -1089,15 +1089,9 @@ void StackImpl::unregisterToken(TokenId id)
 	tokenRegistry.unregisterToken(id);
 }
 
-void StackImpl::pushTokens(const std::vector<SyntaxDescriptor> &tokens, bool inherit)
+void StackImpl::pushTokens(const std::vector<SyntaxDescriptor> &tokens)
 {
-	// Push the tokens onto the token stack
-	for (const SyntaxDescriptor &token: tokens) {
-		std::cout << token.open << std::endl;
-		std::cout << token.close << std::endl;
-		std::cout << token.shortForm << std::endl;
-	}
-	tokenStack.pushTokens(tokens, inherit);
+	tokenStack.pushTokens(tokens);
 }
 
 void StackImpl::popTokens()
diff --git a/src/core/parser/stack/TokenStack.cpp b/src/core/parser/stack/TokenStack.cpp
index 1c26a7c..95424c7 100644
--- a/src/core/parser/stack/TokenStack.cpp
+++ b/src/core/parser/stack/TokenStack.cpp
@@ -16,18 +16,23 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "TokenStack.hpp"
+#include <cassert>
 
-#include <iostream>
+#include "TokenStack.hpp"
 
 namespace ousia {
 namespace parser_stack {
 
-void TokenStack::pushTokens(const std::vector<SyntaxDescriptor> &tokens,
-                            bool inherit)
+void TokenStack::pushTokens(const std::vector<SyntaxDescriptor> &tokens)
+{
+	stack.emplace_back(tokens);
+	std::sort(stack.back().begin(), stack.back().end());
+}
+
+void TokenStack::popTokens()
 {
-	// TODO: Implement inheritance
-	stack.push_back(tokens);
+	assert(!stack.empty() && "too many calls to popTokens");
+	stack.pop_back();
 }
 
 TokenDescriptor TokenStack::lookup(TokenId token) const
@@ -48,7 +53,6 @@ TokenDescriptor TokenStack::lookup(TokenId token) const
 	}
 	return res;
 }
-void TokenStack::popTokens() { stack.pop_back(); }
 
 TokenSet TokenStack::tokens() const
 {
diff --git a/src/core/parser/stack/TokenStack.hpp b/src/core/parser/stack/TokenStack.hpp
index d5deafe..a0c8cff 100644
--- a/src/core/parser/stack/TokenStack.hpp
+++ b/src/core/parser/stack/TokenStack.hpp
@@ -67,7 +67,8 @@ struct TokenDescriptor {
 
 /**
  * The TokenStack class is used by the Stack class to collect all currently
- * enabled user defined tokens.
+ * enabled user defined tokens. Additionally it allows the Stack class to
+ * remember the end tokens for pending open tokens.
  */
 class TokenStack {
 private:
@@ -83,10 +84,8 @@ public:
 	 *
 	 * @param tokens is a list of SyntaxDescriptor instances that should be
 	 * stored on the stack.
-	 * @param inherit if set to true, combines the given tokens with the tokens
-	 * from the last stack level.
 	 */
-	void pushTokens(const std::vector<SyntaxDescriptor> &tokens, bool inherit);
+	void pushTokens(const std::vector<SyntaxDescriptor> &tokens);
 
 	/**
 	 * Removes the previously pushed list of tokens from the stack.