implemented tokenizer test and started implementing CodeTokenizer under supervision of Maester Stoeckel.

1 files changed, 20 insertions, 3 deletions

diff --git a/src/core/utils/Tokenizer.hpp b/src/core/utils/Tokenizer.hpp
index 924b670..eb8eed4 100644
--- a/src/core/utils/Tokenizer.hpp
+++ b/src/core/utils/Tokenizer.hpp
@@ -45,6 +45,9 @@ public:
 	TokenTreeNode(const std::map<std::string, int> &inputs);
 };
 
+static const int TOKEN_NONE = -1;
+static const int TOKEN_TEXT = -2;
+
 struct Token {
 	int tokenId;
 	std::string content;
@@ -63,10 +66,9 @@ struct Token {
 	      endLine(endLine)
 	{
 	}
-};
 
-static const int TOKEN_NONE = -1;
-static const int TOKEN_TEXT = -2;
+	Token() : tokenId(TOKEN_NONE) {}
+};
 
 class Tokenizer {
 private:
@@ -77,6 +79,21 @@ private:
 
 	bool prepare();
 
+protected:
+	/**
+	* This method is an interface to build multiple tokens from a single one in
+	* derived classes. This might be interesting if you want to implement
+	* further logic on text tokens or similar applications.
+	*
+	* @param t a Token the "basic" tokenizer found.
+	* @param peeked a reference to the deque containing all temporary Tokens.
+	* You are supposed to append your tokens there. In the trivial case you just
+	* put the given Token on top of the deque.
+	* @return false if no token was appended to the deque (meaning that you want
+	* to ignore the given token explicitly) and true in all other cases.
+	*/
+	virtual bool doPrepare(const Token &t, std::deque<Token> &peeked);
+
 public:
 	Tokenizer(BufferedCharReader &input, const TokenTreeNode &root);