From 165cf9a5c6ab03dab64d5eb5a5577f8c216bb832 Mon Sep 17 00:00:00 2001
From: Benjamin Paassen <bpaassen@techfak.uni-bielefeld.de>
Date: Fri, 14 Nov 2014 17:41:03 +0100
Subject: implemented tokenizer test and started implementing CodeTokenizer
 under supervision of Maester Stoeckel.

---
 src/core/utils/Tokenizer.cpp | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

(limited to 'src/core/utils/Tokenizer.cpp')

diff --git a/src/core/utils/Tokenizer.cpp b/src/core/utils/Tokenizer.cpp
index 2c36438..164a30f 100644
--- a/src/core/utils/Tokenizer.cpp
+++ b/src/core/utils/Tokenizer.cpp
@@ -96,6 +96,7 @@ bool Tokenizer::prepare()
 				tBuf << c;
 				n = &(n->children.at(c));
 				if (n->tokenId != TOKEN_NONE) {
+					match = n->tokenId;
 					// from here on we found a token. If we have something
 					// in our buffer already, we end the search now.
 					if (!bufEmpty) {
@@ -111,7 +112,7 @@ bool Tokenizer::prepare()
 					// if we are at the end we break off the search.
 					break;
 				}
-				if (n->children.find(c) == root.children.end()) {
+				if (n->children.find(c) == n->children.end()) {
 					// if we do not find a possible continuation anymore,
 					// break off the search.
 					break;
@@ -122,16 +123,21 @@ bool Tokenizer::prepare()
 				input.resetPeek();
 				if (bufEmpty) {
 					// if we did not have text before, construct that token.
-					peeked.push_back(Token{match, tBuf.str(), startColumn,
-					                       startLine, input.getColumn(),
-					                       input.getLine()});
-					return true;
+					if (doPrepare(
+					        Token{match, tBuf.str(), startColumn, startLine,
+					              input.getColumn(), input.getLine()},
+					        peeked)) {
+						return true;
+					}
+
 				} else {
 					// otherwise we return the text before the token.
-					peeked.push_back(Token{TOKEN_TEXT, buffer.str(),
-					                       startColumn, startLine,
-					                       input.getColumn(), input.getLine()});
-					return true;
+					if (doPrepare(Token{TOKEN_TEXT, buffer.str(), startColumn,
+					                    startLine, input.getColumn(),
+					                    input.getLine()},
+					              peeked)) {
+						return true;
+					}
 				}
 			}
 		}
@@ -140,13 +146,19 @@ bool Tokenizer::prepare()
 		input.consumePeek();
 	}
 	if (!bufEmpty) {
-		peeked.push_back(Token{TOKEN_TEXT, buffer.str(), startColumn, startLine,
-		                       input.getColumn(), input.getLine()});
-		return true;
+		return doPrepare(Token{TOKEN_TEXT, buffer.str(), startColumn, startLine,
+		                       input.getColumn(), input.getLine()},
+		                 peeked);
 	}
 	return false;
 }
 
+bool Tokenizer::doPrepare(const Token &t, std::deque<Token> &peeked)
+{
+	peeked.push_back(t);
+	return true;
+}
+
 bool Tokenizer::next(Token &t)
 {
 	if (peeked.empty()) {
-- 
cgit v1.2.3