Addapted Tokenizer to new SourceLocation

1 files changed, 6 insertions, 13 deletions

diff --git a/src/core/Tokenizer.cpp b/src/core/Tokenizer.cpp
index 9d25608..ab4735a 100644
--- a/src/core/Tokenizer.cpp
+++ b/src/core/Tokenizer.cpp
@@ -81,8 +81,7 @@ bool Tokenizer::prepare()
 {
 	std::stringstream buffer;
 	char c;
-	int startColumn = input.getColumn();
-	int startLine = input.getLine();
+	SourcePosition start = input.getOffset();
 	bool bufEmpty = true;
 	while (input.peek(c)) {
 		if (root.children.find(c) != root.children.end()) {
@@ -124,20 +123,16 @@ bool Tokenizer::prepare()
 				if (bufEmpty) {
 					// if we did not have text before, construct that token.
 					if (doPrepare(
-					        Token{match, tBuf.str(), startColumn, startLine,
-					              input.getColumn(), input.getLine()},
+					        Token{match, tBuf.str(), input.getLocation(start)},
 					        peeked)) {
 						return true;
 					} else {
-						startColumn = input.getColumn();
-						startLine = input.getLine();
+						start = input.getOffset();
 						continue;
 					}
 				} else {
 					// otherwise we return the text before the token.
-					if (doPrepare(Token{TOKEN_TEXT, buffer.str(), startColumn,
-					                    startLine, input.getColumn(),
-					                    input.getLine()},
+					if (doPrepare(Token{TOKEN_TEXT, buffer.str(), input.getLocation(start)},
 					              peeked)) {
 						return true;
 					} else{
@@ -146,8 +141,7 @@ bool Tokenizer::prepare()
 						//constructed.
 						buffer.str(std::string());
 						bufEmpty = true;
-						startColumn = input.getColumn();
-						startLine = input.getLine();
+						start = input.getOffset();
 						continue;
 					} 
 				}
@@ -161,8 +155,7 @@ bool Tokenizer::prepare()
 		input.consumePeek();
 	}
 	if (!bufEmpty) {
-		return doPrepare(Token{TOKEN_TEXT, buffer.str(), startColumn, startLine,
-		                       input.getColumn(), input.getLine()},
+		return doPrepare(Token{TOKEN_TEXT, buffer.str(), input.getLocation(start)},
 		                 peeked);
 	}
 	return false;