Addapted Tokenizer to new SourceLocation

3 files changed, 24 insertions, 40 deletions

diff --git a/src/core/CodeTokenizer.cpp b/src/core/CodeTokenizer.cpp
index e9c1bbf..fbc1150 100644
--- a/src/core/CodeTokenizer.cpp
+++ b/src/core/CodeTokenizer.cpp
@@ -26,8 +26,10 @@ Token CodeTokenizer::constructToken(const Token &t)
 {
 	std::string content = buf.str();
 	buf.str(std::string());
-	return Token{returnTokenId,        content,     startToken.startColumn,
-	             startToken.startLine, t.endColumn, t.endLine};
+	return Token{
+	    returnTokenId, content,
+	    SourceLocation{t.location.getSourceId(), startToken.location.getStart(),
+	                   t.location.getEnd()}};
 }
 
 void CodeTokenizer::buffer(const Token &t) { buf << t.content; }
@@ -40,12 +42,6 @@ bool CodeTokenizer::doPrepare(const Token &t, std::deque<Token> &peeked)
 		mode = it->second.mode;
 	}
 
-	if (t.startLine != t.endLine && mode != CodeTokenMode::LINEBREAK) {
-		throw TokenizerException(
-		    "We did not expect a multiline token (except linebreaks). Most "
-		    "likely you did not add a linebreak token to your tokenizer!");
-	}
-
 	switch (state) {
 		case CodeTokenizerState::NORMAL:
 			switch (mode) {
@@ -60,9 +56,8 @@ bool CodeTokenizer::doPrepare(const Token &t, std::deque<Token> &peeked)
 					break;
 				case CodeTokenMode::LINEBREAK:
 					if (!ignoreLinebreaks) {
-						peeked.push_back({it->second.id, t.content,
-						                  t.startColumn, t.startLine,
-						                  t.endColumn, t.endLine});
+						peeked.push_back(
+						    {it->second.id, t.content, t.location});
 					}
 					return !ignoreLinebreaks;
 				default:
@@ -87,18 +82,21 @@ bool CodeTokenizer::doPrepare(const Token &t, std::deque<Token> &peeked)
 									peeked.push_back(Token{
 									    TOKEN_TEXT,
 									    t.content.substr(begin, (int)c - begin),
-									    t.startColumn + begin, t.startLine,
-									    t.startColumn + (int)c, t.endLine});
+									    SourceLocation{
+									        t.location.getSourceId(),
+									        t.location.getStart() + begin,
+									        t.location.getStart() + c}});
 									begin = -1;
 									empty = false;
 								}
 							}
 						}
 						if (begin >= 0) {
-							peeked.push_back(
-							    Token{TOKEN_TEXT, t.content.substr(begin),
-							          t.startColumn + begin, t.startLine,
-							          t.endColumn, t.endLine});
+							peeked.push_back(Token{
+							    TOKEN_TEXT, t.content.substr(begin),
+							    SourceLocation{t.location.getSourceId(),
+							                   t.location.getStart() + begin,
+							                   t.location.getEnd()}});
 							empty = false;
 						}
 					} else {
diff --git a/src/core/Tokenizer.cpp b/src/core/Tokenizer.cpp
index 9d25608..ab4735a 100644
--- a/src/core/Tokenizer.cpp
+++ b/src/core/Tokenizer.cpp
@@ -81,8 +81,7 @@ bool Tokenizer::prepare()
 {
 	std::stringstream buffer;
 	char c;
-	int startColumn = input.getColumn();
-	int startLine = input.getLine();
+	SourcePosition start = input.getOffset();
 	bool bufEmpty = true;
 	while (input.peek(c)) {
 		if (root.children.find(c) != root.children.end()) {
@@ -124,20 +123,16 @@ bool Tokenizer::prepare()
 				if (bufEmpty) {
 					// if we did not have text before, construct that token.
 					if (doPrepare(
-					        Token{match, tBuf.str(), startColumn, startLine,
-					              input.getColumn(), input.getLine()},
+					        Token{match, tBuf.str(), input.getLocation(start)},
 					        peeked)) {
 						return true;
 					} else {
-						startColumn = input.getColumn();
-						startLine = input.getLine();
+						start = input.getOffset();
 						continue;
 					}
 				} else {
 					// otherwise we return the text before the token.
-					if (doPrepare(Token{TOKEN_TEXT, buffer.str(), startColumn,
-					                    startLine, input.getColumn(),
-					                    input.getLine()},
+					if (doPrepare(Token{TOKEN_TEXT, buffer.str(), input.getLocation(start)},
 					              peeked)) {
 						return true;
 					} else{
@@ -146,8 +141,7 @@ bool Tokenizer::prepare()
 						//constructed.
 						buffer.str(std::string());
 						bufEmpty = true;
-						startColumn = input.getColumn();
-						startLine = input.getLine();
+						start = input.getOffset();
 						continue;
 					} 
 				}
@@ -161,8 +155,7 @@ bool Tokenizer::prepare()
 		input.consumePeek();
 	}
 	if (!bufEmpty) {
-		return doPrepare(Token{TOKEN_TEXT, buffer.str(), startColumn, startLine,
-		                       input.getColumn(), input.getLine()},
+		return doPrepare(Token{TOKEN_TEXT, buffer.str(), input.getLocation(start)},
 		                 peeked);
 	}
 	return false;
diff --git a/src/core/Tokenizer.hpp b/src/core/Tokenizer.hpp
index 2b03e17..50e458c 100644
--- a/src/core/Tokenizer.hpp
+++ b/src/core/Tokenizer.hpp
@@ -121,19 +121,12 @@ static const int TOKEN_TEXT = -2;
 struct Token {
 	int tokenId;
 	std::string content;
-	int startColumn;
-	int startLine;
-	int endColumn;
-	int endLine;
+	SourceLocation location;
 
-	Token(int tokenId, std::string content, int startColumn, int startLine,
-	      int endColumn, int endLine)
+	Token(int tokenId, std::string content, SourceLocation location)
 	    : tokenId(tokenId),
 	      content(content),
-	      startColumn(startColumn),
-	      startLine(startLine),
-	      endColumn(endColumn),
-	      endLine(endLine)
+	      location(location)
 	{
 	}