From 3f62168ed0b088eec3cb2903f03966f7d501f564 Mon Sep 17 00:00:00 2001
From: Andreas Stöckel <astoecke@techfak.uni-bielefeld.de>
Date: Thu, 11 Dec 2014 15:26:50 +0100
Subject: moved to CharReader everywhere

---
 src/core/Tokenizer.hpp | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

(limited to 'src/core/Tokenizer.hpp')

diff --git a/src/core/Tokenizer.hpp b/src/core/Tokenizer.hpp
index 8f80150..33327cc 100644
--- a/src/core/Tokenizer.hpp
+++ b/src/core/Tokenizer.hpp
@@ -19,11 +19,12 @@
 #ifndef _OUSIA_TOKENIZER_HPP_
 #define _OUSIA_TOKENIZER_HPP_
 
+#include <cstdint>
+#include <deque>
 #include <istream>
 #include <map>
-#include <deque>
 
-#include "BufferedCharReader.hpp"
+#include <core/common/CharReader.hpp>
 
 namespace ousia {
 
@@ -120,13 +121,13 @@ static const int TOKEN_TEXT = -2;
 struct Token {
 	int tokenId;
 	std::string content;
-	int startColumn;
-	int startLine;
-	int endColumn;
-	int endLine;
+	uint32_t startColumn;
+	uint32_t startLine;
+	uint32_t endColumn;
+	uint32_t endLine;
 
-	Token(int tokenId, std::string content, int startColumn, int startLine,
-	      int endColumn, int endLine)
+	Token(int tokenId, std::string content, uint32_t startColumn, uint32_t startLine,
+	      uint32_t endColumn, uint32_t endLine)
 	    : tokenId(tokenId),
 	      content(content),
 	      startColumn(startColumn),
@@ -160,7 +161,7 @@ struct Token {
  */
 class Tokenizer {
 private:
-	BufferedCharReader &input;
+	CharReader &input;
 	const TokenTreeNode &root;
 	std::deque<Token> peeked;
 	unsigned int peekCursor = 0;
@@ -185,14 +186,14 @@ protected:
 public:
 	/**
 	 * @param input The input of a Tokenizer is given in the form of a
-	 * BufferedCharReader. Please refer to the respective documentation.
+	 * CharReader. Please refer to the respective documentation.
 	 * @param root This is meant to be the root of a TokenTree giving the
 	 * specification of user-defined tokens this Tokenizer should recognize.
 	 * The Tokenizer promises to not change the TokenTree such that you can
 	 * re-use the same specification for multiple inputs.
 	 * Please refer to the TokenTreeNode documentation for more information.
 	 */
-	Tokenizer(BufferedCharReader &input, const TokenTreeNode &root);
+	Tokenizer(CharReader &input, const TokenTreeNode &root);
 
 	/**
 	 * The next method consumes one Token from the input stream and gives
@@ -224,9 +225,9 @@ public:
 	 */
 	void consumePeek();
 
-	const BufferedCharReader &getInput() const { return input; }
+	const CharReader &getInput() const { return input; }
 	
-	BufferedCharReader &getInput() { return input; }
+	CharReader &getInput() { return input; }
 };
 }
 
-- 
cgit v1.2.3