From f713b1d393230e7083727d457623fdac878eb248 Mon Sep 17 00:00:00 2001 From: Andreas Stöckel Date: Sun, 8 Feb 2015 18:48:07 +0100 Subject: DynamicTokenizer now gets the reader as a parameter to read and peek -- the beauty of this tokenizer is that it has no internal state depending on the reader, so it doesn't need to hold a reference to it --- src/plugins/plain/DynamicTokenizer.cpp | 35 +++++++++++++++++----------------- src/plugins/plain/DynamicTokenizer.hpp | 22 ++++++++++----------- 2 files changed, 27 insertions(+), 30 deletions(-) (limited to 'src') diff --git a/src/plugins/plain/DynamicTokenizer.cpp b/src/plugins/plain/DynamicTokenizer.cpp index a8f2317..f2cfcd1 100644 --- a/src/plugins/plain/DynamicTokenizer.cpp +++ b/src/plugins/plain/DynamicTokenizer.cpp @@ -345,14 +345,13 @@ public: /* Class DynamicTokenizer */ -DynamicTokenizer::DynamicTokenizer(CharReader &reader, - WhitespaceMode whitespaceMode) - : reader(reader), whitespaceMode(whitespaceMode), nextTokenTypeId(0) +DynamicTokenizer::DynamicTokenizer(WhitespaceMode whitespaceMode) + : whitespaceMode(whitespaceMode), nextTokenTypeId(0) { } template -bool DynamicTokenizer::next(DynamicToken &token) +bool DynamicTokenizer::next(CharReader &reader, DynamicToken &token) { // If we're in the read mode, reset the char reader peek position to the // current read position @@ -437,28 +436,28 @@ bool DynamicTokenizer::next(DynamicToken &token) return match.hasMatch(); } -bool DynamicTokenizer::read(DynamicToken &token) +bool DynamicTokenizer::read(CharReader &reader,DynamicToken &token) { switch (whitespaceMode) { case WhitespaceMode::PRESERVE: - return next(token); + return next(reader, token); case WhitespaceMode::TRIM: - return next(token); + return next(reader, token); case WhitespaceMode::COLLAPSE: - return next(token); + return next(reader, token); } return false; } -bool DynamicTokenizer::peek(DynamicToken &token) +bool DynamicTokenizer::peek(CharReader &reader,DynamicToken &token) { switch (whitespaceMode) { case WhitespaceMode::PRESERVE: - return next(token); + return next(reader, token); case WhitespaceMode::TRIM: - return next(token); + return next(reader, token); case WhitespaceMode::COLLAPSE: - return next(token); + return next(reader, token); } return false; } @@ -530,16 +529,16 @@ WhitespaceMode DynamicTokenizer::getWhitespaceMode() { return whitespaceMode; } /* Explicitly instantiate all possible instantiations of the "next" member function */ template bool DynamicTokenizer::next( - DynamicToken &token); + CharReader &reader, DynamicToken &token); template bool DynamicTokenizer::next( - DynamicToken &token); + CharReader &reader, DynamicToken &token); template bool DynamicTokenizer::next( - DynamicToken &token); + CharReader &reader,DynamicToken &token); template bool DynamicTokenizer::next( - DynamicToken &token); + CharReader &reader,DynamicToken &token); template bool DynamicTokenizer::next( - DynamicToken &token); + CharReader &reader,DynamicToken &token); template bool DynamicTokenizer::next( - DynamicToken &token); + CharReader &reader,DynamicToken &token); } diff --git a/src/plugins/plain/DynamicTokenizer.hpp b/src/plugins/plain/DynamicTokenizer.hpp index 760bebf..0b4dd39 100644 --- a/src/plugins/plain/DynamicTokenizer.hpp +++ b/src/plugins/plain/DynamicTokenizer.hpp @@ -118,11 +118,6 @@ enum class WhitespaceMode { */ class DynamicTokenizer { private: - /** - * CharReader instance from which the tokens should be read. - */ - CharReader &reader; - /** * Internally used token trie. This object holds all registered tokens. */ @@ -151,23 +146,22 @@ private: * @tparam TextHandler is the type to be used for the textHandler instance. * @tparam read specifies whether the function should start from and advance * the read pointer of the char reader. + * @param reader is the CharReader instance from which the data should be + * read. * @param token is the token structure into which the token information * should be written. * @return false if the end of the stream has been reached, true otherwise. */ template - bool next(DynamicToken &token); + bool next(CharReader &reader, DynamicToken &token); public: /** * Constructor of the DynamicTokenizer class. * - * @param reader is the CharReader that should be used for reading the - * tokens. * @param whitespaceMode specifies how whitespace should be handled. */ - DynamicTokenizer(CharReader &reader, - WhitespaceMode whitespaceMode = WhitespaceMode::COLLAPSE); + DynamicTokenizer(WhitespaceMode whitespaceMode = WhitespaceMode::COLLAPSE); /** * Registers the given string as a token. Returns a const pointer at a @@ -222,23 +216,27 @@ public: * Reads a new token from the CharReader and stores it in the given * DynamicToken instance. * + * @param reader is the CharReader instance from which the data should be + * read. * @param token is a reference at the token instance into which the Token * information should be written. * @return true if a token could be read, false if the end of the stream * has been reached. */ - bool read(DynamicToken &token); + bool read(CharReader &reader, DynamicToken &token); /** * The peek method does not advance the read position of the char reader, * but reads the next token from the current char reader peek position. * + * @param reader is the CharReader instance from which the data should be + * read. * @param token is a reference at the token instance into which the Token * information should be written. * @return true if a token could be read, false if the end of the stream * has been reached. */ - bool peek(DynamicToken &token); + bool peek(CharReader &reader, DynamicToken &token); }; } -- cgit v1.2.3