diff options
Diffstat (limited to 'src/core/parser')
| -rw-r--r-- | src/core/parser/utils/Tokenizer.cpp | 17 | ||||
| -rw-r--r-- | src/core/parser/utils/Tokenizer.hpp | 6 |
2 files changed, 13 insertions, 10 deletions
diff --git a/src/core/parser/utils/Tokenizer.cpp b/src/core/parser/utils/Tokenizer.cpp index 127d1cf..5feb7a5 100644 --- a/src/core/parser/utils/Tokenizer.cpp +++ b/src/core/parser/utils/Tokenizer.cpp @@ -163,7 +163,8 @@ public: Tokenizer::Tokenizer() : nextTokenId(0) {} template <bool tRead> -bool Tokenizer::next(CharReader &reader, Token &token, TokenizedData &data) +bool Tokenizer::next(CharReader &reader, Token &token, + TokenizedData &data) const { // If we're in the read mode, reset the char reader peek position to the // current read position @@ -214,7 +215,6 @@ bool Tokenizer::next(CharReader &reader, Token &token, TokenizedData &data) } } - // If a token has been found and the token is a primary token, check // whether we have to abort, otherwise if we have a non-primary match, // reset it once it can no longer be advanced @@ -229,7 +229,6 @@ bool Tokenizer::next(CharReader &reader, Token &token, TokenizedData &data) // Record all incomming characters data.append(c, charStart, charEnd); - // Swap the lookups and the nextLookups list lookups = std::move(nextLookups); nextLookups.clear(); @@ -278,12 +277,14 @@ bool Tokenizer::next(CharReader &reader, Token &token, TokenizedData &data) return bestMatch.hasMatch(); } -bool Tokenizer::read(CharReader &reader, Token &token, TokenizedData &data) +bool Tokenizer::read(CharReader &reader, Token &token, + TokenizedData &data) const { return next<true>(reader, token, data); } -bool Tokenizer::peek(CharReader &reader, Token &token, TokenizedData &data) +bool Tokenizer::peek(CharReader &reader, Token &token, + TokenizedData &data) const { return next<false>(reader, token, data); } @@ -349,7 +350,9 @@ const Tokenizer::TokenDescriptor &Tokenizer::lookupToken(TokenId id) const /* Explicitly instantiate all possible instantiations of the "next" member function */ -template bool Tokenizer::next<false>(CharReader &, Token &, TokenizedData &); -template bool Tokenizer::next<true>(CharReader &, Token &, TokenizedData &); +template bool Tokenizer::next<false>(CharReader &, Token &, + TokenizedData &) const; +template bool Tokenizer::next<true>(CharReader &, Token &, + TokenizedData &) const; } diff --git a/src/core/parser/utils/Tokenizer.hpp b/src/core/parser/utils/Tokenizer.hpp index 74e3f0d..9137180 100644 --- a/src/core/parser/utils/Tokenizer.hpp +++ b/src/core/parser/utils/Tokenizer.hpp @@ -124,7 +124,7 @@ private: * @return false if the end of the stream has been reached, true otherwise. */ template <bool read> - bool next(CharReader &reader, Token &token, TokenizedData &data); + bool next(CharReader &reader, Token &token, TokenizedData &data) const; public: /** @@ -181,7 +181,7 @@ public: * @return true if a token could be read, false if the end of the stream * has been reached. */ - bool read(CharReader &reader, Token &token, TokenizedData &data); + bool read(CharReader &reader, Token &token, TokenizedData &data) const; /** * The peek method does not advance the read position of the char reader, @@ -196,7 +196,7 @@ public: * @return true if a token could be read, false if the end of the stream * has been reached. */ - bool peek(CharReader &reader, Token &token, TokenizedData &data); + bool peek(CharReader &reader, Token &token, TokenizedData &data) const; }; } |