diff options
Diffstat (limited to 'src/core/parser/utils/Tokenizer.cpp')
| -rw-r--r-- | src/core/parser/utils/Tokenizer.cpp | 7 |
1 files changed, 1 insertions, 6 deletions
diff --git a/src/core/parser/utils/Tokenizer.cpp b/src/core/parser/utils/Tokenizer.cpp index 51787cd..e78b0f4 100644 --- a/src/core/parser/utils/Tokenizer.cpp +++ b/src/core/parser/utils/Tokenizer.cpp @@ -156,6 +156,7 @@ public: return res; } }; + } /* Class Tokenizer */ @@ -229,12 +230,6 @@ bool Tokenizer::next(CharReader &reader, Token &token, TokenizedData &data) } else { // Record all incomming characters data.append(c, charStart, charEnd); - - // Special token processing - // TODO: Build a special state machine for this in another class - if (c == '\n') { - data.mark(Tokens::Newline, 1); - } } // Swap the lookups and the nextLookups list |