diff options
Diffstat (limited to 'src/core/parser/utils/TokenizedData.cpp')
| -rw-r--r-- | src/core/parser/utils/TokenizedData.cpp | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/src/core/parser/utils/TokenizedData.cpp b/src/core/parser/utils/TokenizedData.cpp index aeefa26..bcbbe43 100644 --- a/src/core/parser/utils/TokenizedData.cpp +++ b/src/core/parser/utils/TokenizedData.cpp @@ -26,6 +26,12 @@ #include "TokenizedData.hpp" namespace ousia { +/** + * Maximum token length. + */ +constexpr TokenLength MaxTokenLength = + std::numeric_limits<TokenLength>::max(); + namespace { /** * Structure used to represent the position of a token in the internal @@ -53,12 +59,6 @@ struct TokenMark { bool special; /** - * Maximum token length. - */ - static constexpr TokenLength MaxTokenLength = - std::numeric_limits<TokenLength>::max(); - - /** * Constructor of the TokenMark structure, initializes all members with the * given values. * @@ -450,6 +450,7 @@ public: protectedChars.clear(); offsets.clear(); marks.clear(); + firstLinebreak = 0; currentIndentation = 0; lastIndentation = 0; numLinebreaks = 1; // Assume the stream starts with a linebreak |