diff options
Diffstat (limited to 'src/formats/osdm/DynamicTokenizer.hpp')
-rw-r--r-- | src/formats/osdm/DynamicTokenizer.hpp | 23 |
1 files changed, 1 insertions, 22 deletions
diff --git a/src/formats/osdm/DynamicTokenizer.hpp b/src/formats/osdm/DynamicTokenizer.hpp index 0cac2e8..3e5aeb3 100644 --- a/src/formats/osdm/DynamicTokenizer.hpp +++ b/src/formats/osdm/DynamicTokenizer.hpp @@ -33,6 +33,7 @@ #include <vector> #include <core/common/Location.hpp> +#include <core/common/Whitespace.hpp> #include "TokenTrie.hpp" @@ -96,28 +97,6 @@ struct DynamicToken { }; /** - * Enum specifying the whitespace handling of the DynamicTokenizer class when - * reading non-token text. - */ -enum class WhitespaceMode { - /** - * Preserves all whitespaces as they are found in the source file. - */ - PRESERVE, - - /** - * Trims whitespace at the beginning and the end of the found text. - */ - TRIM, - - /** - * Whitespaces are trimmed and collapsed, multiple whitespace characters - * are replaced by a single space character. - */ - COLLAPSE -}; - -/** * The DynamicTokenizer is used to extract tokens and chunks of text from a * CharReader. It allows to register and unregister tokens while parsing and * to modify the handling of whitespace characters. Note that the |