diff options
author | Andreas Stöckel <astoecke@techfak.uni-bielefeld.de> | 2015-02-14 23:46:30 +0100 |
---|---|---|
committer | Andreas Stöckel <astoecke@techfak.uni-bielefeld.de> | 2015-02-14 23:46:30 +0100 |
commit | 2b0632764c26728675090c4cd0920f1b7c093ed1 (patch) | |
tree | 9e593a52ec2ec38d5d407fcafe8edbd9b389e835 /src/formats/osdm/DynamicTokenizer.hpp | |
parent | 6df10a61562fa4be7c1c13e7457341571fa6139d (diff) |
Moved textHandlers to whitespace handlers
Diffstat (limited to 'src/formats/osdm/DynamicTokenizer.hpp')
-rw-r--r-- | src/formats/osdm/DynamicTokenizer.hpp | 23 |
1 files changed, 1 insertions, 22 deletions
diff --git a/src/formats/osdm/DynamicTokenizer.hpp b/src/formats/osdm/DynamicTokenizer.hpp index 0cac2e8..3e5aeb3 100644 --- a/src/formats/osdm/DynamicTokenizer.hpp +++ b/src/formats/osdm/DynamicTokenizer.hpp @@ -33,6 +33,7 @@ #include <vector> #include <core/common/Location.hpp> +#include <core/common/Whitespace.hpp> #include "TokenTrie.hpp" @@ -96,28 +97,6 @@ struct DynamicToken { }; /** - * Enum specifying the whitespace handling of the DynamicTokenizer class when - * reading non-token text. - */ -enum class WhitespaceMode { - /** - * Preserves all whitespaces as they are found in the source file. - */ - PRESERVE, - - /** - * Trims whitespace at the beginning and the end of the found text. - */ - TRIM, - - /** - * Whitespaces are trimmed and collapsed, multiple whitespace characters - * are replaced by a single space character. - */ - COLLAPSE -}; - -/** * The DynamicTokenizer is used to extract tokens and chunks of text from a * CharReader. It allows to register and unregister tokens while parsing and * to modify the handling of whitespace characters. Note that the |