summaryrefslogtreecommitdiff
path: root/src/formats/osdm/DynamicTokenizer.hpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/formats/osdm/DynamicTokenizer.hpp')
-rw-r--r--src/formats/osdm/DynamicTokenizer.hpp23
1 files changed, 1 insertions, 22 deletions
diff --git a/src/formats/osdm/DynamicTokenizer.hpp b/src/formats/osdm/DynamicTokenizer.hpp
index 0cac2e8..3e5aeb3 100644
--- a/src/formats/osdm/DynamicTokenizer.hpp
+++ b/src/formats/osdm/DynamicTokenizer.hpp
@@ -33,6 +33,7 @@
#include <vector>
#include <core/common/Location.hpp>
+#include <core/common/Whitespace.hpp>
#include "TokenTrie.hpp"
@@ -96,28 +97,6 @@ struct DynamicToken {
};
/**
- * Enum specifying the whitespace handling of the DynamicTokenizer class when
- * reading non-token text.
- */
-enum class WhitespaceMode {
- /**
- * Preserves all whitespaces as they are found in the source file.
- */
- PRESERVE,
-
- /**
- * Trims whitespace at the beginning and the end of the found text.
- */
- TRIM,
-
- /**
- * Whitespaces are trimmed and collapsed, multiple whitespace characters
- * are replaced by a single space character.
- */
- COLLAPSE
-};
-
-/**
* The DynamicTokenizer is used to extract tokens and chunks of text from a
* CharReader. It allows to register and unregister tokens while parsing and
* to modify the handling of whitespace characters. Note that the