summaryrefslogtreecommitdiff
path: root/src/formats/osdm/DynamicTokenizer.hpp
diff options
context:
space:
mode:
authorAndreas Stöckel <astoecke@techfak.uni-bielefeld.de>2015-02-14 23:46:30 +0100
committerAndreas Stöckel <astoecke@techfak.uni-bielefeld.de>2015-02-14 23:46:30 +0100
commit2b0632764c26728675090c4cd0920f1b7c093ed1 (patch)
tree9e593a52ec2ec38d5d407fcafe8edbd9b389e835 /src/formats/osdm/DynamicTokenizer.hpp
parent6df10a61562fa4be7c1c13e7457341571fa6139d (diff)
Moved textHandlers to whitespace handlers
Diffstat (limited to 'src/formats/osdm/DynamicTokenizer.hpp')
-rw-r--r--src/formats/osdm/DynamicTokenizer.hpp23
1 files changed, 1 insertions, 22 deletions
diff --git a/src/formats/osdm/DynamicTokenizer.hpp b/src/formats/osdm/DynamicTokenizer.hpp
index 0cac2e8..3e5aeb3 100644
--- a/src/formats/osdm/DynamicTokenizer.hpp
+++ b/src/formats/osdm/DynamicTokenizer.hpp
@@ -33,6 +33,7 @@
#include <vector>
#include <core/common/Location.hpp>
+#include <core/common/Whitespace.hpp>
#include "TokenTrie.hpp"
@@ -96,28 +97,6 @@ struct DynamicToken {
};
/**
- * Enum specifying the whitespace handling of the DynamicTokenizer class when
- * reading non-token text.
- */
-enum class WhitespaceMode {
- /**
- * Preserves all whitespaces as they are found in the source file.
- */
- PRESERVE,
-
- /**
- * Trims whitespace at the beginning and the end of the found text.
- */
- TRIM,
-
- /**
- * Whitespaces are trimmed and collapsed, multiple whitespace characters
- * are replaced by a single space character.
- */
- COLLAPSE
-};
-
-/**
* The DynamicTokenizer is used to extract tokens and chunks of text from a
* CharReader. It allows to register and unregister tokens while parsing and
* to modify the handling of whitespace characters. Note that the