diff options
Diffstat (limited to 'src/core/utils')
-rw-r--r-- | src/core/utils/Tokenizer.cpp | 13 | ||||
-rw-r--r-- | src/core/utils/Tokenizer.hpp | 8 |
2 files changed, 10 insertions, 11 deletions
diff --git a/src/core/utils/Tokenizer.cpp b/src/core/utils/Tokenizer.cpp index 1a84f0c..38f7585 100644 --- a/src/core/utils/Tokenizer.cpp +++ b/src/core/utils/Tokenizer.cpp @@ -21,11 +21,11 @@ namespace ousia { namespace utils { -static std::unordered_map<char, TokenTreeNode> buildChildren( +static std::map<char, TokenTreeNode> buildChildren( const std::map<std::string, int> &inputs) { - std::std::unordered_map<char, TokenTreeNode> children; - std::unordered_map<char, std::map<std::string, int>> nexts; + std::map<char, TokenTreeNode> children; + std::map<char, std::map<std::string, int>> nexts; for (auto &e : inputs) { const std::string &s = e.first; @@ -57,7 +57,7 @@ static int buildId(const std::map<std::string, int> &inputs) if (e.first.empty()) { if (tokenId != -1) { throw TokenizerException{std::string{"Ambigous token found: "} + - e.second}; + std::to_string(e.second)}; } else { tokenId = e.second; } @@ -67,10 +67,9 @@ static int buildId(const std::map<std::string, int> &inputs) } TokenTreeNode::TokenTreeNode(const std::map<std::string, int> &inputs) - : children(buildChildren(inputs), tokenId(buildId(inputs))) + : children(buildChildren(inputs)), tokenId(buildId(inputs)) + { } - } } - diff --git a/src/core/utils/Tokenizer.hpp b/src/core/utils/Tokenizer.hpp index 1d0db43..24c4f30 100644 --- a/src/core/utils/Tokenizer.hpp +++ b/src/core/utils/Tokenizer.hpp @@ -20,7 +20,7 @@ #define _OUSIA_UTILS_TOKENIZER_HPP_ #include <istream> -#include <unordered_map> +#include <map> #include <queue> namespace ousia { @@ -30,14 +30,14 @@ class TokenizerException : public std::exception { public: const std::string msg; - ArgumentValidatorError(const std::string &msg) : msg(msg){}; + TokenizerException(const std::string &msg) : msg(msg){}; virtual const char *what() const noexcept override { return msg.c_str(); } }; class TokenTreeNode { public: - const std::unordered_map<char, TokenTreeNode> children; + const std::map<char, TokenTreeNode> children; const int tokenId; TokenTreeNode(const std::map<std::string, int> &inputs); @@ -59,7 +59,7 @@ class Tokenizer { private: const std::istream &input; const TokenTreeNode root; - const std::queue<Token> peek; + const std::queue<Token> peekQueue; public: Tokenizer(const TokenTreeNode &root, std::istream &input); |