diff options
Diffstat (limited to 'src/core/utils/Tokenizer.cpp')
| -rw-r--r-- | src/core/utils/Tokenizer.cpp | 13 |
1 files changed, 6 insertions, 7 deletions
diff --git a/src/core/utils/Tokenizer.cpp b/src/core/utils/Tokenizer.cpp index 1a84f0c..38f7585 100644 --- a/src/core/utils/Tokenizer.cpp +++ b/src/core/utils/Tokenizer.cpp @@ -21,11 +21,11 @@ namespace ousia { namespace utils { -static std::unordered_map<char, TokenTreeNode> buildChildren( +static std::map<char, TokenTreeNode> buildChildren( const std::map<std::string, int> &inputs) { - std::std::unordered_map<char, TokenTreeNode> children; - std::unordered_map<char, std::map<std::string, int>> nexts; + std::map<char, TokenTreeNode> children; + std::map<char, std::map<std::string, int>> nexts; for (auto &e : inputs) { const std::string &s = e.first; @@ -57,7 +57,7 @@ static int buildId(const std::map<std::string, int> &inputs) if (e.first.empty()) { if (tokenId != -1) { throw TokenizerException{std::string{"Ambigous token found: "} + - e.second}; + std::to_string(e.second)}; } else { tokenId = e.second; } @@ -67,10 +67,9 @@ static int buildId(const std::map<std::string, int> &inputs) } TokenTreeNode::TokenTreeNode(const std::map<std::string, int> &inputs) - : children(buildChildren(inputs), tokenId(buildId(inputs))) + : children(buildChildren(inputs)), tokenId(buildId(inputs)) + { } - } } - |