summaryrefslogtreecommitdiff
path: root/src/core
diff options
context:
space:
mode:
Diffstat (limited to 'src/core')
-rw-r--r--src/core/utils/Tokenizer.cpp13
-rw-r--r--src/core/utils/Tokenizer.hpp8
2 files changed, 10 insertions, 11 deletions
diff --git a/src/core/utils/Tokenizer.cpp b/src/core/utils/Tokenizer.cpp
index 1a84f0c..38f7585 100644
--- a/src/core/utils/Tokenizer.cpp
+++ b/src/core/utils/Tokenizer.cpp
@@ -21,11 +21,11 @@
namespace ousia {
namespace utils {
-static std::unordered_map<char, TokenTreeNode> buildChildren(
+static std::map<char, TokenTreeNode> buildChildren(
const std::map<std::string, int> &inputs)
{
- std::std::unordered_map<char, TokenTreeNode> children;
- std::unordered_map<char, std::map<std::string, int>> nexts;
+ std::map<char, TokenTreeNode> children;
+ std::map<char, std::map<std::string, int>> nexts;
for (auto &e : inputs) {
const std::string &s = e.first;
@@ -57,7 +57,7 @@ static int buildId(const std::map<std::string, int> &inputs)
if (e.first.empty()) {
if (tokenId != -1) {
throw TokenizerException{std::string{"Ambigous token found: "} +
- e.second};
+ std::to_string(e.second)};
} else {
tokenId = e.second;
}
@@ -67,10 +67,9 @@ static int buildId(const std::map<std::string, int> &inputs)
}
TokenTreeNode::TokenTreeNode(const std::map<std::string, int> &inputs)
- : children(buildChildren(inputs), tokenId(buildId(inputs)))
+ : children(buildChildren(inputs)), tokenId(buildId(inputs))
+
{
}
-
}
}
-
diff --git a/src/core/utils/Tokenizer.hpp b/src/core/utils/Tokenizer.hpp
index 1d0db43..24c4f30 100644
--- a/src/core/utils/Tokenizer.hpp
+++ b/src/core/utils/Tokenizer.hpp
@@ -20,7 +20,7 @@
#define _OUSIA_UTILS_TOKENIZER_HPP_
#include <istream>
-#include <unordered_map>
+#include <map>
#include <queue>
namespace ousia {
@@ -30,14 +30,14 @@ class TokenizerException : public std::exception {
public:
const std::string msg;
- ArgumentValidatorError(const std::string &msg) : msg(msg){};
+ TokenizerException(const std::string &msg) : msg(msg){};
virtual const char *what() const noexcept override { return msg.c_str(); }
};
class TokenTreeNode {
public:
- const std::unordered_map<char, TokenTreeNode> children;
+ const std::map<char, TokenTreeNode> children;
const int tokenId;
TokenTreeNode(const std::map<std::string, int> &inputs);
@@ -59,7 +59,7 @@ class Tokenizer {
private:
const std::istream &input;
const TokenTreeNode root;
- const std::queue<Token> peek;
+ const std::queue<Token> peekQueue;
public:
Tokenizer(const TokenTreeNode &root, std::istream &input);