1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
/*
Ousía
Copyright (C) 2014 Benjamin Paaßen, Andreas Stöckel
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _OUSIA_UTILS_CODE_TOKENIZER_HPP_
#define _OUSIA_UTILS_CODE_TOKENIZER_HPP_
#include <map>
#include <sstream>
#include "BufferedCharReader.hpp"
#include "Tokenizer.hpp"
namespace ousia {
namespace utils {
enum class CodeTokenMode {
STRING_START_END,
LINE_COMMENT,
BLOCK_COMMENT_START,
BLOCK_COMMENT_END,
LINEBREAK,
ESCAPE,
NONE
};
struct CodeTokenDescriptor {
CodeTokenMode mode;
int id;
CodeTokenDescriptor(CodeTokenMode mode, int id) : mode(mode), id(id) {}
};
enum class CodeTokenizerState {
NORMAL,
IN_BLOCK_COMMENT,
IN_LINE_COMMENT,
IN_STRING
};
class CodeTokenizer : public Tokenizer {
private:
std::map<int, CodeTokenDescriptor> descriptors;
CodeTokenizerState state;
std::stringstream buf;
Token startToken;
int returnTokenId;
bool escaped = false;
Token constructToken(const Token& t);
void buffer(const Token& t);
protected:
bool doPrepare(const Token &t, std::deque<Token> &peeked) override;
public:
bool ignoreComments = false;
CodeTokenizer(BufferedCharReader &input, const TokenTreeNode &root,
std::map<int, CodeTokenDescriptor> descriptors)
: Tokenizer(input, root), descriptors(descriptors)
{
}
};
}
}
#endif
|
