diff options
| -rw-r--r-- | src/core/CodeTokenizer.hpp | 23 |
1 files changed, 21 insertions, 2 deletions
diff --git a/src/core/CodeTokenizer.hpp b/src/core/CodeTokenizer.hpp index b392c11..0b34d53 100644 --- a/src/core/CodeTokenizer.hpp +++ b/src/core/CodeTokenizer.hpp @@ -105,8 +105,27 @@ public: /** * * @param input a BufferedCharReader containing the input for this - *tokenizer, - * as with a regular tokenizer. + * tokenizer, as with a regular tokenizer. + * @param root a TokenTreeNode representing the root of the TokenTree. + * Please note that you have to specify all tokenIDs here that you use + * in the descriptors map. + * @param descriptors a map mapping tokenIDs to CodeTokenDescriptors. + * In this way you can specify the meaning of certain Tokens. Say you + * specified the Token "//" with the id 1 in the TokenTree. Then you could + * add the entry "1" with the Mode "LINE_COMMENT" to the descriptors map + * and this CodeTokenizer would recognize the token "//" as starting a + * line comment. + */ + CodeTokenizer(BufferedCharReader &input, const TokenTreeNode &root, + std::map<int, CodeTokenDescriptor> descriptors) + : Tokenizer(input, root), descriptors(descriptors), state(CodeTokenizerState::NORMAL) + { + } + + /** + * + * @param input a BufferedCharReader containing the input for this + * tokenizer, as with a regular tokenizer. * @param root a TokenTreeNode representing the root of the TokenTree. * Please note that you have to specify all tokenIDs here that you use * in the descriptors map. |