Implemented TokenSyntaxDescriptor structure

1 files changed, 15 insertions, 1 deletions

diff --git a/src/core/common/Token.cpp b/src/core/common/Token.cpp
index 8bcdbb5..e454ae4 100644
--- a/src/core/common/Token.cpp
+++ b/src/core/common/Token.cpp
@@ -19,6 +19,20 @@
 #include "Token.hpp"
 
 namespace ousia {
-// Stub to make sure Tokens.hpp is valid
+
+/* Class TokenSyntaxDescriptor */
+
+void TokenSyntaxDescriptor::insertIntoTokenSet(TokenSet &set) const
+{
+	if (start != Tokens::Empty) {
+		set.insert(start);
+	}
+	if (end != Tokens::Empty) {
+		set.insert(end);
+	}
+	if (shortForm != Tokens::Empty) {
+		set.insert(shortForm);
+	}
+}
 }