Implemented TokenRegistry class and corresponding simple test case

4 files changed, 239 insertions, 0 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e2d7f7..6e021fd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -188,6 +188,7 @@ ADD_LIBRARY(ousia_core
 #	src/core/parser/stack/ImportIncludeHandler
 	src/core/parser/stack/State
 #	src/core/parser/stack/Stack
+	src/core/parser/stack/TokenRegistry
 #	src/core/parser/stack/TypesystemHandler
 	src/core/parser/utils/SourceOffsetVector
 	src/core/parser/utils/TokenizedData
@@ -325,6 +326,7 @@ IF(TEST)
 		test/core/parser/ParserScopeTest
 #		test/core/parser/stack/StackTest
 		test/core/parser/stack/StateTest
+		test/core/parser/stack/TokenRegistryTest
 		test/core/parser/utils/SourceOffsetVectorTest
 		test/core/parser/utils/TokenizedDataTest
 		test/core/parser/utils/TokenizerTest
diff --git a/src/core/parser/stack/TokenRegistry.cpp b/src/core/parser/stack/TokenRegistry.cpp
new file mode 100644
index 0000000..21ae109
--- /dev/null
+++ b/src/core/parser/stack/TokenRegistry.cpp
@@ -0,0 +1,72 @@
+/*
+    Ousía
+    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "TokenRegistry.hpp"
+
+namespace ousia {
+namespace parser_stack {
+
+TokenId TokenRegistry::registerToken(const std::string &token)
+{
+	// Check whether the given token is already registered
+	auto it = tokens.find(token);
+	if (it != tokens.end()) {
+		// Increment the reference count
+		size_t &refCount = it->second.second;
+		refCount++;
+
+		// Return the token id
+		return it->second.first;
+	}
+
+	// Register the token in the parser
+	TokenId id = parser.registerToken(token);
+	tokens[token] = std::pair<TokenId, size_t>(id, 1);
+	tokenIds[id] = token;
+	return id;
+}
+
+void TokenRegistry::unregisterToken(TokenId id)
+{
+	// Lookup the token corresponding to the given token id
+	auto tokenIt = tokenIds.find(id);
+	if (tokenIt != tokenIds.end()) {
+		const std::string &token = tokenIt->second;
+		// Lookup the reference count for the corresponding token
+		auto idIt = tokens.find(token);
+		if (idIt != tokens.end()) {
+			// Decrement the reference count, abort if the refCount is larger
+			// than zero
+			size_t &refCount = idIt->second.second;
+			refCount--;
+			if (refCount > 0) {
+				return;
+			}
+
+			// Unregister the token from the parser
+			parser.unregisterToken(id);
+
+			// Unregister the token from the internal tokens map
+			tokens.erase(token);
+		}
+		// Unregister the token from the internal id map
+		tokenIds.erase(id);
+	}
+}
+}
+}
diff --git a/src/core/parser/stack/TokenRegistry.hpp b/src/core/parser/stack/TokenRegistry.hpp
new file mode 100644
index 0000000..21c36b5
--- /dev/null
+++ b/src/core/parser/stack/TokenRegistry.hpp
@@ -0,0 +1,87 @@
+/*
+    Ousía
+    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/**
+ * @file TokenRegistry.hpp
+ *
+ * Contains the TokenRegistry class used for registering all possible tokens
+ * during the parsing process.
+ *
+ * @author Andreas Stöckel (astoecke@techfak.uni-bielefeld.de)
+ */
+
+#ifndef _OUSIA_PARSER_STACK_TOKEN_REGISTRY_HPP_
+#define _OUSIA_PARSER_STACK_TOKEN_REGISTRY_HPP_
+
+#include <string>
+#include <unordered_map>
+
+#include "Callbacks.hpp"
+
+namespace ousia {
+namespace parser_stack {
+
+/**
+ * The TokenRegistry class is used for registering all possible tokens during
+ * the Parsing process. The TokenRegistry class acts as an adapter between the
+ * parser which allocates TokenId for each unique token and the Handler classes
+ * which may register tokens multiple times and expect the same TokenId to be
+ * returned for the same token.
+ */
+class TokenRegistry : public ParserCallbacks {
+private:
+	/**
+	 * Reference at the ParserCallback instance the tokens are relayed to.
+	 */
+	ParserCallbacks &parser;
+
+	/**
+	 * Store containing all TokenId instances for all registered tokens. The map
+	 * maps from the token strings to the corresponding TokenId and a reference
+	 * count.
+	 */
+	std::unordered_map<std::string, std::pair<TokenId, size_t>> tokens;
+
+	/**
+	 * Reverse map containing the string corresponding to a TokenId.
+	 */
+	std::unordered_map<TokenId, std::string> tokenIds;
+
+public:
+	/**
+	 * Constructor of the TokenRegistry class.
+	 *
+	 * @param parser is the underlying parser implementing the ParserCallbacks
+	 * interface to which all calls are relayed.
+	 */
+	TokenRegistry(ParserCallbacks &parser) : parser(parser) {}
+
+	/* No copy construction */
+	TokenRegistry(const TokenRegistry &) = delete;
+
+	/* No assignment */
+	TokenRegistry &operator=(const TokenRegistry &) = delete;
+
+	TokenId registerToken(const std::string &token) override;
+	void unregisterToken(TokenId id) override;
+};
+}
+}
+
+#endif /* _OUSIA_PARSER_STACK_TOKEN_REGISTRY_HPP_ */
+
diff --git a/test/core/parser/stack/TokenRegistryTest.cpp b/test/core/parser/stack/TokenRegistryTest.cpp
new file mode 100644
index 0000000..390851e
--- /dev/null
+++ b/test/core/parser/stack/TokenRegistryTest.cpp
@@ -0,0 +1,78 @@
+/*
+    Ousía
+    Copyright (C) 2014, 2015  Benjamin Paaßen, Andreas Stöckel
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <gtest/gtest.h>
+
+#include <core/parser/stack/TokenRegistry.hpp>
+
+namespace ousia {
+namespace parser_stack {
+
+class ParserCallbacksProxy : public ParserCallbacks {
+public:
+	size_t registerTokenCount = 0;
+	size_t unregisterTokenCount = 0;
+
+	TokenId registerToken(const std::string &token) override
+	{
+		registerTokenCount++;
+		return registerTokenCount;
+	}
+
+	void unregisterToken(TokenId id) override { unregisterTokenCount++; }
+};
+
+TEST(TokenRegistry, simple)
+{
+	ParserCallbacksProxy parser;
+	TokenRegistry registry(parser);
+
+	ASSERT_EQ(0U, parser.registerTokenCount);
+	ASSERT_EQ(0U, parser.unregisterTokenCount);
+
+	ASSERT_EQ(1U, registry.registerToken("test"));
+	ASSERT_EQ(1U, registry.registerToken("test"));
+	ASSERT_EQ(2U, registry.registerToken("test2"));
+	ASSERT_EQ(2U, registry.registerToken("test2"));
+	ASSERT_EQ(2U, parser.registerTokenCount);
+	ASSERT_EQ(0U, parser.unregisterTokenCount);
+
+	registry.unregisterToken(1);
+	ASSERT_EQ(2U, parser.registerTokenCount);
+	ASSERT_EQ(0U, parser.unregisterTokenCount);
+
+	registry.unregisterToken(1);
+	ASSERT_EQ(2U, parser.registerTokenCount);
+	ASSERT_EQ(1U, parser.unregisterTokenCount);
+
+	registry.unregisterToken(1);
+	ASSERT_EQ(2U, parser.registerTokenCount);
+	ASSERT_EQ(1U, parser.unregisterTokenCount);
+
+	registry.unregisterToken(2);
+	ASSERT_EQ(2U, parser.registerTokenCount);
+	ASSERT_EQ(1U, parser.unregisterTokenCount);
+
+	registry.unregisterToken(2);
+	ASSERT_EQ(2U, parser.registerTokenCount);
+	ASSERT_EQ(2U, parser.unregisterTokenCount);
+}
+
+}
+}
+