From e2fd79ac8c85ac6191f6ed895fa5cdff091f7551 Mon Sep 17 00:00:00 2001 From: Andreas Stöckel Date: Sun, 1 Mar 2015 16:28:35 +0100 Subject: Implemented TokenRegistry class and corresponding simple test case --- src/core/parser/stack/TokenRegistry.cpp | 72 +++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/core/parser/stack/TokenRegistry.cpp (limited to 'src/core/parser/stack/TokenRegistry.cpp') diff --git a/src/core/parser/stack/TokenRegistry.cpp b/src/core/parser/stack/TokenRegistry.cpp new file mode 100644 index 0000000..21ae109 --- /dev/null +++ b/src/core/parser/stack/TokenRegistry.cpp @@ -0,0 +1,72 @@ +/* + Ousía + Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include "TokenRegistry.hpp" + +namespace ousia { +namespace parser_stack { + +TokenId TokenRegistry::registerToken(const std::string &token) +{ + // Check whether the given token is already registered + auto it = tokens.find(token); + if (it != tokens.end()) { + // Increment the reference count + size_t &refCount = it->second.second; + refCount++; + + // Return the token id + return it->second.first; + } + + // Register the token in the parser + TokenId id = parser.registerToken(token); + tokens[token] = std::pair(id, 1); + tokenIds[id] = token; + return id; +} + +void TokenRegistry::unregisterToken(TokenId id) +{ + // Lookup the token corresponding to the given token id + auto tokenIt = tokenIds.find(id); + if (tokenIt != tokenIds.end()) { + const std::string &token = tokenIt->second; + // Lookup the reference count for the corresponding token + auto idIt = tokens.find(token); + if (idIt != tokens.end()) { + // Decrement the reference count, abort if the refCount is larger + // than zero + size_t &refCount = idIt->second.second; + refCount--; + if (refCount > 0) { + return; + } + + // Unregister the token from the parser + parser.unregisterToken(id); + + // Unregister the token from the internal tokens map + tokens.erase(token); + } + // Unregister the token from the internal id map + tokenIds.erase(id); + } +} +} +} -- cgit v1.2.3 From 231f426708babe0964495ac28a54f0f2835c084a Mon Sep 17 00:00:00 2001 From: Andreas Stöckel Date: Mon, 2 Mar 2015 00:35:36 +0100 Subject: Unregistering all registered tokens when TokenRegistry is destroyed --- src/core/parser/stack/TokenRegistry.cpp | 8 ++++ src/core/parser/stack/TokenRegistry.hpp | 47 +++++++++++++++---- test/core/parser/stack/TokenRegistryTest.cpp | 68 +++++++++++++++------------- 3 files changed, 81 insertions(+), 42 deletions(-) (limited to 'src/core/parser/stack/TokenRegistry.cpp') diff --git a/src/core/parser/stack/TokenRegistry.cpp b/src/core/parser/stack/TokenRegistry.cpp index 21ae109..c135b98 100644 --- a/src/core/parser/stack/TokenRegistry.cpp +++ b/src/core/parser/stack/TokenRegistry.cpp @@ -16,11 +16,19 @@ along with this program. If not, see . */ +#include "Callbacks.hpp" #include "TokenRegistry.hpp" namespace ousia { namespace parser_stack { +TokenRegistry::~TokenRegistry() +{ + for (const auto &tid: tokenIds) { + parser.unregisterToken(tid.first); + } +} + TokenId TokenRegistry::registerToken(const std::string &token) { // Check whether the given token is already registered diff --git a/src/core/parser/stack/TokenRegistry.hpp b/src/core/parser/stack/TokenRegistry.hpp index 21c36b5..545db39 100644 --- a/src/core/parser/stack/TokenRegistry.hpp +++ b/src/core/parser/stack/TokenRegistry.hpp @@ -19,7 +19,7 @@ /** * @file TokenRegistry.hpp * - * Contains the TokenRegistry class used for registering all possible tokens + * Contains the TokenRegistry class used for registering all user defined tokens * during the parsing process. * * @author Andreas Stöckel (astoecke@techfak.uni-bielefeld.de) @@ -31,19 +31,22 @@ #include #include -#include "Callbacks.hpp" +#include namespace ousia { namespace parser_stack { +// Forward declarations +class ParserCallbacks; + /** - * The TokenRegistry class is used for registering all possible tokens during - * the Parsing process. The TokenRegistry class acts as an adapter between the - * parser which allocates TokenId for each unique token and the Handler classes - * which may register tokens multiple times and expect the same TokenId to be - * returned for the same token. + * The TokenRegistry class is used for registering all user defined tokens + * during the Parsing process. The TokenRegistry class acts as an adapter + * between the parser which allocates a TokenId for each unique token and the + * Handler classes which may register the same token multiple times and expect + * the same TokenId to be returned for the same token. */ -class TokenRegistry : public ParserCallbacks { +class TokenRegistry { private: /** * Reference at the ParserCallback instance the tokens are relayed to. @@ -71,14 +74,38 @@ public: */ TokenRegistry(ParserCallbacks &parser) : parser(parser) {} + /** + * Destructor of the TokenRegistry class, removes all registered tokens from + * the parser. + */ + ~TokenRegistry(); + /* No copy construction */ TokenRegistry(const TokenRegistry &) = delete; /* No assignment */ TokenRegistry &operator=(const TokenRegistry &) = delete; - TokenId registerToken(const std::string &token) override; - void unregisterToken(TokenId id) override; + /** + * Registers the given string token in the underlying parser and returns the + * TokenId of that token. If the same token string is given multiple times, + * the same TokenId is returned. The token is only registered once in the + * parser. + * + * @param token is the token that should be registered. + * @return the TokenId associated with this token. + */ + TokenId registerToken(const std::string &token); + + /** + * Unregisters the token with the given TokenId from the parser. Note that + * the token will only be unregistered if unregisterToken() has been called + * as many times as registerToken() for the same token. + * + * @param id is the id of the token returned by registerToken() that should + * be unregistered. + */ + void unregisterToken(TokenId id); }; } } diff --git a/test/core/parser/stack/TokenRegistryTest.cpp b/test/core/parser/stack/TokenRegistryTest.cpp index 390851e..20d6cd0 100644 --- a/test/core/parser/stack/TokenRegistryTest.cpp +++ b/test/core/parser/stack/TokenRegistryTest.cpp @@ -18,6 +18,7 @@ #include +#include #include namespace ousia { @@ -40,39 +41,42 @@ public: TEST(TokenRegistry, simple) { ParserCallbacksProxy parser; - TokenRegistry registry(parser); - - ASSERT_EQ(0U, parser.registerTokenCount); - ASSERT_EQ(0U, parser.unregisterTokenCount); - - ASSERT_EQ(1U, registry.registerToken("test")); - ASSERT_EQ(1U, registry.registerToken("test")); - ASSERT_EQ(2U, registry.registerToken("test2")); - ASSERT_EQ(2U, registry.registerToken("test2")); - ASSERT_EQ(2U, parser.registerTokenCount); - ASSERT_EQ(0U, parser.unregisterTokenCount); - - registry.unregisterToken(1); - ASSERT_EQ(2U, parser.registerTokenCount); - ASSERT_EQ(0U, parser.unregisterTokenCount); - - registry.unregisterToken(1); - ASSERT_EQ(2U, parser.registerTokenCount); - ASSERT_EQ(1U, parser.unregisterTokenCount); - - registry.unregisterToken(1); - ASSERT_EQ(2U, parser.registerTokenCount); - ASSERT_EQ(1U, parser.unregisterTokenCount); - - registry.unregisterToken(2); - ASSERT_EQ(2U, parser.registerTokenCount); - ASSERT_EQ(1U, parser.unregisterTokenCount); - - registry.unregisterToken(2); - ASSERT_EQ(2U, parser.registerTokenCount); - ASSERT_EQ(2U, parser.unregisterTokenCount); + { + TokenRegistry registry(parser); + + ASSERT_EQ(0U, parser.registerTokenCount); + ASSERT_EQ(0U, parser.unregisterTokenCount); + + ASSERT_EQ(1U, registry.registerToken("test")); + ASSERT_EQ(1U, registry.registerToken("test")); + ASSERT_EQ(2U, registry.registerToken("test2")); + ASSERT_EQ(2U, registry.registerToken("test2")); + ASSERT_EQ(3U, registry.registerToken("test3")); + ASSERT_EQ(3U, parser.registerTokenCount); + ASSERT_EQ(0U, parser.unregisterTokenCount); + + registry.unregisterToken(1); + ASSERT_EQ(3U, parser.registerTokenCount); + ASSERT_EQ(0U, parser.unregisterTokenCount); + + registry.unregisterToken(1); + ASSERT_EQ(3U, parser.registerTokenCount); + ASSERT_EQ(1U, parser.unregisterTokenCount); + + registry.unregisterToken(1); + ASSERT_EQ(3U, parser.registerTokenCount); + ASSERT_EQ(1U, parser.unregisterTokenCount); + + registry.unregisterToken(2); + ASSERT_EQ(3U, parser.registerTokenCount); + ASSERT_EQ(1U, parser.unregisterTokenCount); + + registry.unregisterToken(2); + ASSERT_EQ(3U, parser.registerTokenCount); + ASSERT_EQ(2U, parser.unregisterTokenCount); + } + ASSERT_EQ(3U, parser.unregisterTokenCount); } - } } -- cgit v1.2.3