From 2b9ac0c0a26b2c32277f3e17e12f69c2ec27bd4f Mon Sep 17 00:00:00 2001 From: Andreas Stöckel Date: Wed, 4 Mar 2015 21:43:36 +0100 Subject: Prepared implementation of user defined tokens --- src/core/parser/stack/DocumentHandler.cpp | 97 +++++++++++++++++++++++-------- src/core/parser/stack/DocumentHandler.hpp | 25 +++++++- src/core/parser/stack/Stack.cpp | 42 +++++++++++-- src/core/parser/stack/TokenStack.cpp | 16 ++--- src/core/parser/stack/TokenStack.hpp | 31 ---------- 5 files changed, 141 insertions(+), 70 deletions(-) (limited to 'src/core') diff --git a/src/core/parser/stack/DocumentHandler.cpp b/src/core/parser/stack/DocumentHandler.cpp index c239583..78629ac 100644 --- a/src/core/parser/stack/DocumentHandler.cpp +++ b/src/core/parser/stack/DocumentHandler.cpp @@ -42,8 +42,10 @@ bool DocumentHandler::startCommand(Variant::mapType &args) Rooted document = context().getProject()->createDocument(args["name"].asString()); document->setLocation(location()); + scope().push(document); scope().setFlag(ParserFlag::POST_HEAD, false); + scope().setFlag(ParserFlag::POST_USER_DEFINED_TOKEN_REGISTRATION, false); return true; } @@ -55,6 +57,30 @@ void DocumentHandler::end() { scope().pop(logger()); } DocumentChildHandler::DocumentChildHandler(const HandlerData &handlerData) : Handler(handlerData), isExplicitField(false) { + // Register all user defined tokens if this has not yet been done + if (!scope().getFlag(ParserFlag::POST_USER_DEFINED_TOKEN_REGISTRATION)) { + registerUserDefinedTokens(); + } +} + +void DocumentChildHandler::registerUserDefinedTokens() +{ + // Set the POST_USER_DEFINED_TOKEN_REGISTRATION flag, to prevent this method + // from being called again + scope().setFlag(ParserFlag::POST_USER_DEFINED_TOKEN_REGISTRATION, true); + + // Fetch the underlying document and all ontologies registered in the + // document and register all user defined tokens in the parser + Rooted doc = scope().selectOrThrow(); + for (Rooted ontology : doc->getOntologies()) { + std::vector tokens = + ontology->getAllTokenDescriptors(); + for (TokenDescriptor *token : tokens) { + if (!token->special) { + token->id = registerToken(token->token); + } + } + } } void DocumentChildHandler::preamble(Rooted &parentNode, size_t &fieldIdx, @@ -79,16 +105,42 @@ void DocumentChildHandler::preamble(Rooted &parentNode, size_t &fieldIdx, } } +void DocumentChildHandler::pushDocumentField(Handle parent, + Handle fieldDescr, + size_t fieldIdx, bool transparent) +{ + // Push the field onto the scope + Rooted field = + new DocumentField(manager(), parent, fieldIdx, transparent); + field->setLocation(location()); + scope().push(field); + + // Push all possible tokens onto the stack + //pushTokens(fieldDescr->getPermittedTokens()); +} + +void DocumentChildHandler::popDocumentField() +{ + // Pop the field from the scope, make sure it actually is a DocumentField + assert(scope().getLeaf()->isa(&RttiTypes::DocumentField)); + scope().pop(logger()); + + // Pop the registered tokens from the stack + //popTokens(); +} + void DocumentChildHandler::createPath(const NodeVector &path, DocumentEntity *&parent, size_t p0) { size_t S = path.size(); for (size_t p = p0; p < S; p = p + 2) { // add the field. - Rooted field{new DocumentField( - manager(), scope().getLeaf(), - parent->getDescriptor()->getFieldDescriptorIndex(), true)}; - scope().push(field); + const ssize_t fieldIdx = + parent->getDescriptor()->getFieldDescriptorIndex(); + const Rooted fieldDescr = + parent->getDescriptor()->getFieldDescriptor(fieldIdx); + pushDocumentField(scope().getLeaf(), fieldDescr, fieldIdx, true); + // add the transparent/implicit structure element. Rooted transparent = parent->createChildStructuredEntity(path[p].cast(), @@ -99,11 +151,11 @@ void DocumentChildHandler::createPath(const NodeVector &path, scope().push(transparent); parent = static_cast(transparent.get()); } - // add the last field. - Rooted field{new DocumentField( - manager(), scope().getLeaf(), - parent->getDescriptor()->getFieldDescriptorIndex(), true)}; - scope().push(field); + // add the field. + const ssize_t fieldIdx = parent->getDescriptor()->getFieldDescriptorIndex(); + const Rooted fieldDescr = + parent->getDescriptor()->getFieldDescriptor(fieldIdx); + pushDocumentField(scope().getLeaf(), fieldDescr, fieldIdx, true); // Generally allow explicit fields in the new field scope().setFlag(ParserFlag::POST_EXPLICIT_FIELDS, false); @@ -199,10 +251,11 @@ bool DocumentChildHandler::startCommand(Variant::mapType &args) "references to the beginning."), location()); } else { - Rooted field{new DocumentField( - manager(), parentNode, newFieldIdx, false)}; - field->setLocation(location()); - scope().push(field); + pushDocumentField( + parentNode, + parent->getDescriptor()->getFieldDescriptor( + newFieldIdx), + newFieldIdx, false); isExplicitField = true; return true; } @@ -230,7 +283,7 @@ bool DocumentChildHandler::startCommand(Variant::mapType &args) // if we have transparent elements above us in the structure // tree we try to unwind them before we give up. // pop the implicit field. - scope().pop(logger()); + popDocumentField(); // pop the implicit element. scope().pop(logger()); continue; @@ -298,7 +351,7 @@ bool DocumentChildHandler::startAnnotation(Variant::mapType &args) // if we have transparent elements above us in the structure // tree we try to unwind them before we give up. // pop the implicit field. - scope().pop(logger()); + popDocumentField(); // pop the implicit element. scope().pop(logger()); continue; @@ -390,8 +443,7 @@ void DocumentChildHandler::end() case HandlerType::COMMAND: case HandlerType::ANNOTATION_START: // In case of explicit fields we do not want to pop something from - // the - // stack. + // the stack. if (!isExplicitField) { // pop the "main" element. scope().pop(logger()); @@ -437,10 +489,7 @@ bool DocumentChildHandler::fieldStart(bool &isDefault, size_t fieldIdx) isDefault = fieldIdx == fields.size() - 1; } // push the field on the stack. - Rooted field{ - new DocumentField(manager(), parentNode, fieldIdx, false)}; - field->setLocation(location()); - scope().push(field); + pushDocumentField(parentNode, fields[fieldIdx], fieldIdx, false); // Generally allow explicit fields in the new field scope().setFlag(ParserFlag::POST_EXPLICIT_FIELDS, false); @@ -453,7 +502,7 @@ void DocumentChildHandler::fieldEnd() assert(scope().getLeaf()->isa(&RttiTypes::DocumentField)); // Pop the field from the stack. - scope().pop(logger()); + popDocumentField(); // Pop all remaining transparent elements. while (scope().getLeaf()->isa(&RttiTypes::StructuredEntity) && @@ -461,7 +510,7 @@ void DocumentChildHandler::fieldEnd() // Pop the transparent element. scope().pop(logger()); // Pop the transparent field. - scope().pop(logger()); + popDocumentField(); } } @@ -598,4 +647,4 @@ namespace RttiTypes { const Rtti DocumentField = RttiBuilder( "DocumentField").parent(&Node); } -} \ No newline at end of file +} diff --git a/src/core/parser/stack/DocumentHandler.hpp b/src/core/parser/stack/DocumentHandler.hpp index 6f9f4a2..47534fb 100644 --- a/src/core/parser/stack/DocumentHandler.hpp +++ b/src/core/parser/stack/DocumentHandler.hpp @@ -98,9 +98,14 @@ private: bool isExplicitField; /** - * Code shared by both the start(), fieldStart() and the data() method. - * Checks whether the parser currently is in a field and returns the name - * of this field. + * Registers all user defined tokens in the parser. + */ + void registerUserDefinedTokens(); + + /** + * Code shared by the startCommand(), startAnnotation(), fieldStart() and + * the data() method. Checks whether the parser currently is in a field and + * returns the name of this field. * * @param parentNode is the next possible parent node (a document, * a structured entity, an annotation entity or a field). @@ -163,6 +168,20 @@ private: bool convertData(Handle field, Variant &data, Logger &logger); + /** + * Pushes a new DocumentField onto the scope stack and registers all + * premitted tokens in the parser. + */ + void pushDocumentField(Handle parent, + Handle fieldDescr, size_t fieldIdx, + bool transparent); + + /** + * Pops a DocumentField from the scope stack and retracts the permitted + * tokens from the parser. + */ + void popDocumentField(); + public: DocumentChildHandler(const HandlerData &handlerData); diff --git a/src/core/parser/stack/Stack.cpp b/src/core/parser/stack/Stack.cpp index bd16b43..436d3a5 100644 --- a/src/core/parser/stack/Stack.cpp +++ b/src/core/parser/stack/Stack.cpp @@ -35,6 +35,9 @@ #include #endif +// TODO: Remove +#include + namespace ousia { namespace parser_stack { namespace { @@ -302,6 +305,11 @@ private: */ TokenRegistry tokenRegistry; + /** + * Collection of all currently enabled tokens. + */ + TokenStack tokenStack; + /** * Pointer at a TokenizedDataReader instance from which the data should * currently be read. @@ -361,6 +369,12 @@ private: */ HandlerInfo ¤tInfo(); + /** + * Returns a reference at the current HandlerInfo instance (or a stub + * HandlerInfo instance if the stack is empty). + */ + const HandlerInfo ¤tInfo() const; + /** * Returns a reference at the last HandlerInfo instance (or a stub * HandlerInfo instance if the stack has only one element). @@ -591,7 +605,9 @@ std::string StackImpl::currentCommandName() const TokenSet StackImpl::currentTokens() const { - // TODO: Implement + if (currentInfo().state().supportsTokens) { + return tokenStack.tokens(); + } return TokenSet{}; } @@ -605,6 +621,12 @@ HandlerInfo &StackImpl::currentInfo() { return stack.empty() ? EmptyHandlerInfo : stack.back(); } + +const HandlerInfo &StackImpl::currentInfo() const +{ + return stack.empty() ? EmptyHandlerInfo : stack.back(); +} + HandlerInfo &StackImpl::lastInfo() { return stack.size() < 2U ? EmptyHandlerInfo : stack[stack.size() - 2]; @@ -658,7 +680,8 @@ bool StackImpl::prepareCurrentHandler(bool startImplicitDefaultField) // end it and repeat bool canHaveImplicitDefaultField = info.type() == HandlerType::COMMAND || - info.type() == HandlerType::TOKEN || (info.type() == HandlerType::ANNOTATION_START && info.range); + info.type() == HandlerType::TOKEN || + (info.type() == HandlerType::ANNOTATION_START && info.range); if (info.hadDefaultField || !startImplicitDefaultField || !info.valid || !canHaveImplicitDefaultField) { // We cannot end the command if it is marked as "range" command @@ -757,6 +780,7 @@ void StackImpl::handleData() void StackImpl::handleToken(const Token &token) { + std::cout << "Got token " << token.id << std::endl; // TODO: Implement // Just eat them for now } @@ -1060,12 +1084,19 @@ void StackImpl::unregisterToken(TokenId id) void StackImpl::pushTokens(const std::vector &tokens) { - // TODO + // Push the tokens onto the token stack + for (const SyntaxDescriptor &token: tokens) { + std::cout << token.open << std::endl; + std::cout << token.close << std::endl; + std::cout << token.shortForm << std::endl; + } + tokenStack.pushTokens(tokens); } void StackImpl::popTokens() { - // TODO + // Pop the last set of tokens from the token stack. + tokenStack.popTokens(); } Variant StackImpl::readData() @@ -1123,7 +1154,8 @@ void Stack::annotationStart(const Variant &className, impl->annotationStart(className, args, range); } -void Stack::annotationEnd(const Variant &className, const Variant::mapType &args) +void Stack::annotationEnd(const Variant &className, + const Variant::mapType &args) { #if STACK_DEBUG_OUTPUT std::cout << "STACK: annotationEnd " << className << " " << args diff --git a/src/core/parser/stack/TokenStack.cpp b/src/core/parser/stack/TokenStack.cpp index ac1d94e..b6fc6e1 100644 --- a/src/core/parser/stack/TokenStack.cpp +++ b/src/core/parser/stack/TokenStack.cpp @@ -18,6 +18,8 @@ #include "TokenStack.hpp" +#include + namespace ousia { namespace parser_stack { @@ -26,17 +28,17 @@ void TokenStack::pushTokens(const std::vector &tokens) stack.push_back(tokens); } -void TokenStack::popTokens() { stack.pop_back(); } +void TokenStack::popTokens() { + stack.pop_back(); +} TokenSet TokenStack::tokens() const { - if (stack.empty() && parentStack != nullptr) { - return parentStack->tokens(); - } - TokenSet res; - for (const SyntaxDescriptor &descr : stack.back()) { - descr.insertIntoTokenSet(res); + if (!stack.empty()) { + for (const SyntaxDescriptor &descr : stack.back()) { + descr.insertIntoTokenSet(res); + } } return res; } diff --git a/src/core/parser/stack/TokenStack.hpp b/src/core/parser/stack/TokenStack.hpp index f2e7edc..667b976 100644 --- a/src/core/parser/stack/TokenStack.hpp +++ b/src/core/parser/stack/TokenStack.hpp @@ -43,44 +43,13 @@ namespace parser_stack { */ class TokenStack { private: - /** - * Shared pointer at the parent TokenStack instance. May be nullptr, in - * which case no parent TokenStack instance exists. - */ - const TokenStack *parentStack; - /** * Stack containing vectors of TokenSyntaxDescriptor instances as given by * the user. */ std::vector> stack; - /** - * Constructor of the TokenStack class. - * - * @param parentStack is a pointer at the underlying parentStack instance - * to which calls should be forwarded if no data has been pushed onto this - * stack instance. - */ - TokenStack(const TokenStack *parentStack) : parentStack(parentStack) {} - public: - /** - * Default constructor of the TokenStack class with no reference at a parent - * stack. - */ - TokenStack() : TokenStack(nullptr) {} - - /** - * Constructor of the TokenStack class with a reference at a parent - * TokenStack instance. - * - * @param parentStack is a reference at a parent TokenStack instance. If no - * data has yet been pushed onto this instance, calls will be forwarded to - * the parent stack. - */ - TokenStack(const TokenStack &parentStack) : TokenStack(&parentStack) {} - /** * Pushes a list of SyntaxDescriptor instances onto the internal stack. * -- cgit v1.2.3