diff options
author | Andreas Stöckel <astoecke@techfak.uni-bielefeld.de> | 2015-02-14 23:42:05 +0100 |
---|---|---|
committer | Andreas Stöckel <astoecke@techfak.uni-bielefeld.de> | 2015-02-14 23:42:05 +0100 |
commit | efe60ac3c3a8725ac71329c0bb19fa9d9c58f399 (patch) | |
tree | 97802b0eec92d51be02b94c939c1285feeaf3b4f /test/plugins | |
parent | c1776468bc3daab431d0e2b51589dd12df595227 (diff) |
Moved specific file format parsers to formats/ folder, moved old tokenizer to css code (this is the only place where it is actually used)
Diffstat (limited to 'test/plugins')
-rw-r--r-- | test/plugins/css/CodeTokenizerTest.cpp | 100 | ||||
-rw-r--r-- | test/plugins/css/TokenizerTest.cpp | 118 | ||||
-rw-r--r-- | test/plugins/xml/XmlParserTest.cpp | 314 |
3 files changed, 218 insertions, 314 deletions
diff --git a/test/plugins/css/CodeTokenizerTest.cpp b/test/plugins/css/CodeTokenizerTest.cpp new file mode 100644 index 0000000..2d4d5a7 --- /dev/null +++ b/test/plugins/css/CodeTokenizerTest.cpp @@ -0,0 +1,100 @@ +/* + Ousía + Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include <gtest/gtest.h> + +#include <core/CodeTokenizer.hpp> + +namespace ousia { + +static const int BLOCK_COMMENT = 30; +static const int LINE_COMMENT = 31; +static const int STRING = 20; +static const int ESCAPE = 21; +static const int LINEBREAK = 21; +static const int CURLY_OPEN = 40; +static const int CURLY_CLOSE = 41; + +TEST(CodeTokenizer, testTokenizer) +{ + CharReader reader{ + "/**\n" // 1 + " * Some Block Comment\n" // 2 + " */\n" // 3 + "var my_string = 'My \\'String\\'';\n" // 4 + "// and a line comment\n" // 5 + "var my_obj = { a = 4;}", 0}; // 6 + // 123456789012345678901234567890123456789 + // 0 1 2 3 + TokenTreeNode root{{{"/*", 1}, + {"*/", 2}, + {"//", 3}, + {"'", 4}, + {"\\", 5}, + {"{", CURLY_OPEN}, + {"}", CURLY_CLOSE}, + {"\n", 6}}}; + std::map<int, CodeTokenDescriptor> descriptors{ + // the block comment start Token has the id 1 and if the Tokenizer + // returns a Block Comment Token that should have the id 10. + {1, {CodeTokenMode::BLOCK_COMMENT_START, BLOCK_COMMENT}}, + {2, {CodeTokenMode::BLOCK_COMMENT_END, BLOCK_COMMENT}}, + {3, {CodeTokenMode::LINE_COMMENT, LINE_COMMENT}}, + {4, {CodeTokenMode::STRING_START_END, STRING}}, + {5, {CodeTokenMode::ESCAPE, ESCAPE}}, + {6, {CodeTokenMode::LINEBREAK, LINEBREAK}}}; + + std::vector<Token> expected = { + {BLOCK_COMMENT, "*\n * Some Block Comment\n ", SourceLocation{0, 0, 29}}, + {LINEBREAK, "\n", SourceLocation{0, 29, 30}}, + {TOKEN_TEXT, "var", SourceLocation{0, 30, 33}}, + {TOKEN_TEXT, "my_string", SourceLocation{0, 34, 43}}, + {TOKEN_TEXT, "=", SourceLocation{0, 44, 45}}, + {STRING, "My 'String'", SourceLocation{0, 46, 61}}, + {TOKEN_TEXT, ";", SourceLocation{0, 61, 62}}, + {LINEBREAK, "\n", SourceLocation{0, 62, 63}}, + // this is slightly counter-intuitive but makes sense if you think about + // it: As a line comment is ended by a line break the line break is + // technically still a part of the line comment and thus the ending + // is in the next line. + {LINE_COMMENT, " and a line comment", SourceLocation{0, 63, 85}}, + {TOKEN_TEXT, "var", SourceLocation{0, 85, 88}}, + {TOKEN_TEXT, "my_obj", SourceLocation{0, 89, 95}}, + {TOKEN_TEXT, "=", SourceLocation{0, 96, 97}}, + {CURLY_OPEN, "{", SourceLocation{0, 98, 99}}, + {TOKEN_TEXT, "a", SourceLocation{0, 100, 101}}, + {TOKEN_TEXT, "=", SourceLocation{0, 102, 103}}, + {TOKEN_TEXT, "4;", SourceLocation{0, 104, 106}}, + {CURLY_CLOSE, "}", SourceLocation{0, 106, 107}}, + }; + + CodeTokenizer tokenizer{reader, root, descriptors}; + + Token t; + for (auto &te : expected) { + EXPECT_TRUE(tokenizer.next(t)); + EXPECT_EQ(te.tokenId, t.tokenId); + EXPECT_EQ(te.content, t.content); + EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId()); + EXPECT_EQ(te.location.getStart(), t.location.getStart()); + EXPECT_EQ(te.location.getEnd(), t.location.getEnd()); + } + ASSERT_FALSE(tokenizer.next(t)); +} +} + diff --git a/test/plugins/css/TokenizerTest.cpp b/test/plugins/css/TokenizerTest.cpp new file mode 100644 index 0000000..c53f93d --- /dev/null +++ b/test/plugins/css/TokenizerTest.cpp @@ -0,0 +1,118 @@ +/* + Ousía + Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include <gtest/gtest.h> + +#include <core/common/CharReader.hpp> + +#include <core/Tokenizer.hpp> + +namespace ousia { +TEST(TokenTreeNode, testConstructor) +{ + TokenTreeNode root{{{"a", 1}, {"aab", 2}, {"aac", 3}, {"abd", 4}}}; + + ASSERT_EQ(-1, root.tokenId); + ASSERT_EQ(1U, root.children.size()); + ASSERT_TRUE(root.children.find('a') != root.children.end()); + + const TokenTreeNode &a = root.children.at('a'); + ASSERT_EQ(1, a.tokenId); + ASSERT_EQ(2U, a.children.size()); + ASSERT_TRUE(a.children.find('a') != a.children.end()); + ASSERT_TRUE(a.children.find('b') != a.children.end()); + + const TokenTreeNode &aa = a.children.at('a'); + ASSERT_EQ(-1, aa.tokenId); + ASSERT_EQ(2U, aa.children.size()); + ASSERT_TRUE(aa.children.find('b') != aa.children.end()); + ASSERT_TRUE(aa.children.find('c') != aa.children.end()); + + const TokenTreeNode &aab = aa.children.at('b'); + ASSERT_EQ(2, aab.tokenId); + ASSERT_EQ(0U, aab.children.size()); + + const TokenTreeNode &aac = aa.children.at('c'); + ASSERT_EQ(3, aac.tokenId); + ASSERT_EQ(0U, aac.children.size()); + + const TokenTreeNode &ab = a.children.at('b'); + ASSERT_EQ(-1, ab.tokenId); + ASSERT_EQ(1U, ab.children.size()); + ASSERT_TRUE(ab.children.find('d') != ab.children.end()); + + const TokenTreeNode &abd = ab.children.at('d'); + ASSERT_EQ(4, abd.tokenId); + ASSERT_EQ(0U, abd.children.size()); +} + +TEST(Tokenizer, testTokenization) +{ + TokenTreeNode root{{{"/", 1}, {"/*", 2}, {"*/", 3}}}; + + CharReader reader{"Test/Test /* Block Comment */", 0}; + // 012345678901234567890123456789 + // 0 1 2 + + std::vector<Token> expected = { + {TOKEN_TEXT, "Test", SourceLocation{0, 0, 4}}, + {1, "/", SourceLocation{0, 4, 5}}, + {TOKEN_TEXT, "Test ", SourceLocation{0, 5, 10}}, + {2, "/*", SourceLocation{0, 10, 12}}, + {TOKEN_TEXT, " Block Comment ", SourceLocation{0, 12, 27}}, + {3, "*/", SourceLocation{0, 27, 29}}}; + + Tokenizer tokenizer{reader, root}; + + Token t; + for (auto &te : expected) { + EXPECT_TRUE(tokenizer.next(t)); + EXPECT_EQ(te.tokenId, t.tokenId); + EXPECT_EQ(te.content, t.content); + EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId()); + EXPECT_EQ(te.location.getStart(), t.location.getStart()); + EXPECT_EQ(te.location.getEnd(), t.location.getEnd()); + } + ASSERT_FALSE(tokenizer.next(t)); +} + +TEST(Tokenizer, testIncompleteTokens) +{ + TokenTreeNode root{{{"ab", 1}, {"c", 2}}}; + + CharReader reader{"ac", 0}; + + std::vector<Token> expected = { + {TOKEN_TEXT, "a", SourceLocation{0, 0, 1}}, + {2, "c", SourceLocation{0, 1, 2}}}; + + Tokenizer tokenizer{reader, root}; + + Token t; + for (auto &te : expected) { + EXPECT_TRUE(tokenizer.next(t)); + EXPECT_EQ(te.tokenId, t.tokenId); + EXPECT_EQ(te.content, t.content); + EXPECT_EQ(te.location.getSourceId(), t.location.getSourceId()); + EXPECT_EQ(te.location.getStart(), t.location.getStart()); + EXPECT_EQ(te.location.getEnd(), t.location.getEnd()); + } + ASSERT_FALSE(tokenizer.next(t)); +} +} + diff --git a/test/plugins/xml/XmlParserTest.cpp b/test/plugins/xml/XmlParserTest.cpp deleted file mode 100644 index af1ef56..0000000 --- a/test/plugins/xml/XmlParserTest.cpp +++ /dev/null @@ -1,314 +0,0 @@ -/* - Ousía - Copyright (C) 2014, 2015 Benjamin Paaßen, Andreas Stöckel - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -#include <iostream> - -#include <gtest/gtest.h> - -#include <core/common/CharReader.hpp> -#include <core/common/SourceContextReader.hpp> -#include <core/model/Domain.hpp> -#include <core/model/Node.hpp> -#include <core/model/Project.hpp> -#include <core/frontend/TerminalLogger.hpp> -#include <core/StandaloneEnvironment.hpp> - -#include <plugins/filesystem/FileLocator.hpp> -#include <plugins/xml/XmlParser.hpp> - -namespace ousia { - -namespace RttiTypes { -extern const Rtti Document; -extern const Rtti Domain; -extern const Rtti Typesystem; -} - -struct XmlStandaloneEnvironment : public StandaloneEnvironment { - XmlParser xmlParser; - FileLocator fileLocator; - - XmlStandaloneEnvironment(ConcreteLogger &logger) - : StandaloneEnvironment(logger) - { - fileLocator.addDefaultSearchPaths(); - fileLocator.addUnittestSearchPath("xmlparser"); - - registry.registerDefaultExtensions(); - registry.registerParser({"text/vnd.ousia.oxm", "text/vnd.ousia.oxd"}, - {&RttiTypes::Node}, &xmlParser); - registry.registerResourceLocator(&fileLocator); - } -}; - -static TerminalLogger logger(std::cerr, true); - -TEST(XmlParser, mismatchedTag) -{ - XmlStandaloneEnvironment env(logger); - env.parse("mismatchedTag.oxm", "", "", RttiSet{&RttiTypes::Document}); - ASSERT_TRUE(logger.hasError()); -} - -TEST(XmlParser, generic) -{ - XmlStandaloneEnvironment env(logger); - env.parse("generic.oxm", "", "", RttiSet{&RttiTypes::Node}); -#ifdef MANAGER_GRAPHVIZ_EXPORT - env.manager.exportGraphviz("xmlDocument.dot"); -#endif -} - -static void checkAttributes(Handle<StructType> expected, - Handle<Descriptor> desc) -{ - if (expected == nullptr) { - ASSERT_TRUE(desc->getAttributesDescriptor()->getAttributes().empty()); - } else { - ASSERT_EQ(expected->getName(), - desc->getAttributesDescriptor()->getName()); - auto &attrs_exp = expected->getAttributes(); - auto &attrs = desc->getAttributesDescriptor()->getAttributes(); - ASSERT_EQ(attrs_exp.size(), attrs.size()); - for (size_t i = 0; i < attrs_exp.size(); i++) { - ASSERT_EQ(attrs_exp[i]->getName(), attrs[i]->getName()); - ASSERT_EQ(attrs_exp[i]->getType(), attrs[i]->getType()); - ASSERT_EQ(attrs_exp[i]->isOptional(), attrs[i]->isOptional()); - ASSERT_EQ(attrs_exp[i]->getDefaultValue(), - attrs[i]->getDefaultValue()); - } - } -} - -static void checkStructuredClass( - Handle<Node> n, const std::string &name, Handle<Domain> domain, - Variant cardinality = Cardinality::any(), - Handle<StructType> attributesDescriptor = nullptr, - Handle<StructuredClass> superclass = nullptr, bool transparent = false, - bool root = false) -{ - ASSERT_FALSE(n == nullptr); - Handle<StructuredClass> sc = n.cast<StructuredClass>(); - ASSERT_FALSE(sc == nullptr); - ASSERT_EQ(name, sc->getName()); - ASSERT_EQ(domain, sc->getParent()); - ASSERT_EQ(cardinality, sc->getCardinality()); - ASSERT_EQ(transparent, sc->isTransparent()); - ASSERT_EQ(root, sc->hasRootPermission()); - checkAttributes(attributesDescriptor, sc); -} - -static Rooted<StructuredClass> checkStructuredClass( - const std::string &resolve, const std::string &name, Handle<Domain> domain, - Variant cardinality = Cardinality::any(), - Handle<StructType> attributesDescriptor = nullptr, - Handle<StructuredClass> superclass = nullptr, bool transparent = false, - bool root = false) -{ - auto res = domain->resolve(&RttiTypes::StructuredClass, resolve); - if (res.size() != 1) { - throw OusiaException("resolution error!"); - } - Handle<StructuredClass> sc = res[0].node.cast<StructuredClass>(); - checkStructuredClass(sc, name, domain, cardinality, attributesDescriptor, - superclass, transparent, root); - return sc; -} - -static void checkAnnotationClass( - Handle<Node> n, const std::string &name, Handle<Domain> domain, - Handle<StructType> attributesDescriptor = nullptr) -{ - ASSERT_FALSE(n == nullptr); - Handle<AnnotationClass> ac = n.cast<AnnotationClass>(); - ASSERT_FALSE(ac == nullptr); - ASSERT_EQ(name, ac->getName()); - ASSERT_EQ(domain, ac->getParent()); - checkAttributes(attributesDescriptor, ac); -} - -static Rooted<AnnotationClass> checkAnnotationClass( - const std::string &resolve, const std::string &name, Handle<Domain> domain, - Handle<StructType> attributesDescriptor = nullptr) -{ - auto res = domain->resolve(&RttiTypes::AnnotationClass, resolve); - if (res.size() != 1) { - throw OusiaException("resolution error!"); - } - Handle<AnnotationClass> ac = res[0].node.cast<AnnotationClass>(); - checkAnnotationClass(ac, name, domain, attributesDescriptor); - return ac; -} - -static void checkFieldDescriptor( - Handle<Node> n, const std::string &name, Handle<Descriptor> parent, - NodeVector<StructuredClass> children, - FieldDescriptor::FieldType type = FieldDescriptor::FieldType::TREE, - Handle<Type> primitiveType = nullptr, bool optional = false) -{ - ASSERT_FALSE(n == nullptr); - Handle<FieldDescriptor> field = n.cast<FieldDescriptor>(); - ASSERT_FALSE(field.isNull()); - ASSERT_EQ(name, field->getName()); - ASSERT_EQ(parent, field->getParent()); - ASSERT_EQ(type, field->getFieldType()); - ASSERT_EQ(primitiveType, field->getPrimitiveType()); - ASSERT_EQ(optional, field->isOptional()); - // check the children. - ASSERT_EQ(children.size(), field->getChildren().size()); - for (unsigned int c = 0; c < children.size(); c++) { - ASSERT_EQ(children[c], field->getChildren()[c]); - } -} - -static void checkFieldDescriptor( - Handle<Descriptor> desc, Handle<Descriptor> parent, - NodeVector<StructuredClass> children, - const std::string &name = DEFAULT_FIELD_NAME, - FieldDescriptor::FieldType type = FieldDescriptor::FieldType::TREE, - Handle<Type> primitiveType = nullptr, bool optional = false) -{ - auto res = desc->resolve(&RttiTypes::FieldDescriptor, name); - ASSERT_EQ(1, res.size()); - checkFieldDescriptor(res[0].node, name, parent, children, type, - primitiveType, optional); -} - -static void checkFieldDescriptor( - Handle<Descriptor> desc, NodeVector<StructuredClass> children, - const std::string &name = DEFAULT_FIELD_NAME, - FieldDescriptor::FieldType type = FieldDescriptor::FieldType::TREE, - Handle<Type> primitiveType = nullptr, bool optional = false) -{ - checkFieldDescriptor(desc, desc, children, name, type, primitiveType, - optional); -} - -TEST(XmlParser, domainParsing) -{ - XmlStandaloneEnvironment env(logger); - Rooted<Node> book_domain_node = - env.parse("book_domain.oxm", "", "", RttiSet{&RttiTypes::Domain}); - ASSERT_FALSE(book_domain_node == nullptr); - ASSERT_FALSE(logger.hasError()); - // check the domain node. - Rooted<Domain> book_domain = book_domain_node.cast<Domain>(); - ASSERT_EQ("book", book_domain->getName()); - // get the book struct node. - Cardinality single; - single.merge({1}); - Rooted<StructType> bookAuthor{ - new StructType(book_domain->getManager(), "", nullptr)}; - bookAuthor->addAttribute( - {new Attribute(book_domain->getManager(), "author", - env.project->getSystemTypesystem()->getStringType(), - "")}, - logger); - Rooted<StructuredClass> book = checkStructuredClass( - "book", "book", book_domain, single, bookAuthor, nullptr, false, true); - // get the chapter struct node. - Rooted<StructuredClass> chapter = - checkStructuredClass("chapter", "chapter", book_domain); - Rooted<StructuredClass> section = - checkStructuredClass("section", "section", book_domain); - Rooted<StructuredClass> subsection = - checkStructuredClass("subsection", "subsection", book_domain); - Rooted<StructuredClass> paragraph = - checkStructuredClass("paragraph", "paragraph", book_domain, - Cardinality::any(), nullptr, nullptr, true, false); - Rooted<StructuredClass> text = - checkStructuredClass("text", "text", book_domain, Cardinality::any(), - nullptr, nullptr, true, false); - - // check the FieldDescriptors. - checkFieldDescriptor(book, {chapter, paragraph}); - checkFieldDescriptor(chapter, {section, paragraph}); - checkFieldDescriptor(section, {subsection, paragraph}); - checkFieldDescriptor(subsection, {paragraph}); - checkFieldDescriptor(paragraph, {text}); - checkFieldDescriptor( - text, {}, DEFAULT_FIELD_NAME, FieldDescriptor::FieldType::PRIMITIVE, - env.project->getSystemTypesystem()->getStringType(), false); - - // check parent handling using the headings domain. - Rooted<Node> headings_domain_node = - env.parse("headings_domain.oxm", "", "", RttiSet{&RttiTypes::Domain}); - ASSERT_FALSE(headings_domain_node == nullptr); - ASSERT_FALSE(logger.hasError()); - Rooted<Domain> headings_domain = headings_domain_node.cast<Domain>(); - // now there should be a heading struct. - Rooted<StructuredClass> heading = - checkStructuredClass("heading", "heading", headings_domain, single, - nullptr, nullptr, true, false); - // which should be a reference to the paragraph descriptor. - checkFieldDescriptor(heading, paragraph, {text}); - // and each struct in the book domain (except for text) should have a - // heading field now. - checkFieldDescriptor(book, {heading}, "heading", - FieldDescriptor::FieldType::SUBTREE, nullptr, true); - checkFieldDescriptor(chapter, {heading}, "heading", - FieldDescriptor::FieldType::SUBTREE, nullptr, true); - checkFieldDescriptor(section, {heading}, "heading", - FieldDescriptor::FieldType::SUBTREE, nullptr, true); - checkFieldDescriptor(subsection, {heading}, "heading", - FieldDescriptor::FieldType::SUBTREE, nullptr, true); - checkFieldDescriptor(paragraph, {heading}, "heading", - FieldDescriptor::FieldType::SUBTREE, nullptr, true); - - // check annotation handling using the comments domain. - Rooted<Node> comments_domain_node = - env.parse("comments_domain.oxm", "", "", RttiSet{&RttiTypes::Domain}); - ASSERT_FALSE(comments_domain_node == nullptr); - ASSERT_FALSE(logger.hasError()); - Rooted<Domain> comments_domain = comments_domain_node.cast<Domain>(); - // now we should be able to find a comment annotation. - Rooted<AnnotationClass> comment_anno = - checkAnnotationClass("comment", "comment", comments_domain); - // as well as a comment struct - Rooted<StructuredClass> comment = - checkStructuredClass("comment", "comment", comments_domain); - // and a reply struct - Rooted<StructuredClass> reply = - checkStructuredClass("reply", "reply", comments_domain); - // check the fields for each of them. - { - std::vector<Rooted<Descriptor>> descs{comment_anno, comment, reply}; - for (auto &d : descs) { - checkFieldDescriptor(d, {paragraph}, "content", - FieldDescriptor::FieldType::SUBTREE, nullptr, - false); - checkFieldDescriptor(d, {reply}, "replies", - FieldDescriptor::FieldType::SUBTREE, nullptr, - false); - } - } - // paragraph should have comment as child now as well. - checkFieldDescriptor(paragraph, {text, comment}); - // as should heading, because it references the paragraph default field. - checkFieldDescriptor(heading, paragraph, {text, comment}); -} - -TEST(XmlParser, documentParsing) -{ - XmlStandaloneEnvironment env(logger); - Rooted<Node> book_domain_node = - env.parse("simple_book.oxd", "", "", RttiSet{&RttiTypes::Document}); - //TODO: Check result -} -} - |