4 files changed, 98 insertions, 91 deletions

diff --git a/src/core/model/Ontology.cpp b/src/core/model/Ontology.cpp
index e56d628..bc7b1a7 100644
--- a/src/core/model/Ontology.cpp
+++ b/src/core/model/Ontology.cpp
@@ -315,24 +315,22 @@ bool FieldDescriptor::doValidate(Logger &logger) const
 	} else {
 		valid = valid & validateName(logger);
 	}
-	// check start and end token.
-	if (!startToken.special && !startToken.token.empty() &&
-	    !Utils::isUserDefinedToken(startToken.token)) {
+	// check open and close token.
+	if (!openToken.isValid()) {
 		// TODO: Correct error message.
 		logger.error(std::string("Field \"") + getNameOrDefaultName() +
 		                 "\" of descriptor \"" + parentName +
-		                 "\" has an invalid custom start token: " +
-		                 startToken.token,
+		                 "\" has an invalid custom open token: " +
+		                 openToken.token,
 		             *this);
 		valid = false;
 	}
-	if (!endToken.special && !endToken.token.empty() &&
-	    !Utils::isUserDefinedToken(endToken.token)) {
+	if (!closeToken.isValid()) {
 		// TODO: Correct error message.
 		logger.error(std::string("Field \"") + getNameOrDefaultName() +
 		                 "\" of descriptor \"" + parentName +
-		                 "\" has an invalid custom end token: " +
-		                 endToken.token,
+		                 "\" has an invalid custom close token: " +
+		                 closeToken.token,
 		             *this);
 		valid = false;
 	}
@@ -524,19 +522,17 @@ bool Descriptor::doValidate(Logger &logger) const
 	}
 
 	// check start and end token.
-	if (!startToken.special && !startToken.token.empty() &&
-	    !Utils::isUserDefinedToken(startToken.token)) {
+	if (!openToken.isValid()) {
 		logger.error(std::string("Descriptor \"") + getName() +
 		                 "\" has an invalid custom start token: " +
-		                 startToken.token,
+		                 openToken.token,
 		             *this);
 		valid = false;
 	}
-	if (!endToken.special && !endToken.token.empty() &&
-	    !Utils::isUserDefinedToken(endToken.token)) {
+	if (!closeToken.isValid()) {
 		logger.error(std::string("Descriptor \"") + getName() +
 		                 "\" has an invalid custom end token: " +
-		                 endToken.token,
+		                 closeToken.token,
 		             *this);
 		valid = false;
 	}
@@ -818,8 +814,7 @@ bool StructuredClass::doValidate(Logger &logger) const
 	}
 
 	// check short token.
-	if (!shortToken.special && !shortToken.token.empty() &&
-	    !Utils::isUserDefinedToken(shortToken.token)) {
+	if (!shortToken.isValid()) {
 		logger.error(std::string("Descriptor \"") + getName() +
 		                 "\" has an invalid custom short form token: " +
 		                 shortToken.token,
@@ -1092,22 +1087,22 @@ static void gatherTokenDescriptors(
     std::unordered_set<FieldDescriptor *> &visited)
 {
 	// add the TokenDescriptors for the Descriptor itself.
-	if (!desc->getStartToken().isEmpty()) {
-		res.push_back(desc->getStartTokenPointer());
+	if (!desc->getOpenToken().isEmpty()) {
+		res.push_back(desc->getOpenTokenPointer());
 	}
-	if (!desc->getEndToken().isEmpty()) {
-		res.push_back(desc->getEndTokenPointer());
+	if (!desc->getCloseToken().isEmpty()) {
+		res.push_back(desc->getCloseTokenPointer());
 	}
 	// add the TokenDescriptors for its FieldDescriptors.
 	for (auto fd : desc->getFieldDescriptors()) {
 		if (!visited.insert(fd.get()).second) {
 			continue;
 		}
-		if (!fd->getStartToken().isEmpty()) {
-			res.push_back(fd->getStartTokenPointer());
+		if (!fd->getOpenToken().isEmpty()) {
+			res.push_back(fd->getOpenTokenPointer());
 		}
-		if (!fd->getEndToken().isEmpty()) {
-			res.push_back(fd->getEndTokenPointer());
+		if (!fd->getCloseToken().isEmpty()) {
+			res.push_back(fd->getCloseTokenPointer());
 		}
 	}
 }
diff --git a/src/core/model/Ontology.hpp b/src/core/model/Ontology.hpp
index c90093c..079640c 100644
--- a/src/core/model/Ontology.hpp
+++ b/src/core/model/Ontology.hpp
@@ -228,8 +228,8 @@ private:
 	Owned<Type> primitiveType;
 	bool optional;
 	bool primitive;
-	TokenDescriptor startToken;
-	TokenDescriptor endToken;
+	TokenDescriptor openToken;
+	TokenDescriptor closeToken;
 	WhitespaceMode whitespaceMode;
 
 protected:
@@ -469,7 +469,7 @@ public:
 	}
 
 	/**
-	 * Returns a pointer to the start TokenDescriptor. This Token is used as a
+	 * Returns a pointer to the open TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor starts.
 	 *
 	 * Note that this does not invalidate the FieldDescriptor. So use with
@@ -477,54 +477,54 @@ public:
 	 *
 	 * @return a pointer to the start TokenDescriptor.
 	 */
-	TokenDescriptor *getStartTokenPointer() { return &startToken; }
+	TokenDescriptor *getOpenTokenPointer() { return &openToken; }
 
 	/**
-	 * Returns a copy of the start TokenDescriptor. This Token is used as a
+	 * Returns a copy of the open TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor starts.
 	 *
 	 * @return a copy of the start TokenDescriptor.
 	 */
-	TokenDescriptor getStartToken() const { return startToken; }
+	TokenDescriptor getOpenToken() const { return openToken; }
 
 	/**
-	 * Sets the start TokenDescriptor. This Token is used as a
-	 * signifier during parsing that an instance of this FieldDescriptor starts.
+	 * Sets the open TokenDescriptor. This Token is used as a signifier during
+	 * parsing that an instance of this FieldDescriptor starts.
 	 *
-	 * @param st the new start TokenDescriptor.
+	 * @param t the new open TokenDescriptor.
 	 */
-	void setStartToken(TokenDescriptor st)
+	void setOpenToken(TokenDescriptor t)
 	{
 		invalidate();
-		startToken = st;
+		openToken = t;
 	}
 
 	/**
-	 * Returns a pointer to the end TokenDescriptor. This Token is used as a
+	 * Returns a pointer to the close TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor ends.
 	 *
-	 * @return a pointer to the end TokenDescriptor.
+	 * @return a pointer to the close TokenDescriptor.
 	 */
-	TokenDescriptor *getEndTokenPointer() { return &endToken; }
+	TokenDescriptor *getCloseTokenPointer() { return &closeToken; }
 
 	/**
-	 * Returns a copy of the end TokenDescriptor. This Token is used as a
+	 * Returns a copy of the close TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor ends.
 	 *
-	 * @return a copy of the end TokenDescriptor.
+	 * @return a copy of the close TokenDescriptor.
 	 */
-	TokenDescriptor getEndToken() const { return endToken; }
+	TokenDescriptor getCloseToken() const { return closeToken; }
 
 	/**
-	 * Sets the end TokenDescriptor. This Token is used as a
+	 * Sets the close TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor ends.
 	 *
-	 * @param e the new end TokenDescriptor.
+	 * @param t the new close TokenDescriptor.
 	 */
-	void setEndToken(TokenDescriptor e)
+	void setCloseToken(TokenDescriptor t)
 	{
 		invalidate();
-		endToken = e;
+		closeToken = t;
 	}
 
 	/**
@@ -555,7 +555,7 @@ public:
 	 */
 	SyntaxDescriptor getSyntaxDescriptor(ssize_t depth = -1)
 	{
-		SyntaxDescriptor stx{startToken.id, endToken.id, Tokens::Empty,
+		SyntaxDescriptor stx{openToken.id, closeToken.id, Tokens::Empty,
 		                     const_cast<FieldDescriptor *>(this), depth};
 		return stx;
 	}
@@ -608,8 +608,8 @@ class Descriptor : public Node {
 private:
 	Owned<StructType> attributesDescriptor;
 	NodeVector<FieldDescriptor> fieldDescriptors;
-	TokenDescriptor startToken;
-	TokenDescriptor endToken;
+	TokenDescriptor openToken;
+	TokenDescriptor closeToken;
 
 	bool addAndSortFieldDescriptor(Handle<FieldDescriptor> fd, Logger &logger);
 
@@ -860,31 +860,31 @@ public:
 	NodeVector<StructuredClass> getPermittedChildren() const;
 
 	/**
-	 * Returns a pointer to the start TokenDescriptor. This Token is used as a
+	 * Returns a pointer to the open TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor starts.
 	 *
-	 * @return a pointer to the start TokenDescriptor.
+	 * @return a pointer to the open TokenDescriptor.
 	 */
-	TokenDescriptor *getStartTokenPointer() { return &startToken; }
+	TokenDescriptor *getOpenTokenPointer() { return &openToken; }
 
 	/**
-	 * Returns a copy of the start TokenDescriptor. This Token is used as a
+	 * Returns a copy of the open TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor starts.
 	 *
-	 * @return a copy of the start TokenDescriptor.
+	 * @return a copy of the open TokenDescriptor.
 	 */
-	TokenDescriptor getStartToken() const { return startToken; }
+	TokenDescriptor getOpenToken() const { return openToken; }
 
 	/**
-	 * Sets the start TokenDescriptor. This Token is used as a
+	 * Sets the open TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor starts.
 	 *
-	 * @param st the new start TokenDescriptor.
+	 * @param t the new start TokenDescriptor.
 	 */
-	void setStartToken(TokenDescriptor st)
+	void setOpenToken(TokenDescriptor t)
 	{
 		invalidate();
-		startToken = st;
+		openToken = t;
 	}
 
 	/**
@@ -893,7 +893,7 @@ public:
 	 *
 	 * @return a pointer to the end TokenDescriptor.
 	 */
-	TokenDescriptor *getEndTokenPointer() { return &endToken; }
+	TokenDescriptor *getCloseTokenPointer() { return &closeToken; }
 
 	/**
 	 * Returns a copy of the end TokenDescriptor. This Token is used as a
@@ -901,18 +901,18 @@ public:
 	 *
 	 * @return a copy of the end TokenDescriptor.
 	 */
-	TokenDescriptor getEndToken() const { return endToken; }
+	TokenDescriptor getCloseToken() const { return closeToken; }
 
 	/**
 	 * Sets the end TokenDescriptor. This Token is used as a
 	 * signifier during parsing that an instance of this FieldDescriptor ends.
 	 *
-	 * @param e the new end TokenDescriptor.
+	 * @param t the new end TokenDescriptor.
 	 */
-	void setEndToken(TokenDescriptor e)
+	void setCloseToken(TokenDescriptor t)
 	{
 		invalidate();
-		endToken = e;
+		closeToken = t;
 	}
 
 	/**
@@ -922,7 +922,7 @@ public:
 	 */
 	virtual SyntaxDescriptor getSyntaxDescriptor(ssize_t depth = -1)
 	{
-		SyntaxDescriptor stx{startToken.id, endToken.id, Tokens::Empty,
+		SyntaxDescriptor stx{openToken.id, closeToken.id, Tokens::Empty,
 		                     const_cast<Descriptor *>(this), depth};
 		return stx;
 	}
@@ -938,11 +938,6 @@ public:
 	 */
 	std::vector<SyntaxDescriptor> getPermittedTokens() const;
 };
-/*
- * TODO: We should discuss Cardinalities one more time. Is it smart to define
- * cardinalities independent of context? Should we not have at least have the
- * possibility to define it context-dependently?
- */
 
 /**
  * A StructuredClass specifies nodes in the StructureTree of a document that
@@ -1220,7 +1215,7 @@ public:
 	 */
 	SyntaxDescriptor getSyntaxDescriptor(ssize_t depth = -1) override
 	{
-		SyntaxDescriptor stx{getStartToken().id, getEndToken().id,
+		SyntaxDescriptor stx{getOpenToken().id, getCloseToken().id,
 		                     shortToken.id, const_cast<StructuredClass *>(this),
 		                     depth};
 		return stx;
diff --git a/src/core/model/Syntax.cpp b/src/core/model/Syntax.cpp
index bd17bff..a97acf7 100644
--- a/src/core/model/Syntax.cpp
+++ b/src/core/model/Syntax.cpp
@@ -16,13 +16,21 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "Syntax.hpp"
+#include <core/common/Utils.hpp>
 
 #include "Ontology.hpp"
+#include "Syntax.hpp"
 
 namespace ousia {
 
-/* Class TokenSyntaxDescriptor */
+/* Class TokenDescriptor */
+
+bool TokenDescriptor::isValid() const
+{
+	return special || isEmpty() || Utils::isUserDefinedToken(token);
+}
+
+/* Class SyntaxDescriptor */
 
 bool SyntaxDescriptor::isAnnotation() const
 {
@@ -39,11 +47,11 @@ bool SyntaxDescriptor::isStruct() const
 
 void SyntaxDescriptor::insertIntoTokenSet(TokenSet &set) const
 {
-	if (start != Tokens::Empty) {
-		set.insert(start);
+	if (open != Tokens::Empty) {
+		set.insert(open);
 	}
-	if (end != Tokens::Empty) {
-		set.insert(end);
+	if (close != Tokens::Empty) {
+		set.insert(close);
 	}
 	if (shortForm != Tokens::Empty) {
 		set.insert(shortForm);
@@ -52,7 +60,7 @@ void SyntaxDescriptor::insertIntoTokenSet(TokenSet &set) const
 
 bool SyntaxDescriptor::isEmpty() const
 {
-	return start == Tokens::Empty && end == Tokens::Empty &&
+	return open == Tokens::Empty && close == Tokens::Empty &&
 	       shortForm == Tokens::Empty;
 }
 }
diff --git a/src/core/model/Syntax.hpp b/src/core/model/Syntax.hpp
index 4da3408..4adb329 100644
--- a/src/core/model/Syntax.hpp
+++ b/src/core/model/Syntax.hpp
@@ -74,6 +74,15 @@ struct TokenDescriptor {
 	 * @return true if and only if neither a string nor an ID is given.
 	 */
 	bool isEmpty() const { return token.empty() && id == Tokens::Empty; }
+
+	/**
+	 * Returns true if the token is valid, which is the case if this class is
+	 * either marked as special token or is empty or does have a valid token
+	 * string set.
+	 *
+	 * @return true if the token descriptor is valid, false otherwise.
+	 */
+	bool isValid() const;
 };
 
 /**
@@ -86,17 +95,17 @@ struct TokenDescriptor {
  */
 struct SyntaxDescriptor {
 	/**
-	 * Possible start token or Tokens::Empty if no token is set.
+	 * Possible open token or Tokens::Empty if no token is set.
 	 */
-	TokenId start;
+	TokenId open;
 
 	/**
-	 * Possible end token or Tokens::Empty if no token is set.
+	 * Possible close token or Tokens::Empty if no token is set.
 	 */
-	TokenId end;
+	TokenId close;
 
 	/**
-	 * Possible representation token or Tokens::Empty if no token is set.
+	 * Possible short form token or Tokens::Empty if no token is set.
 	 */
 	TokenId shortForm;
 
@@ -118,8 +127,8 @@ struct SyntaxDescriptor {
 	 * descriptor handle to nullptr.
 	 */
 	SyntaxDescriptor()
-	    : start(Tokens::Empty),
-	      end(Tokens::Empty),
+	    : open(Tokens::Empty),
+	      close(Tokens::Empty),
 	      shortForm(Tokens::Empty),
 	      descriptor(nullptr),
 	      depth(-1)
@@ -129,18 +138,18 @@ struct SyntaxDescriptor {
 	/**
 	 * Member initializer constructor.
 	 *
-	 * @param start is a possible start token.
-	 * @param end is a possible end token.
+	 * @param open is a possible open token.
+	 * @param close is a possible close token.
 	 * @param shortForm is a possible short form token.
 	 * @param descriptor The Descriptor this SyntaxDescriptor belongs to.
 	 * @param depth Given the current leaf in the parsed document the depth of a
 	 * SyntaxDescriptor is defined as the number of transparent elements that
 	 * would be needed to construct an instance of the referenced descriptor.
 	 */
-	SyntaxDescriptor(TokenId start, TokenId end, TokenId shortForm,
+	SyntaxDescriptor(TokenId open, TokenId close, TokenId shortForm,
 	                 Handle<Node> descriptor, ssize_t depth)
-	    : start(start),
-	      end(end),
+	    : open(open),
+	      close(close),
 	      shortForm(shortForm),
 	      descriptor(descriptor),
 	      depth(depth)
@@ -193,4 +202,4 @@ struct SyntaxDescriptor {
 	bool isEmpty() const;
 };
 }
-#endif
\ No newline at end of file
+#endif