Implemented registration of user-defined tokens, fixed comment handling (do not issue multiple data events if a comment occurs, just skip the comment data like in TeX)

2 files changed, 47 insertions, 14 deletions

diff --git a/src/formats/osml/OsmlStreamParser.cpp b/src/formats/osml/OsmlStreamParser.cpp
index e467dc5..823075a 100644
--- a/src/formats/osml/OsmlStreamParser.cpp
+++ b/src/formats/osml/OsmlStreamParser.cpp
@@ -16,6 +16,10 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <cassert>
+#include <stack>
+#include <vector>
+
 #include <core/common/CharReader.hpp>
 #include <core/common/Logger.hpp>
 #include <core/common/Utils.hpp>
@@ -27,9 +31,6 @@
 
 #include "OsmlStreamParser.hpp"
 
-#include <stack>
-#include <vector>
-
 namespace ousia {
 
 namespace {
@@ -127,7 +128,7 @@ private:
 	/**
 	 * Set to true if this is a command with clear begin and end.
 	 */
-	bool hasRange: 1;
+	bool hasRange;
 
 public:
 	/**
@@ -407,6 +408,9 @@ public:
 
 	State parse();
 
+	TokenId registerToken(const std::string &token);
+	void unregisterToken(TokenId token);
+
 	const TokenizedData &getData() const { return data; }
 	const Variant &getCommandName() const { return cmd().getName(); }
 	const Variant &getCommandArguments() const { return cmd().getArguments(); }
@@ -700,10 +704,11 @@ OsmlStreamParserImpl::State OsmlStreamParserImpl::parseCommand(
 void OsmlStreamParserImpl::parseBlockComment()
 {
 	Token token;
+	TokenizedData commentData;
 	size_t depth = 1;
-	while (tokenizer.read(reader, token, data)) {
+	while (tokenizer.read(reader, token, commentData)) {
 		// Throw the comment data away
-		data.clear();
+		commentData.clear();
 
 		if (token.id == OsmlTokens.BlockCommentEnd) {
 			depth--;
@@ -822,6 +827,14 @@ OsmlStreamParserImpl::State OsmlStreamParserImpl::parse()
 		} else if (type == Tokens::Data) {
 			reader.consumePeek();
 			continue;
+		} else if (type == OsmlTokens.LineComment) {
+			reader.consumePeek();
+			parseLineComment();
+			continue;
+		} else if (type == OsmlTokens.BlockCommentStart) {
+			reader.consumePeek();
+			parseBlockComment();
+			continue;
 		}
 
 		// A non-text token was reached, make sure all pending data commands
@@ -836,11 +849,7 @@ OsmlStreamParserImpl::State OsmlStreamParserImpl::parse()
 		// Synchronize the location with the current token location
 		location = token.location;
 
-		if (token.id == OsmlTokens.LineComment) {
-			parseLineComment();
-		} else if (token.id == OsmlTokens.BlockCommentStart) {
-			parseBlockComment();
-		} else if (token.id == OsmlTokens.FieldStart) {
+		if (token.id == OsmlTokens.FieldStart) {
 			cmd().pushField(false, token.location);
 			return State::FIELD_START;
 		} else if (token.id == OsmlTokens.FieldEnd) {
@@ -914,6 +923,16 @@ OsmlStreamParserImpl::State OsmlStreamParserImpl::parse()
 	return State::END;
 }
 
+TokenId OsmlStreamParserImpl::registerToken(const std::string &token)
+{
+	return tokenizer.registerToken(token, false);
+}
+
+void OsmlStreamParserImpl::unregisterToken(TokenId token)
+{
+	assert(tokenizer.unregisterToken(token));
+}
+
 /* Class OsmlStreamParser */
 
 OsmlStreamParser::OsmlStreamParser(CharReader &reader, Logger &logger)
@@ -955,4 +974,13 @@ bool OsmlStreamParser::inDefaultField() const { return impl->inDefaultField(); }
 
 bool OsmlStreamParser::inRangeCommand() const { return impl->inRangeCommand(); }
 
+TokenId OsmlStreamParser::registerToken(const std::string &token)
+{
+	return impl->registerToken(token);
+}
+
+void OsmlStreamParser::unregisterToken(TokenId token)
+{
+	impl->unregisterToken(token);
+}
 }
diff --git a/src/formats/osml/OsmlStreamParser.hpp b/src/formats/osml/OsmlStreamParser.hpp
index 10d5296..b7e64f7 100644
--- a/src/formats/osml/OsmlStreamParser.hpp
+++ b/src/formats/osml/OsmlStreamParser.hpp
@@ -32,6 +32,8 @@
 #include <cstdint>
 #include <memory>
 
+#include <core/parser/stack/Callbacks.hpp>
+
 namespace ousia {
 
 // Forward declarations
@@ -50,10 +52,10 @@ class Variant;
  * syntactically valid and tries to recorver from most errors. If an error is
  * irrecoverable (this is the case for errors with wrong nesting of commands or
  * fields, as this would lead to too many consecutive errors) a
- * LoggableException is thrown. The OsmlStreamParser can be compared to a SAX
- * parser for XML.
+ * LoggableException is thrown. In short, the OsmlStreamParser can be described
+ * as a SAX parser for OSML.
  */
-class OsmlStreamParser {
+class OsmlStreamParser: public parser_stack::ParserCallbacks {
 public:
 	/**
 	 * Enum used to indicate which state the OsmlStreamParser class is in
@@ -204,6 +206,9 @@ public:
 	 * "{!" syntax).
 	 */
 	bool inDefaultField() const;
+
+	TokenId registerToken(const std::string &token) override;
+	void unregisterToken(TokenId token) override;
 };
 }