Identifiers may not end with an underscore

5 files changed, 50 insertions, 3 deletions

diff --git a/src/core/common/Utils.cpp b/src/core/common/Utils.cpp
index a87ff6d..67920c2 100644
--- a/src/core/common/Utils.cpp
+++ b/src/core/common/Utils.cpp
@@ -36,7 +36,7 @@ bool Utils::isIdentifier(const std::string &name)
 		}
 		first = false;
 	}
-	return !first;
+	return !first && isIdentifierEndCharacter(name.back());
 }
 
 bool Utils::isIdentifierOrEmpty(const std::string &name)
@@ -150,4 +150,4 @@ bool Utils::isUserDefinedToken(const std::string &token)
 	}
 	return false;
 }
-}
\ No newline at end of file
+}
diff --git a/src/core/common/Utils.hpp b/src/core/common/Utils.hpp
index d9e26da..c3b49a0 100644
--- a/src/core/common/Utils.hpp
+++ b/src/core/common/Utils.hpp
@@ -66,6 +66,14 @@ public:
 	}
 
 	/**
+	 * Returns true if the given character is in [A-Za-z0-9].
+	 */
+	static bool isIdentifierEndCharacter(const char c)
+	{
+		return isAlphanumeric(c);
+	}
+
+	/**
 	 * Returns true if the given character is in [A-Za-z0-9_-].
 	 */
 	static bool isIdentifierCharacter(const char c)
diff --git a/src/formats/osml/OsmlStreamParser.cpp b/src/formats/osml/OsmlStreamParser.cpp
index daf800a..acad57b 100644
--- a/src/formats/osml/OsmlStreamParser.cpp
+++ b/src/formats/osml/OsmlStreamParser.cpp
@@ -441,7 +441,15 @@ Variant OsmlStreamParserImpl::parseIdentifier(size_t start, bool allowNSSep)
 		// Abort if this character is not a valid identifer character
 		if ((first && Utils::isIdentifierStartCharacter(c)) ||
 		    (!first && Utils::isIdentifierCharacter(c))) {
-			identifier.push_back(c);
+			if (Utils::isIdentifierEndCharacter(c) ||
+			    (reader.fetchPeek(c2) && Utils::isIdentifierCharacter(c2))) {
+				identifier.push_back(c);
+			} else {
+				// Break if a non-identifier-end character is reached and the
+				// next character is a non-identifer character
+				reader.resetPeek();
+				break;
+			}
 		} else if (c == ':' && hasCharSinceNSSep && reader.fetchPeek(c2) &&
 		           Utils::isIdentifierStartCharacter(c2)) {
 			identifier.push_back(c);
diff --git a/test/core/common/UtilsTest.cpp b/test/core/common/UtilsTest.cpp
index 2aaa430..f1a9af3 100644
--- a/test/core/common/UtilsTest.cpp
+++ b/test/core/common/UtilsTest.cpp
@@ -31,6 +31,7 @@ TEST(Utils, isIdentifier)
 	EXPECT_FALSE(Utils::isIdentifier("0t-_EST"));
 	EXPECT_FALSE(Utils::isIdentifier("_A"));
 	EXPECT_FALSE(Utils::isIdentifier("invalid key"));
+	EXPECT_FALSE(Utils::isIdentifier("A_"));
 	EXPECT_FALSE(Utils::isIdentifier(""));
 }
 
diff --git a/test/formats/osml/OsmlStreamParserTest.cpp b/test/formats/osml/OsmlStreamParserTest.cpp
index d47f529..d45a799 100644
--- a/test/formats/osml/OsmlStreamParserTest.cpp
+++ b/test/formats/osml/OsmlStreamParserTest.cpp
@@ -1435,5 +1435,35 @@ TEST(OsmlStreamParser, userDefinedTokens)
 	assertText(reader, " said.", tokens, WhitespaceMode::PRESERVE, 34, 40);
 	assertEnd(reader);
 }
+
+TEST(OsmlStreamParser, commandWithUnderscoreAndEnd)
+{
+	const char *testString = "\\sum_";
+	//                         01234
+	//                         0
+
+	CharReader charReader(testString);
+
+	OsmlStreamParser parser(charReader, logger);
+
+	assertCommandStart(parser, "sum", false, Variant::mapType{}, 0, 4);
+	assertData(parser, "_", 4, 5);
+	assertEnd(parser);
+}
+
+TEST(OsmlStreamParser, commandWithUnderscore)
+{
+	const char *testString = "\\sum_ a";
+	//                         0123456
+	//                         0
+
+	CharReader charReader(testString);
+
+	OsmlStreamParser parser(charReader, logger);
+
+	assertCommandStart(parser, "sum", false, Variant::mapType{}, 0, 4);
+	assertData(parser, "_ a", 4, 7);
+	assertEnd(parser);
+}
 }