Implemented storing SourceLocation of parsed Variants

3 files changed, 205 insertions, 27 deletions

diff --git a/src/core/common/VariantReader.cpp b/src/core/common/VariantReader.cpp
index 7403b4a..ef71740 100644
--- a/src/core/common/VariantReader.cpp
+++ b/src/core/common/VariantReader.cpp
@@ -674,6 +674,12 @@ std::pair<bool, Variant> VariantReader::parseGeneric(
 	char c;
 	bool hadError = false;
 
+	// Skip all peeked characters
+	reader.consumePeek();
+
+	// Read the start offset
+	const SourceOffset start = reader.getOffset();
+
 	// Parse generic tokens until the end of the stream or the delimiter is
 	// reached
 	while (reader.peek(c) && !delims.count(c)) {
@@ -694,7 +700,9 @@ std::pair<bool, Variant> VariantReader::parseGeneric(
 	if (arr.size() == 1) {
 		return std::make_pair(!hadError, arr[0]);
 	} else {
-		return std::make_pair(!hadError, Variant{arr});
+		Variant res{arr};
+		res.setLocation({reader.getSourceId(), start, reader.getOffset()});
+		return std::make_pair(!hadError, res);
 	}
 }
 
@@ -712,10 +720,15 @@ std::pair<bool, Variant> VariantReader::parseGenericToken(
 	}
 	reader.resetPeek();
 
+	// Fetch the start offset
+	const SourceOffset start = reader.getOffset();
+
 	// Parse a string if a quote is reached
 	if (c == '"' || c == '\'') {
 		auto res = parseString(reader, logger);
-		return std::make_pair(res.first, res.second.c_str());
+		Variant v = Variant::fromString(res.second);
+		v.setLocation({reader.getSourceId(), start, reader.getOffset()});
+		return std::make_pair(res.first, v);
 	}
 
 	// Try to parse everything that looks like a number as number
@@ -727,12 +740,15 @@ std::pair<bool, Variant> VariantReader::parseGenericToken(
 		if (n.parse(readerFork, loggerFork, delims)) {
 			readerFork.commit();
 			loggerFork.commit();
+
+			Variant v;
 			if (n.isInt()) {
-				return std::make_pair(
-				    true, Variant{static_cast<Variant::intType>(n.intValue())});
+				v = Variant{static_cast<Variant::intType>(n.intValue())};
 			} else {
-				return std::make_pair(true, n.doubleValue());
+				v = Variant{n.doubleValue()};
 			}
+			v.setLocation({reader.getSourceId(), start, reader.getOffset()});
+			return std::make_pair(true, v);
 		}
 		reader.resetPeek();
 	}
@@ -745,14 +761,19 @@ std::pair<bool, Variant> VariantReader::parseGenericToken(
 		if (res.first) {
 			readerFork.commit();
 			loggerFork.commit();
-			return std::make_pair(true, Variant{res.second});
+			Variant v{res.second};
+			v.setLocation({reader.getSourceId(), start, reader.getOffset()});
+			return std::make_pair(true, v);
 		}
 		reader.resetPeek();
 	}
 
 	// Try to parse an object
 	if (c == '[') {
-		return parseComplex(reader, logger, 0, ComplexMode::BOTH);
+		auto res = parseComplex(reader, logger, 0, ComplexMode::BOTH);
+		res.second.setLocation(
+		    {reader.getSourceId(), start, reader.getOffset()});
+		return res;
 	}
 
 	// Otherwise parse a single token
@@ -764,41 +785,55 @@ std::pair<bool, Variant> VariantReader::parseGenericToken(
 	}
 
 	// Handling for special primitive values
+	bool isSpecial = false;
+	Variant v;
 	if (res.first) {
 		if (res.second == "true") {
-			return std::make_pair(true, Variant{true});
-		}
-		if (res.second == "false") {
-			return std::make_pair(true, Variant{false});
-		}
-		if (res.second == "null") {
-			return std::make_pair(true, Variant{nullptr});
+			v = Variant{true};
+			isSpecial = true;
+		} else if (res.second == "false") {
+			v = Variant{false};
+			isSpecial = true;
+		} else if (res.second == "null") {
+			v = Variant{nullptr};
+			isSpecial = true;
 		}
 	}
 
 	// Check whether the parsed string is a valid identifier -- if yes, flag it
 	// as "magic" string
-	if (Utils::isIdentifier(res.second)) {
-		Variant v;
-		v.setMagic(res.second.c_str());
-		return std::make_pair(res.first, v);
-	} else {
-		return std::make_pair(res.first, Variant::fromString(res.second));
+	if (!isSpecial) {
+		if (Utils::isIdentifier(res.second)) {
+			v.setMagic(res.second.c_str());
+		} else {
+			v = Variant::fromString(res.second);
+		}
 	}
+	v.setLocation({reader.getSourceId(), start, reader.getOffset()});
+	return std::make_pair(res.first, v);
 }
 
 std::pair<bool, Variant> VariantReader::parseGenericString(
-    const std::string &str, Logger &logger)
+    const std::string &str, Logger &logger, SourceId sourceId, size_t offs)
 {
-	CharReader reader{str};
+	CharReader reader{str, sourceId, offs};
 	LoggerFork loggerFork = logger.fork();
+
+	// Try to parse a single token
 	std::pair<bool, Variant> res =
 	    parseGenericToken(reader, loggerFork, std::unordered_set<char>{}, true);
+
+	// If the string was actually consisted of a single token, return that token
 	if (reader.atEnd()) {
 		loggerFork.commit();
 		return res;
 	}
-	return std::make_pair(true, Variant::fromString(str));
+
+	// Otherwise return the given string as a string, set the location of the
+	// string correctly
+	Variant v = Variant::fromString(str);
+	v.setLocation({sourceId, offs, offs + str.size()});
+	return std::make_pair(true, v);
 }
 }
 
diff --git a/src/core/common/VariantReader.hpp b/src/core/common/VariantReader.hpp
index d939415..7f00251 100644
--- a/src/core/common/VariantReader.hpp
+++ b/src/core/common/VariantReader.hpp
@@ -277,6 +277,10 @@ public:
 	 * @param str is the string from which the value should be read.
 	 * @param logger is the logger instance to which errors or warnings will be
 	 * written.
+	 * @param sourceId is an optional descriptor of the source file from which
+	 * the element is being read.
+	 * @param offs is the by offset in the source file at which the string
+	 * starts.
 	 * @return a pair indicating whether the operation was successful and the
 	 * extracted variant value. Note that the variant value most times contains
 	 * some meaningful data that can be worked with even if the operation was
@@ -285,8 +289,9 @@ public:
 	 * variant.) Information on why the operation has failed is passed to the
 	 * logger.
 	 */
-	static std::pair<bool, Variant> parseGenericString(const std::string &str,
-	                                                   Logger &logger);
+	static std::pair<bool, Variant> parseGenericString(
+	    const std::string &str, Logger &logger,
+	    SourceId sourceId = InvalidSourceId, size_t offs = 0);
 };
 }
 
diff --git a/test/core/common/VariantReaderTest.cpp b/test/core/common/VariantReaderTest.cpp
index 7e49352..f6a699b 100644
--- a/test/core/common/VariantReaderTest.cpp
+++ b/test/core/common/VariantReaderTest.cpp
@@ -728,97 +728,146 @@ TEST(VariantReader, parseGenericToken)
 	// Simple case, unescaped string
 	{
 		CharReader reader("hello world");
+		//                 01234567890
+		//                 0         1
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isString());
 		ASSERT_FALSE(res.second.isMagic());
 		ASSERT_EQ("hello world", res.second.asString());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
 	}
 
 	// Simple case, double quoted string
 	{
 		CharReader reader(" \"hello world\"    ");
+		//                 0 123456789012 34567
+		//                 0          1
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isString());
 		ASSERT_FALSE(res.second.isMagic());
 		ASSERT_EQ("hello world", res.second.asString());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(14U, loc.getEnd());
 	}
 
 	// Simple case, single quoted string
 	{
 		CharReader reader(" 'hello world'    ");
+		//                 012345678901234567
+		//                 0         1
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isString());
 		ASSERT_FALSE(res.second.isMagic());
 		ASSERT_EQ("hello world", res.second.asString());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(14U, loc.getEnd());
 	}
 
 	// String with whitespaces at the beginning.
 	{
-		CharReader reader("   \' test\'");
+		CharReader reader("   ' test'");
+		//                 0123456789
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_EQ(" test", res.second);
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(3U, loc.getStart());
+		ASSERT_EQ(10U, loc.getEnd());
 	}
 
 	// Integer
 	{
 		CharReader reader("1234");
+		//                 0123
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isInt());
 		ASSERT_EQ(1234, res.second.asInt());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(4U, loc.getEnd());
 	}
 
 	// Double
 	{
 		CharReader reader("1234.5");
+		//                 012345
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isDouble());
 		ASSERT_EQ(1234.5, res.second.asDouble());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(6U, loc.getEnd());
 	}
 
 	// Boolean (true)
 	{
 		CharReader reader("true");
+		//                 0123
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isBool());
 		ASSERT_TRUE(res.second.asBool());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(4U, loc.getEnd());
 	}
 
 	// Boolean (false)
 	{
 		CharReader reader("false");
+		//                 01234
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isBool());
 		ASSERT_FALSE(res.second.asBool());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
 	}
 
 	// Nullptr
 	{
 		CharReader reader("null");
+		//                 0123
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, true);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isNull());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(4U, loc.getEnd());
 	}
 
 	// Simple case, unescaped string
 	{
 		CharReader reader("hello world");
-
+		//                 01234567890
+		//                 0         1
 		{
 			auto res =
 			    VariantReader::parseGenericToken(reader, logger, {';'}, false);
@@ -826,6 +875,10 @@ TEST(VariantReader, parseGenericToken)
 			ASSERT_TRUE(res.second.isString());
 			ASSERT_TRUE(res.second.isMagic());
 			ASSERT_EQ("hello", res.second.asString());
+
+			SourceLocation loc = res.second.getLocation();
+			ASSERT_EQ(0U, loc.getStart());
+			ASSERT_EQ(5U, loc.getEnd());
 		}
 
 		{
@@ -835,29 +888,45 @@ TEST(VariantReader, parseGenericToken)
 			ASSERT_TRUE(res.second.isString());
 			ASSERT_TRUE(res.second.isMagic());
 			ASSERT_EQ("world", res.second.asString());
+
+			SourceLocation loc = res.second.getLocation();
+			ASSERT_EQ(6U, loc.getStart());
+			ASSERT_EQ(11U, loc.getEnd());
 		}
 	}
 
 	// Simple case, double quoted string
 	{
 		CharReader reader(" \"hello world\"    ");
+		//                 0 123456789012 34567
+		//                 0          1
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, false);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isString());
 		ASSERT_FALSE(res.second.isMagic());
 		ASSERT_EQ("hello world", res.second.asString());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(14U, loc.getEnd());
 	}
 
 	// Simple case, single quoted string
 	{
 		CharReader reader(" 'hello world'    ");
+		//                 012345678901234567
+		//                 0         1
 		auto res =
 		    VariantReader::parseGenericToken(reader, logger, {';'}, false);
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isString());
 		ASSERT_FALSE(res.second.isMagic());
 		ASSERT_EQ("hello world", res.second.asString());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(1U, loc.getStart());
+		ASSERT_EQ(14U, loc.getEnd());
 	}
 }
 
@@ -866,23 +935,35 @@ TEST(VariantReader, parseGeneric)
 	// Simple case, int.
 	{
 		CharReader reader("0");
+		//                 0
 		auto res = VariantReader::parseGeneric(reader, logger, {';'});
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isInt());
 		ASSERT_EQ(0, res.second.asInt());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(1U, loc.getEnd());
 	}
 	// Simple case, unescaped string
 	{
 		CharReader reader("hello");
+		//                 01234
 		auto res = VariantReader::parseGeneric(reader, logger, {';'});
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isMagic());
 		ASSERT_EQ("hello", res.second.asMagic());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
 	}
 
 	// Simple case, unescaped string with multiple array entries
 	{
 		CharReader reader("hello world");
+		//                 01234567890
+		//                 0         1
 		auto res = VariantReader::parseGeneric(reader, logger, {';'});
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isArray());
@@ -893,11 +974,17 @@ TEST(VariantReader, parseGeneric)
 		ASSERT_TRUE(arr[1].isMagic());
 		ASSERT_EQ("hello", arr[0].asMagic());
 		ASSERT_EQ("world", arr[1].asMagic());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(11U, loc.getEnd());
 	}
 
 	// Delimiter test
 	{
 		CharReader reader("hello; world");
+		//                 012345678901
+		//                 0         1
 		auto res = VariantReader::parseGeneric(reader, logger, {';'});
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isMagic());
@@ -906,11 +993,17 @@ TEST(VariantReader, parseGeneric)
 		char c;
 		ASSERT_TRUE(reader.peek(c));
 		ASSERT_EQ(';', c);
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(5U, loc.getEnd());
 	}
 
 	// More complex CSS-like case
 	{
 		CharReader reader("1px solid blue");
+		//                 01234567890123
+		//                 0         1
 		auto res = VariantReader::parseGeneric(reader, logger, {';'});
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isArray());
@@ -923,6 +1016,30 @@ TEST(VariantReader, parseGeneric)
 		ASSERT_EQ("1px", arr[0].asString());
 		ASSERT_EQ("solid", arr[1].asMagic());
 		ASSERT_EQ("blue", arr[2].asMagic());
+
+		{
+			SourceLocation loc = res.second.getLocation();
+			ASSERT_EQ(0U, loc.getStart());
+			ASSERT_EQ(14U, loc.getEnd());
+		}
+
+		{
+			SourceLocation loc = arr[0].getLocation();
+			ASSERT_EQ(0U, loc.getStart());
+			ASSERT_EQ(3U, loc.getEnd());
+		}
+
+		{
+			SourceLocation loc = arr[1].getLocation();
+			ASSERT_EQ(4U, loc.getStart());
+			ASSERT_EQ(9U, loc.getEnd());
+		}
+
+		{
+			SourceLocation loc = arr[2].getLocation();
+			ASSERT_EQ(10U, loc.getStart());
+			ASSERT_EQ(14U, loc.getEnd());
+		}
 	}
 }
 
@@ -931,34 +1048,55 @@ TEST(VariantReader, parseGenericString)
 	// Simple case, unescaped string
 	{
 		auto res = VariantReader::parseGenericString("foo", logger);
+		//                                            012
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isMagic());
 		ASSERT_EQ("foo", res.second.asMagic());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(3U, loc.getEnd());
 	}
 
 	// Simple case, unescaped string with space
 	{
 		auto res = VariantReader::parseGenericString("foo bar", logger);
+		//                                            0123456
 		ASSERT_TRUE(res.first);
 		ASSERT_FALSE(res.second.isMagic());
 		ASSERT_TRUE(res.second.isString());
 		ASSERT_EQ("foo bar", res.second.asString());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(7U, loc.getEnd());
 	}
 
 	// Parse double
 	{
 		auto res = VariantReader::parseGenericString("12.3", logger);
+		//                                            0123
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isDouble());
 		ASSERT_EQ(12.3, res.second.asDouble());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(4U, loc.getEnd());
 	}
 
 	// Parse string
 	{
 		auto res = VariantReader::parseGenericString("6 times 7 is 42", logger);
+		//                                            012345678901234
+		//                                            0         1
 		ASSERT_TRUE(res.first);
 		ASSERT_TRUE(res.second.isString());
 		ASSERT_EQ("6 times 7 is 42", res.second.asString());
+
+		SourceLocation loc = res.second.getLocation();
+		ASSERT_EQ(0U, loc.getStart());
+		ASSERT_EQ(15U, loc.getEnd());
 	}
 }