added more unit test for the new BufferClass, fixed a few bugs (merely inefficiencies -- no major problems found so far)

3 files changed, 249 insertions, 8 deletions

diff --git a/src/core/utils/CharReader.cpp b/src/core/utils/CharReader.cpp
index 84f562d..33eab45 100644
--- a/src/core/utils/CharReader.cpp
+++ b/src/core/utils/CharReader.cpp
@@ -16,7 +16,10 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <iostream>
+
 #include <algorithm>
+#include <limits>
 
 #include "CharReader.hpp"
 
@@ -34,8 +37,8 @@ Buffer::Buffer(ReadCallback callback, void *userData)
       startOffset(0),
       firstDead(0)
 {
-	// Insert a first empty bucket and set the start buffer correctly
-	nextBucket();
+	// Load a first block of data from the stream
+	stream();
 	startBucket = buckets.begin();
 }
 
@@ -72,20 +75,39 @@ void Buffer::advance(BucketList::const_iterator &it) const
 	}
 }
 
-
 Buffer::Bucket &Buffer::nextBucket()
 {
+	constexpr size_t MAXVAL = std::numeric_limits<size_t>::max();
+
 	// Fetch the minimum bucket index
-	size_t minBucketIdx = 0;
+	size_t minBucketIdx = MAXVAL;
 	for (size_t i = 0; i < cursors.size(); i++) {
 		if (alive[i]) {
-			minBucketIdx = std::min(minBucketIdx, cursors[i].bucketIdx);
+			// Fetch references to the bucket and the cursor
+			const Cursor &cur = cursors[i];
+			const Bucket &bucket = *(cur.bucket);
+
+			// Increment the bucket index by one, if the cursor is at the end
+			// of the bucket (only valid if the LOOKBACK_SIZE is set to zero)
+			size_t bIdx = cur.bucketIdx;
+			if (LOOKBACK_SIZE == 0 && cur.bucketOffs == bucket.size()) {
+				bIdx++;
+			}
+
+			// Decrement the bucket index by one, if the previous bucket still
+			// needs to be reached and cannot be overridden
+			if (bIdx > 0 && cur.bucketOffs < LOOKBACK_SIZE) {
+				bIdx--;
+			}
+
+			// Set the bucket index to the minium
+			minBucketIdx = std::min(minBucketIdx, bIdx);
 		}
 	}
 
 	// If there is space between the current start bucket and the read
 	// cursor, the start bucket can be safely overridden.
-	if (minBucketIdx > 0) {
+	if (minBucketIdx > 0 && minBucketIdx != MAXVAL) {
 		// All cursor bucket indices will be decreased by one
 		for (size_t i = 0; i < cursors.size(); i++) {
 			cursors[i].bucketIdx--;
diff --git a/src/core/utils/CharReader.hpp b/src/core/utils/CharReader.hpp
index 4986d3e..23e88b7 100644
--- a/src/core/utils/CharReader.hpp
+++ b/src/core/utils/CharReader.hpp
@@ -63,9 +63,17 @@ public:
 
 private:
 	/**
-	 * Number of bytes to request from the input stream.
+	 * Number of bytes to request from the input stream. Set to 64 KiB because
+	 * this seems to be a nice value for I/O operations according to multiple
+	 * sources.
 	 */
-	static constexpr size_t REQUEST_SIZE = 16 * 1024;
+	static constexpr size_t REQUEST_SIZE = 64 * 1024;
+
+	/**
+	 * Number of bytes the buffer guarantees to be capable of looking back
+	 * for extracting the current context.
+	 */
+	static constexpr size_t LOOKBACK_SIZE = 128;
 
 	/**
 	 * Type used internally to represent one chunk of memory.
diff --git a/test/core/utils/CharReaderTest.cpp b/test/core/utils/CharReaderTest.cpp
index ef022a9..260b135 100644
--- a/test/core/utils/CharReaderTest.cpp
+++ b/test/core/utils/CharReaderTest.cpp
@@ -16,6 +16,7 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <sstream>
 #include <string>
 #include <iostream>
 
@@ -40,6 +41,9 @@ TEST(Buffer, simpleRead)
 	// We're not at the end of the stream
 	ASSERT_FALSE(buf.atEnd(cursor));
 
+	// The cursor must be at zero
+	ASSERT_EQ(0, buf.offset(cursor));
+
 	// Try to read the test string
 	std::string res;
 	while (buf.read(cursor, c)) {
@@ -49,10 +53,30 @@ TEST(Buffer, simpleRead)
 	// The cursor must be at the end
 	ASSERT_TRUE(buf.atEnd(cursor));
 
+	// The cursor must be one byond the last byte
+	ASSERT_EQ(testStr.size(), buf.offset(cursor));
+
 	// The two strings must equal
 	ASSERT_STREQ(testStr.c_str(), res.c_str());
 }
 
+TEST(Buffer, cursorManagement)
+{
+	Buffer buf{""};
+
+	Buffer::CursorId c1 = buf.createCursor();
+	Buffer::CursorId c2 = buf.createCursor();
+	Buffer::CursorId c3 = buf.createCursor();
+
+	ASSERT_EQ(0, c1);
+	ASSERT_EQ(1, c2);
+	ASSERT_EQ(2, c3);
+
+	buf.deleteCursor(c2);
+	Buffer::CursorId c4 = buf.createCursor();
+	ASSERT_EQ(1, c4);
+}
+
 TEST(Buffer, twoCursors)
 {
 	std::string testStr{"this is a test"};
@@ -93,6 +117,193 @@ TEST(Buffer, twoCursors)
 	ASSERT_EQ(testStr, res2);
 }
 
+TEST(Buffer, copyCursors)
+{
+	std::string testStr{"test1 test2 test3"};
+
+	// Create buffer with the test string
+	char c;
+	Buffer buf{testStr};
+
+	// Create two read cursors
+	Buffer::CursorId cur1 = buf.createCursor();
+	Buffer::CursorId cur2 = buf.createCursor();
+
+	ASSERT_FALSE(buf.atEnd(cur1));
+	ASSERT_FALSE(buf.atEnd(cur2));
+
+	// Read the first six characters with cursor one
+	std::string res1;
+	for (int i = 0; i < 6; i++) {
+		if (buf.read(cur1, c)) {
+			res1.append(&c, 1);
+		}
+	}
+	ASSERT_EQ("test1 ", res1);
+	ASSERT_FALSE(buf.atEnd(cur1));
+
+	// Copy cur1 to cur2, free cur1
+	buf.copyCursor(cur1, cur2);
+	buf.deleteCursor(cur1);
+
+	std::string res2;
+	for (int i = 0; i < 6; i++) {
+		if (buf.read(cur2, c)) {
+			res2.append(&c, 1);
+		}
+	}
+	ASSERT_EQ("test2 ", res2);
+	ASSERT_FALSE(buf.atEnd(cur2));
+
+	// Create a new cursor as copy of cur2
+	Buffer::CursorId cur3 = buf.createCursor(cur2);
+	std::string res3;
+	for (int i = 0; i < 6; i++) {
+		if (buf.read(cur3, c)) {
+			res3.append(&c, 1);
+		}
+	}
+	ASSERT_EQ("test3", res3);
+
+	ASSERT_TRUE(buf.atEnd(cur3));
+}
+
+// Generates some pseudo-random data
+// (inspired by "Numerical Recipes, Third Edition", Chapter 7.17)
+static std::vector<char> generateData(size_t len)
+{
+	const uint32_t B1 = 17;
+	const uint32_t B2 = 15;
+	const uint32_t B3 = 5;
+	uint32_t v = 0xF3A99148;
+	std::vector<char> res;
+	for (size_t i = 0; i < len; i++) {
+		v = v ^ (v >> B1);
+		v = v ^ (v << B2);
+		v = v ^ (v >> B3);
+		res.push_back(v & 0xFF);
+	}
+	return res;
+}
+
+struct VectorReadState {
+	size_t offs;
+	const std::vector<char> &data;
+
+	VectorReadState(const std::vector<char> &data) : offs(0), data(data) {}
+};
+
+static size_t readFromVector(char *buf, size_t size, void *userData)
+{
+	VectorReadState &state = *(static_cast<VectorReadState *>(userData));
+	size_t tar = std::min(state.offs + size, state.data.size());
+	for (size_t i = state.offs; i < tar; i++) {
+		*buf = state.data[i];
+		buf++;
+	}
+	size_t res = tar - state.offs;
+
+//	std::cout << "called readFromVector, read from " << state.offs << " to "
+//	          << tar << ", total " << res << " byte, requested " << size
+//	          << " byte" << std::endl;
+
+	state.offs = tar;
+	return res;
+}
+
+static constexpr size_t DATA_LENGTH = 256 * 1024 + 795;
+static const std::vector<char> DATA = generateData(DATA_LENGTH);
+
+TEST(Buffer, simpleStream)
+{
+	VectorReadState state(DATA);
+
+	Buffer buf{readFromVector, &state};
+	Buffer::CursorId cursor = buf.createCursor();
+
+	char c;
+	std::vector<char> res;
+	while (buf.read(cursor, c)) {
+		res.push_back(c);
+	}
+
+	// We must be at the end of the buffer and the cursor offset must be set
+	// correctly
+	ASSERT_TRUE(buf.atEnd(cursor));
+	ASSERT_EQ(DATA_LENGTH, buf.offset(cursor));
+
+	// The read data and the original data must be equal
+	ASSERT_EQ(DATA, res);
+}
+
+TEST(Buffer, streamTwoCursors)
+{
+	VectorReadState state(DATA);
+
+	Buffer buf{readFromVector, &state};
+	Buffer::CursorId cur1 = buf.createCursor();
+	Buffer::CursorId cur2 = buf.createCursor();
+
+	char c;
+
+	std::vector<char> res1;
+	while (buf.read(cur1, c)) {
+		res1.push_back(c);
+	}
+
+	ASSERT_TRUE(buf.atEnd(cur1));
+	ASSERT_FALSE(buf.atEnd(cur2));
+	ASSERT_EQ(DATA_LENGTH, buf.offset(cur1));
+	ASSERT_EQ(0, buf.offset(cur2));
+
+	std::vector<char> res2;
+	while (buf.read(cur2, c)) {
+		res2.push_back(c);
+	}
+
+	ASSERT_TRUE(buf.atEnd(cur1));
+	ASSERT_TRUE(buf.atEnd(cur2));
+	ASSERT_EQ(DATA_LENGTH, buf.offset(cur1));
+	ASSERT_EQ(DATA_LENGTH, buf.offset(cur2));
+
+	// The read data and the original data must be equal
+	ASSERT_EQ(DATA, res1);
+	ASSERT_EQ(DATA, res2);
+}
+
+TEST(Buffer, streamTwoCursorsInterleaved)
+{
+	VectorReadState state(DATA);
+
+	Buffer buf{readFromVector, &state};
+	Buffer::CursorId cur1 = buf.createCursor();
+	Buffer::CursorId cur2 = buf.createCursor();
+
+	char c;
+
+	std::vector<char> res1;
+	std::vector<char> res2;
+	while (!buf.atEnd(cur1) || !buf.atEnd(cur2)) {
+		for (int i = 0; i < 100; i++) {
+			if (buf.read(cur1, c)) {
+				res1.push_back(c);
+			}
+		}
+		for (int i = 0; i < 120; i++) {
+			if (buf.read(cur2, c)) {
+				res2.push_back(c);
+			}
+		}
+	}
+
+	ASSERT_EQ(DATA_LENGTH, buf.offset(cur1));
+	ASSERT_EQ(DATA_LENGTH, buf.offset(cur2));
+
+	// The read data and the original data must be equal
+	ASSERT_EQ(DATA, res1);
+	ASSERT_EQ(DATA, res2);
+}
+
 }
 }