added initial buffer and input stream mode to BufferedCharReader -- needs some further refactoring still (see TODO)

1 files changed, 88 insertions, 52 deletions

diff --git a/src/core/BufferedCharReader.hpp b/src/core/BufferedCharReader.hpp
index 0d72347..ec76b03 100644
--- a/src/core/BufferedCharReader.hpp
+++ b/src/core/BufferedCharReader.hpp
@@ -16,30 +16,39 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+/**
+ * @file BufferedCharReader.hpp
+ *
+ * Contains the BufferedCharReader class which is used for reading/peeking
+ * single characters from an input stream or string.
+ *
+ * @author Andreas Stöckel (astoecke@techfak.uni-bielefeld.de)
+ */
+
 #ifndef _OUSIA_BUFFERED_CHAR_READER_H_
 #define _OUSIA_BUFFERED_CHAR_READER_H_
 
 #include <deque>
 #include <string>
+#include <istream>
 #include <cstdint>
 
 namespace ousia {
 
+// TODO: Better split this class into multiple classes with base class
+// BufferedCharReader where each sub class represents one method of supplying
+// the input data (feeding, initial string, input stream).
+
 /**
  * The BufferedCharReader class is used for storing incomming data that
  * is fed into the pipeline as well as reading/peeking single characters
  * from that buffer. Additionally it counts the current column/row
  * (with correct handling for UTF-8) and contains an internal state
- * machine that handles the detection of linebreaks.
- *
- * Additionally the BufferedCharReader performs the following tasks:
- * 1. Convert the incomming character encoding to UTF-8 (TODO: implement)
- * 2. Convert arbitrary linebreaks to a single "\n"
+ * machine that handles the detection of linebreaks and converts these to a
+ * single '\n'.
  */
 class BufferedCharReader {
-
 private:
-
 	/**
 	 * The ReadCursor structure is responsible for representing the read
 	 * position within the text an all state machine states belonging to the
@@ -48,12 +57,6 @@ private:
 	 */
 	struct ReadCursor {
 		/**
-		 * Specifies whether this is a destructive cursor (bytes are discarded
-		 * once they were read from the buffer).
-		 */
-		const bool destructive;
-
-		/**
 		 * The line the cursor currently points to.
 		 */
 		unsigned int line;
@@ -75,6 +78,12 @@ private:
 		unsigned int bufferPos;
 
 		/**
+		 * Specifies whether this is a destructive cursor (bytes are discarded
+		 * once they were read from the buffer).
+		 */
+		const bool destructive;
+
+		/**
 		 * State variable used in the internal state machine of the
 		 * line feed detection.
 		 */
@@ -83,45 +92,50 @@ private:
 		/**
 		 * Constructor of the ReadCursor structure.
 		 *
+		 * @param line is the start line.
+		 * @param column is the start column.
 		 * @param destructive specifies whether the ReadCursor is destructive
 		 * (consumes all read characters, as used in the "read cursor") or
 		 * non-destructive (as used in the "peek cursor").
 		 */
-		ReadCursor(const bool destructive);
+		ReadCursor(unsigned int line, unsigned int column, bool destructive);
 
 		/**
 		 * Copys the data from another ReadCursor without overriding the
 		 * "destructive" flag.
+		 *
+		 * @param cursor is the cursor that should be copied.
 		 */
 		void assign(const ReadCursor &cursor);
-
-		/**
-		 * Resets the cursor without changing the "destructive" flag.
-		 */
-		void reset();
 	};
 
 	/**
-	 * Queue containing the data that has been fed into the char reader.
+	 * Pointer at an (optional) input stream used for reading a chunk of data
+	 * whenever the input buffer depletes.
 	 */
-	std::deque<std::string> buffer;
+	std::istream *inputStream;
 
 	/**
-	 * The read and the peek cursor. 
+	 * The read and the peek cursor.
 	 */
 	ReadCursor readCursor, peekCursor;
 
 	/**
-	 * Determines whether the reader has been closed.
+	 * Set to true if there is no more input data.
 	 */
-	bool closed;
+	bool depleted;
+
+	/**
+	 * Queue containing the data that has been fed into the char reader.
+	 */
+	std::deque<std::string> buffer;
 
 	/**
 	 * Substitute any combination of linebreaks in the incomming code with "\n".
 	 * Returns true if the current character is meant as output, false
 	 * otherwise.
 	 */
-	bool substituteLinebreaks(ReadCursor *cursor, char *c);
+	bool substituteLinebreaks(ReadCursor &cursor, char *c);
 
 	/**
 	 * Reads a character from the input buffer and advances the given read
@@ -137,7 +151,7 @@ private:
 	 * @param returns true if there was enough data in the buffer, false
 	 * otherwise.
 	 */
-	bool readCharacterAtCursor(ReadCursor *cursor, char *c);
+	bool readCharacterAtCursor(ReadCursor &cursor, char *c);
 
 	/**
 	 * Function that is called for each read character -- updates the row and
@@ -148,31 +162,35 @@ private:
 public:
 
 	/**
-	 * Constructor of the buffered char reader class.
+	 * Constructor of the buffered char reader class with empty buffer as input.
+	 * This operates the BufferedCharReader in a mode where new data has to be
+	 * fed using the "feed" function and explicitly closed using the "close"
+	 * function.
+	 *
+	 * @param line is the start line.
+	 * @param column is the start column.
 	 */
-	BufferedCharReader();
+	BufferedCharReader(int line = 1, int column = 1);
 
-	/**
-	 * Resets the reader to its initial state.
-	 */
-	void reset();
 
 	/**
-	 * Feeds new data into the internal buffer of the BufferedCharReader
-	 * class.
+	 * Constructor of the buffered char reader class with a string as input.
 	 *
-	 * @param data is a string containing the data that should be
-	 * appended to the internal buffer.
-	 * @return true if the operation was successful, false otherwise (e.g.
-	 * because the reader is closed).
+	 * @param str is a string containing the input data.
+	 * @param line is the start line.
+	 * @param column is the start column.
 	 */
-	bool feed(const std::string &data);
+	BufferedCharReader(const std::string &str, int line = 1, int column = 1);
 
 	/**
-	 * Marks the end of the input, allowing successors in the pipeline
-	 * to react properly (e.g. creating the end of stream token).
+	 * Constructor of the buffered char reader class with a string as input.
+	 *
+	 * @param inputStream is the input stream from which incomming data should
+	 * be read.
+	 * @param line is the start line.
+	 * @param column is the start column.
 	 */
-	void close();
+	BufferedCharReader(std::istream &inputStream, int line = 1, int column = 1);
 
 	/**
 	 * Peeks a single character. If called multiple times, returns the
@@ -209,29 +227,47 @@ public:
 	void resetPeek();
 
 	/**
+	 * Feeds new data into the internal buffer of the BufferedCharReader
+	 * class. Only applicable if the buffered char reader was constructed
+	 * without an input stream or string.
+	 *
+	 * @param data is a string containing the data that should be
+	 * appended to the internal buffer.
+	 */
+	void feed(const std::string &data);
+
+	/**
+	 * Tells the buffered char reader that no more data will be fed.
+	 * Only applicable if the buffered char reader was constructed without an
+	 * input stream or string.
+	 *
+	 * @param data is a string containing the data that should be
+	 * appended to the internal buffer.
+	 */
+	void close();
+
+	/**
 	 * Returns true if there are no more characters as the stream was
 	 * closed.
+	 *
+	 * @return true if there is no more data.
 	 */
 	bool atEnd();
 
 	/**
 	 * Returns the current line (starting with one).
+	 *
+	 * @return the current line number.
 	 */
-	inline int getLine()
-	{
-		return readCursor.line;
-	}
+	inline int getLine() { return readCursor.line; }
 
 	/**
 	 * Returns the current column (starting with one).
+	 *
+	 * @return the current column number.
 	 */
-	inline int getColumn()
-	{
-		return readCursor.column;
-	}
-
+	inline int getColumn() { return readCursor.column; }
 };
-
 }
 
 #endif /* _OUSIA_BUFFERED_CHAR_READER_H_ */