From c272044c8ffddd9ef236067b4c2d2464b9a1e94c Mon Sep 17 00:00:00 2001 From: Andreas Stöckel Date: Mon, 8 Dec 2014 20:13:21 +0100 Subject: implemented new CharReader class (not tested yet) --- src/core/utils/CharReader.cpp | 180 +++++++++++++++++++++++ src/core/utils/CharReader.hpp | 283 ++++++++++++++++++++++++++++++++++++- test/core/utils/CharReaderTest.cpp | 5 - 3 files changed, 457 insertions(+), 11 deletions(-) diff --git a/src/core/utils/CharReader.cpp b/src/core/utils/CharReader.cpp index bf25a01..61bbd64 100644 --- a/src/core/utils/CharReader.cpp +++ b/src/core/utils/CharReader.cpp @@ -19,11 +19,31 @@ #include #include +#include + #include "CharReader.hpp" namespace ousia { namespace utils { +/* Helper functions */ + +/** + * istreamReadCallback is used internally by the Buffer calss to stream data + * from an input stream. + * + * @param buf is points a the target memory region. + * @param size is the requested number of bytes. + * @param userData is a pointer at some user defined data. + * @return the actual number of bytes read. If the result is smaller than + * the requested size, this tells the Buffer that the end of the input + * stream is reached. + */ +static size_t istreamReadCallback(char *buf, size_t size, void *userData) +{ + return (static_cast(userData))->read(buf, size).gcount(); +} + /* Class Buffer */ Buffer::Buffer(ReadCallback callback, void *userData) @@ -40,6 +60,8 @@ Buffer::Buffer(ReadCallback callback, void *userData) startBucket = buckets.begin(); } +Buffer::Buffer(std::istream &istream) : Buffer(istreamReadCallback, &istream) {} + Buffer::Buffer(const std::string &str) : callback(nullptr), userData(nullptr), @@ -331,6 +353,164 @@ bool Buffer::read(Buffer::CursorId cursor, char &c) advance(cur.bucket); } } + +/* CharReader::Cursor class */ + +void CharReader::Cursor::assign(std::shared_ptr buffer, + CharReader::Cursor &cursor) +{ + // Copy the cursor position + buffer->copyCursor(cursor.cursor, this->cursor); + + // Copy the state + line = cursor.line; + column = cursor.column; + state = cursor.state; + lastLinebreak = cursor.lastLinebreak; +} + +/* CharReader class */ + +CharReader::CharReader(std::shared_ptr buffer) + : buffer(buffer), + readCursor(buffer->createCursor()), + peekCursor(buffer->createCursor()) +{ +} + +CharReader::CharReader(const std::string &str, size_t line, size_t column) + : buffer(new Buffer{str}), + readCursor(buffer->createCursor(), line, column), + peekCursor(buffer->createCursor(), line, column) +{ +} + +CharReader::CharReader(std::istream &istream, size_t line, size_t column) + : buffer(new Buffer{istream}), + readCursor(buffer->createCursor(), line, column), + peekCursor(buffer->createCursor(), line, column) +{ +} + +CharReader::~CharReader() +{ + buffer->deleteCursor(readCursor.cursor); + buffer->deleteCursor(peekCursor.cursor); +} + +bool CharReader::substituteLinebreaks(Cursor &cursor, char &c) +{ + if (c == '\n' || c == '\r') { + switch (cursor.state) { + case LinebreakState::NONE: + // We got a first linebreak character -- output a '\n' + if (c == '\n') { + cursor.state = LinebreakState::HAS_LF; + } else { + cursor.state = LinebreakState::HAS_CR; + } + c = '\n'; + return true; + case LinebreakState::HAS_LF: + // If a LF is followed by a LF, output a new linefeed + if (c == '\n') { + cursor.state = LinebreakState::HAS_LF; + return true; + } + + // Otherwise, don't handle this character (part of "\n\r") + cursor.state = LinebreakState::NONE; + return false; + case LinebreakState::HAS_CR: + // If a CR is followed by a CR, output a new linefeed + if (c == '\r') { + cursor.state = LinebreakState::HAS_CR; + c = '\n'; + return true; + } + + // Otherwise, don't handle this character (part of "\r\n") + cursor.state = LinebreakState::NONE; + return false; + } + } + + // No linebreak character, reset the linebreak state + cursor.state = LinebreakState::NONE; + return true; +} + +bool CharReader::readAtCursor(Cursor &cursor, char &c) +{ + while (true) { + // Return false if we're at the end of the stream + if (!buffer->read(cursor.cursor, c)) { + return false; + } + + // Substitute linebreak characters with a single '\n' + if (substituteLinebreaks(cursor, c)) { + if (c == '\n') { + // A linebreak was reached, go to the next line + cursor.line++; + cursor.column = 1; + cursor.lastLinebreak = buffer->offset(cursor.cursor); + } else { + // Ignore UTF-8 continuation bytes + if (!((c & 0x80) && !(c & 0x40))) { + cursor.column++; + } + } + + return true; + } + } +} + +bool CharReader::peek(char &c) { return readAtCursor(peekCursor, c); } + +bool CharReader::read(char &c) { return readAtCursor(readCursor, c); } + +void CharReader::resetPeek() { peekCursor.assign(buffer, readCursor); } + +void CharReader::consumePeek() { readCursor.assign(buffer, peekCursor); } + +bool CharReader::consumeWhitespace() +{ + char c; + while (peek(c)) { + if (!Utils::isWhitespace(c)) { + resetPeek(); + return true; + } + consumePeek(); + } + return false; +} + +CharReaderFork CharReader::fork() +{ + return CharReaderFork(buffer, readCursor, peekCursor); +} + +/* Class CharReaderFork */ + +CharReaderFork::CharReaderFork(std::shared_ptr buffer, + CharReader::Cursor &parentReadCursor, + CharReader::Cursor &parentPeekCursor) + : CharReader(buffer), + parentReadCursor(parentReadCursor), + parentPeekCursor(parentPeekCursor) +{ + readCursor.assign(buffer, parentReadCursor); + peekCursor.assign(buffer, parentPeekCursor); +} + +void CharReaderFork::commit() +{ + parentReadCursor.assign(buffer, readCursor); + parentPeekCursor.assign(buffer, peekCursor); +} } } diff --git a/src/core/utils/CharReader.hpp b/src/core/utils/CharReader.hpp index 8d97d39..a48f5ad 100644 --- a/src/core/utils/CharReader.hpp +++ b/src/core/utils/CharReader.hpp @@ -27,7 +27,9 @@ #ifndef _OUSIA_CHAR_READER_HPP_ #define _OUSIA_CHAR_READER_HPP_ +#include #include +#include #include namespace ousia { @@ -45,9 +47,8 @@ public: * Callback function which is called whenever new data is requested from the * input stream. * - * @param buf is a pointer at the memory region to which the data should be - * writtern. - * @param size is the size of the + * @param buf is points a the target memory region. + * @param size is the requested number of bytes. * @param userData is a pointer at some user defined data given in the * constructor. * @return the actual number of bytes read. If the result is smaller than @@ -218,6 +219,14 @@ public: */ Buffer(ReadCallback callback, void *userData); + /** + * Initializes the Buffer with a reference to an std::istream from which + * data will be read. + * + * @param istream is the input stream from which the data should be read. + */ + Buffer(std::istream &istream); + /** * Initializes the Buffer with the contents of the given string, after * this operation the Buffer has a fixed size. @@ -266,14 +275,16 @@ public: /** * Moves a cursor by offs bytes. Note that moving backwards is theoretically - * limited by the LOOKBACK_SIZE of the Buffer, practically it will most likely - * be limited by the REQUEST_SIZE, so you can got at most 64 KiB backwards. + * limited by the LOOKBACK_SIZE of the Buffer, practically it will most + * likely be limited by the REQUEST_SIZE, so you can got at most 64 KiB + * backwards. * * @param cursor is the cursor that should be moved. * @param relativeOffs is a positive or negative integer number specifying * the number of bytes the cursor should be moved forward (positive numbers) * or backwards (negative numbers). - * @return the actual number of bytes the cursor was moved. + * @return the actual number of bytes the cursor was moved. This number is + * smaller than the relativeOffs given in the constructor if the */ ssize_t moveCursor(CursorId cursor, ssize_t relativeOffs); @@ -311,6 +322,266 @@ public: bool read(CursorId cursor, char &c); }; +// Forward declaration +class CharReaderFork; + +/** + * Used within parsers for convenient access to single characters in an input + * stream or buffer. It allows reading and peeking single characters from a + * buffer. Additionally it counts the current column/row (with correct handling + * for UTF-8) and contains an internal state machine that handles the detection + * of linebreaks and converts these to a single '\n'. + */ +class CharReader { +protected: + /** + * Enum to represent the current state of the internal state machine that + * replaces the linebreaks from multiple platforms to a single '\n'. + */ + enum class LinebreakState { NONE, HAS_LF, HAS_CR }; + + /** + * Internally used cursor structure for managing the read and the peek + * cursor. + */ + struct Cursor { + /** + * Corresponding cursor in the underlying buffer instance. + */ + const Buffer::CursorId cursor; + + /** + * Current line the cursor is in. + */ + size_t line; + + /** + * Current column the cursor is in. + */ + size_t column; + + /** + * State of the linebreak replacement statemachine. + */ + LinebreakState state; + + /** + * Contains the absolute offset in the input stream containing the + * position of the last linebreak. This is used for extracting the + * context (the line) in which an error occured. + */ + size_t lastLinebreak; + + /** + * Constructor of the Cursor class. + * + * @param cursor is the underlying cursor in the Buffer instance. + */ + Cursor(Buffer::CursorId cursor, size_t line = 1, size_t column = 1) + : cursor(cursor), + line(line), + column(column), + state(LinebreakState::NONE), + lastLinebreak(0) + { + } + + /** + * Assigns one cursor to another. + * + * @param buffer is the underlying buffer instance the internal cursor + * belongs to. + * @param cursor is the cursor from which the state should be copied. + */ + void assign(std::shared_ptr buffer, Cursor &cursor); + }; + +private: + /** + * Substitutes "\r", "\n\r", "\r\n" with a single "\n". + * + * @param cursor is the cursor from which the character should be read. + * @param c a reference to the character that should be written. + * @return true if another character needs to be read. + */ + bool substituteLinebreaks(Cursor &cursor, char &c); + + /** + * Reads a single character from the given cursor. + * + * @param cursor is the cursor from which the character should be read. + * @param c a reference to the character that should be written. + * @return true if a character was read, false if the end of the stream has + * been reached. + */ + bool readAtCursor(Cursor &cursor, char &c); + +protected: + /** + * Reference pointing at the underlying buffer. + */ + std::shared_ptr buffer; + + /** + * Cursor used for reading. + */ + Cursor readCursor; + + /** + * Cursor used for peeking. + */ + Cursor peekCursor; + + /** + * Protected constructor of the CharReader base class. Creates new read + * and peek cursors for the given buffer. + * + * @param buffer is a reference to the underlying Buffer class responsible + * for allowing to read from a single input stream from multiple locations. + */ + CharReader(std::shared_ptr buffer); + +public: + /** + * Creates a new CharReader instance from a string. + * + * @param str is a string containing the input data. + * @param line is the start line. + * @param column is the start column. + */ + CharReader(const std::string &str, size_t line = 1, size_t column = 1); + + /** + * Creates a new CharReader instance for an input stream. + * + * @param istream is the input stream from which incomming data should be + * read. + * @param line is the start line. + * @param column is the start column. + */ + CharReader(std::istream &istream, size_t line = 1, size_t column = 1); + + /** + * Deletes the used cursors from the underlying buffer instance. + */ + ~CharReader(); + + // No copy + CharReader(const Buffer &) = delete; + + // No assign + CharReader &operator=(const Buffer &) = delete; + + /** + * Peeks a single character. If called multiple times, returns the + * character after the previously peeked character. + * + * @param c is a reference to the character to which the result should be + * written. + * @return true if the character was successfully read, false if there are + * no more characters to be read in the buffer. + */ + bool peek(char &c); + + /** + * Reads a character from the input data. If "peek" was called + * beforehand resets the peek pointer. + * + * @param c is a reference to the character to which the result should be + * written. + * @return true if the character was successfully read, false if there are + * no more characters to be read in the buffer. + */ + bool read(char &c); + + /** + * Resets the peek pointer to the "read" pointer. + */ + void resetPeek(); + + /** + * Advances the read pointer to the peek pointer -- so if the "peek" + * function was called, "read" will now return the character after + * the last peeked character. + */ + void consumePeek(); + + /** + * Moves the read cursor to the next non-whitespace character. Returns + * false, if the end of the stream was reached. + * + * @return false if the end of the stream was reached, false othrwise. + */ + bool consumeWhitespace(); + + /** + * Creates a new CharReader located at the same position as this CharReader + * instance, yet the new CharReader can be used independently of this + * CharReader. Use the "commit" function of the returned CharReader to + * copy the state of the forked CharReaderFork to this CharReader. + * + * @return a CharReaderFork instance positioned at the same location as this + * CharReader instance. + */ + CharReaderFork fork(); + + /** + * Returns true if there are no more characters as the stream was + * closed. + * + * @return true if there is no more data. + */ + bool atEnd() const { return buffer->atEnd(readCursor.cursor); } + + /** + * Returns the current line (starting with one). + * + * @return the current line number. + */ + int getLine() const { return readCursor.line; } + + /** + * Returns the current column (starting with one). + * + * @return the current column number. + */ + int getColumn() const { return readCursor.column; } +}; + +/** + * A CharReaderFork is returned whenever the "fork" function of the CharReader + * class is used. Its "commit" function can be used to move the underlying + * CharReader instance to the location of the CharReaderFork instance. Otherwise + * the read location of the underlying CharReader is left unchanged. + */ +class CharReaderFork : public CharReader { +private: + friend CharReader; + + /** + * The reader cursor of the underlying CharReader instance. + */ + CharReader::Cursor &parentReadCursor; + + /** + * The peek cursor of the underlying CharReader instance. + */ + CharReader::Cursor &parentPeekCursor; + + /** + * Constructor of the CharReaderFork class. + */ + CharReaderFork(std::shared_ptr buffer, + CharReader::Cursor &parentReadCursor, + CharReader::Cursor &parentPeekCursor); + +public: + /** + * Moves the read and peek cursor of the parent CharReader to the location + * of the read and peek cursor in the fork. + */ + void commit(); +}; } } diff --git a/test/core/utils/CharReaderTest.cpp b/test/core/utils/CharReaderTest.cpp index 1518928..c9a1f64 100644 --- a/test/core/utils/CharReaderTest.cpp +++ b/test/core/utils/CharReaderTest.cpp @@ -259,11 +259,6 @@ static size_t readFromVector(char *buf, size_t size, void *userData) buf++; } size_t res = tar - state.offs; - -// std::cout << "called readFromVector, read from " << state.offs << " to " -// << tar << ", total " << res << " byte, requested " << size -// << " byte" << std::endl; - state.offs = tar; return res; } -- cgit v1.2.3