diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 4c655c2..a0cc019 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -46,10 +46,13 @@ jobs: - name: Install test coverage tools run: | apt update - apt install -y gcovr lcov + apt install -y gcovr + + - name: Install lcov2.0 + run: script/ci_install_lcov.sh - name: Configure - run: cmake -S test -B build -D CMAKE_BUILD_TYPE=Debug -D CMAKE_CXX_FLAGS="-Wall -fprofile-arcs -ftest-coverage --coverage" + run: cmake -S test -B build -D CMAKE_BUILD_TYPE=Debug -D CMAKE_CXX_FLAGS="-Wall -fprofile-arcs -ftest-coverage --coverage -fno-elide-constructors -fno-default-inline" - name: Build run: cmake --build build -j ${{steps.cores.outputs.count}} @@ -60,7 +63,7 @@ jobs: - name: Generate coverage report run: | - lcov -d . -c -o out.info --rc lcov_branch_coverage=1 --no-external + lcov -d . -c -o out.info --rc branch_coverage=1 --no-external --filter branch --filter line --ignore-errors mismatch lcov -e out.info '*include/ss*hpp' -o filtered.info - name: Invoke coveralls diff --git a/.gitignore b/.gitignore index 38a1e99..eb880af 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,3 @@ build/ hbuild/ subprojects/* !subprojects/*.wrap -ssp.cpp -ssp.bin diff --git a/README.md b/README.md index 148792c..6f00e9c 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,13 @@ [![windows-msys2-clang](https://github.com/red0124/ssp/workflows/win-msys2-clang-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msys2-clang.yml) [![windows-msvc](https://github.com/red0124/ssp/workflows/win-msvc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msvc.yml) -A header only "csv" parser which is fast and versatile with modern C++ api. Requires compiler with C++17 support. [Can also be used to convert strings to specific types.](#the-converter) +A header only CSV parser which is fast and versatile with modern C++ API. Requires compiler with C++17 support. [Can also be used to efficiently convert strings to specific types.](#the-converter) Conversion for floating point values invoked using [fast-float](https://github.com/fastfloat/fast_float) . \ Function traits taken from *qt-creator* . # Example -Lets say we have a csv file containing students in a given format \ and we want to parse and print all the valid values: +Lets say we have a CSV file containing students in a given format (Id,Age,Grade) and we want to parse and print all the valid values: ```shell $ cat students.csv @@ -58,6 +58,7 @@ Bill (Heath) Gates 65 3.3 * Can work without exceptions * [Works with headers](#headers) * [Works with quotes, escapes and spacings](#setup) + * [Works with CSV data stored in buffers](#buffer-mode) * [Works with values containing new lines](#multiline) * [Columns and rows can be ignored](#special-types) * [Works with any type of delimiter](#delimiter) @@ -158,7 +159,7 @@ while (!p.eof()) { The alternate example with exceptions disabled will be used to show some of the features of the library. The **`get_next`** method returns a tuple of objects specified inside the template type list. -If a conversion could not be applied, the method would return a tuple of default constructed objects, and the **`valid`** method would return **`false`**, for example if the third (grade) column in our csv could not be converted to a float the conversion would fail. +If a conversion could not be applied, the method would return a tuple of default constructed objects, and the **`valid`** method would return **`false`**, for example if the third (grade) column in our CSV could not be converted to a float the conversion would fail. If **`get_next`** is called with a **`tuple`** as template parameter it would behave identically to passing the same tuple parameters to **`get_next`**: ```cpp @@ -202,14 +203,27 @@ struct student { auto tied() { return std::tie(id, age, grade); } }; ``` -The method can be used to compare the object, serialize it, deserialize it, etc. Now **`get_next`** can accept such a struct and deduce the types to which to convert the csv. +The method can be used to compare the object, serialize it, deserialize it, etc. Now **`get_next`** can accept such a struct and deduce the types to which to convert the CSV. ```cpp // returns student auto s = p.get_next(); ``` This works with the iteration loop too. -*Note, the order in which the members of the tied method are returned must match the order of the elements in the csv*. +*Note, the order in which the members of the tied method are returned must match the order of the elements in the CSV*. +## Buffer mode +The parser also works with buffers containing CSV data instead of files. To parse buffer data with the parser simply create the parser by giving it the buffer, as **`const char*`**, and its size. The initial example using a buffer instead of a file would look similar to this: +```cpp +std::string buffer = "James Bailey,65,2.5\nBrian S. Wolfe,40,1.9\n"; + +ss::parser p{buffer.c_str(), buffer.size()}; + +for (const auto& [id, age, grade] : p.iterate()) { + std::cout << id << ' ' << age << ' ' << grade << std::endl; +} + +return 0; +``` ## Setup By default, many of the features supported by the parser are disabled. They can be enabled within the template parameters of the parser. For example, to enable quoting and escaping the parser would look like: ```cpp @@ -241,7 +255,7 @@ Empty lines can be ignored by defining **`ss::ignore_empty`** within the setup p ```cpp ss::parser p{file_name}; ``` -If this setup option is not set then reading an empty line will result in an error (unless only one column is present within the csv). +If this setup option is not set then reading an empty line will result in an error (unless only one column is present within the CSV). ### Quoting Quoting can be enabled by defining **`ss::quote`** within the setup parameters. A single character can be defined as the quoting character, for example to use **`"`** as a quoting character: @@ -290,7 +304,7 @@ Escaping and quoting can be used to leave the space if needed. ``` ### Multiline -Multiline can be enabled by defining **`ss::multilne`** within the setup parameters. It enables the possibility to have the new line characters within rows. The new line character needs to be either escaped or within quotes so either **`ss::escape`** or **`ss::quote`** need to be enabled. There is a specific problem when using multiline, for example, if a row had an unterminated quote, the parser would assume it to be a new line within the row, so until another quote is found, it will treat it as one line which is fine usually, but it can cause the whole csv file to be treated as a single line by mistake. To prevent this **`ss::multiline_restricted`** can be used which accepts an unsigned number representing the maximum number of lines which can be allowed as a single multiline. Examples: +Multiline can be enabled by defining **`ss::multilne`** within the setup parameters. It enables the possibility to have the new line characters within rows. The new line character needs to be either escaped or within quotes so either **`ss::escape`** or **`ss::quote`** need to be enabled. There is a specific problem when using multiline, for example, if a row had an unterminated quote, the parser would assume it to be a new line within the row, so until another quote is found, it will treat it as one line which is fine usually, but it can cause the whole CSV file to be treated as a single line by mistake. To prevent this **`ss::multiline_restricted`** can be used which accepts an unsigned number representing the maximum number of lines which can be allowed as a single multiline. Examples: ```cpp ss::parser, ss::escape<'\\'>> p{file_name}; @@ -341,7 +355,7 @@ Gates 65 3.3' ``` ## Special types -Passing **`void`** makes the parser ignore a column. In the initial example **`void`** could be given as the second template parameter to ignore the second (age) column in the csv, a tuple of only 2 parameters would be retuned: +Passing **`void`** makes the parser ignore a column. In the initial example **`void`** could be given as the second template parameter to ignore the second (age) column in the CSV, a tuple of only 2 parameters would be retuned: ```cpp // returns std::tuple auto [id, grade] = p.get_next(); @@ -383,6 +397,12 @@ if (std::holds_alternative(grade)) { // grade set as char } ``` +Passing **`char`** and types that are aliases to it such as **`uint8_t`** and **`int8_t`** make the parser interpret the input data as a single character in a similar way to how **`std::cin`** does it. To read numeric values into something like **`uint8_t`** the **`ss::uint8`** and **`ss::int8`** types can be used. These are wrappers arround the corresponding char aliases and can be implicitly converted to and from them. When these types are given to the parser he will try to read the given data and store it in the underlying element, but this time as a numeric value instead of a single character. +```cpp +// returns std::tuple +auto [id, age, grade] = p.get_next(); +uint8_t age_copy = age; +``` ## Restrictions Custom **`restrictions`** can be used to narrow down the conversions of unwanted values. **`ss::ir`** (in range) and **`ss::ne`** (none empty) are some of those: @@ -454,12 +474,13 @@ The **`eof`** method can be used to detect if the end of the file was reached. Detailed error messages can be accessed via the **`error_msg`** method, and to enable them **`ss::string_error`** needs to be included in the setup. If **`ss::string_error`** is not defined, the **`error_msg`** method will not be defined either. The line number can be fetched using the **`line`** method. - +The cursor position can be fetched using the **`position`** method. ```cpp -const std::string& parser::error_msg(); -bool parser::valid(); -bool parser::eof(); -size_t parser::line(); +const std::string& parser::error_msg() const; +bool parser::valid() const; +bool parser::eof() const; +size_t parser::line() const; +size_t parser::position() const; // ... ss::parser parser; @@ -474,7 +495,7 @@ ss::parser parser; ## Substitute conversions -The parser can also be used to effectively parse files whose rows are not always in the same format (not a classical csv but still csv-like). A more complicated example would be the best way to demonstrate such a scenario.\ +The parser can also be used to effectively parse files whose rows are not always in the same format (not a classical CSV but still CSV-like). A more complicated example would be the best way to demonstrate such a scenario.\ ***Important, substitute conversions do not work when throw_on_error is enabled.*** Supposing we have a file containing different shapes in given formats: diff --git a/include/ss/common.hpp b/include/ss/common.hpp index c9185f8..9338128 100644 --- a/include/ss/common.hpp +++ b/include/ss/common.hpp @@ -28,15 +28,24 @@ inline void assert_throw_on_error_not_defined() { "'throw_on_error' is enabled"); } +inline void* strict_realloc(void* ptr, size_t size) { + ptr = std::realloc(ptr, size); + if (!ptr) { + throw std::bad_alloc{}; + } + + return ptr; +} + #if __unix__ -inline ssize_t get_line(char** lineptr, size_t* n, FILE* stream) { +inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) { return getline(lineptr, n, stream); } #else -using ssize_t = int64_t; +using ssize_t = intptr_t; -ssize_t get_line(char** lineptr, size_t* n, FILE* fp) { +ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) { if (lineptr == nullptr || n == nullptr || fp == nullptr) { errno = EINVAL; return -1; @@ -46,36 +55,24 @@ ssize_t get_line(char** lineptr, size_t* n, FILE* fp) { if (*lineptr == nullptr || *n < sizeof(buff)) { size_t new_n = sizeof(buff); - auto new_lineptr = static_cast(realloc(*lineptr, new_n)); - if (new_lineptr == nullptr) { - errno = ENOMEM; - return -1; - } - - *lineptr = new_lineptr; + *lineptr = static_cast(strict_realloc(*lineptr, new_n)); *n = new_n; } (*lineptr)[0] = '\0'; - while (fgets(buff, sizeof(buff), fp) != nullptr) { - size_t line_used = strlen(*lineptr); - size_t buff_used = strlen(buff); + size_t line_used = 0; + while (std::fgets(buff, sizeof(buff), fp) != nullptr) { + line_used = std::strlen(*lineptr); + size_t buff_used = std::strlen(buff); if (*n <= buff_used + line_used) { size_t new_n = *n * 2; - - auto new_lineptr = static_cast(realloc(*lineptr, new_n)); - if (new_lineptr == nullptr) { - errno = ENOMEM; - return -1; - } - - *lineptr = new_lineptr; + *lineptr = static_cast(strict_realloc(*lineptr, new_n)); *n = new_n; } - memcpy(*lineptr + line_used, buff, buff_used); + std::memcpy(*lineptr + line_used, buff, buff_used); line_used += buff_used; (*lineptr)[line_used] = '\0'; @@ -84,7 +81,7 @@ ssize_t get_line(char** lineptr, size_t* n, FILE* fp) { } } - return -1; + return (line_used != 0) ? line_used : -1; } #endif diff --git a/include/ss/extract.hpp b/include/ss/extract.hpp index a26828d..64b6b1b 100644 --- a/include/ss/extract.hpp +++ b/include/ss/extract.hpp @@ -77,6 +77,38 @@ std::enable_if_t, std::optional> to_num( #endif +//////////////// +// numeric_wrapper +//////////////// + +template +struct numeric_wrapper { + using type = T; + + numeric_wrapper() = default; + numeric_wrapper(numeric_wrapper&&) = default; + numeric_wrapper(const numeric_wrapper&) = default; + + numeric_wrapper& operator=(numeric_wrapper&&) = default; + numeric_wrapper& operator=(const numeric_wrapper&) = default; + + numeric_wrapper(T other) : value{other} { + } + + operator T() { + return value; + } + + operator T() const { + return value; + } + + T value; +}; + +using int8 = numeric_wrapper; +using uint8 = numeric_wrapper; + template std::enable_if_t, std::optional> to_num( const char* const begin, const char* const end) { @@ -89,6 +121,18 @@ std::enable_if_t, std::optional> to_num( return ret; } +template +std::enable_if_t, std::optional> to_num( + const char* const begin, const char* const end) { + T ret; + auto [ptr, ec] = std::from_chars(begin, end, ret.value); + + if (ec != std::errc() || ptr != end) { + return std::nullopt; + } + return ret; +} + //////////////// // extract //////////////// @@ -103,7 +147,8 @@ struct unsupported_type { template std::enable_if_t && !std::is_floating_point_v && !is_instance_of_v && - !is_instance_of_v, + !is_instance_of_v && + !is_instance_of_v, bool> extract(const char*, const char*, T&) { static_assert(error::unsupported_type::value, @@ -112,7 +157,9 @@ extract(const char*, const char*, T&) { } template -std::enable_if_t || std::is_floating_point_v, bool> +std::enable_if_t || std::is_floating_point_v || + is_instance_of_v, + bool> extract(const char* begin, const char* end, T& value) { auto optional_value = to_num(begin, end); if (!optional_value) { @@ -169,9 +216,9 @@ inline bool extract(const char* begin, const char* end, bool& value) { } } else { size_t size = end - begin; - if (size == 4 && strncmp(begin, "true", size) == 0) { + if (size == 4 && std::strncmp(begin, "true", size) == 0) { value = true; - } else if (size == 5 && strncmp(begin, "false", size) == 0) { + } else if (size == 5 && std::strncmp(begin, "false", size) == 0) { value = false; } else { return false; diff --git a/include/ss/parser.hpp b/include/ss/parser.hpp index 10f98e1..154a881 100644 --- a/include/ss/parser.hpp +++ b/include/ss/parser.hpp @@ -5,6 +5,7 @@ #include "exception.hpp" #include "extract.hpp" #include "restrictions.hpp" +#include #include #include #include @@ -48,6 +49,23 @@ public: } } + parser(const char* const csv_data_buffer, size_t csv_data_size, + const std::string& delim = ss::default_delimiter) + : file_name_{"buffer line"}, + reader_{csv_data_buffer, csv_data_size, delim} { + if (csv_data_buffer) { + read_line(); + if constexpr (ignore_header) { + ignore_next(); + } else { + raw_header_ = reader_.get_buffer(); + } + } else { + handle_error_null_buffer(); + eof_ = true; + } + } + parser(parser&& other) = default; parser& operator=(parser&& other) = default; @@ -84,10 +102,14 @@ public: } size_t line() const { - return reader_.line_number_ > 1 ? reader_.line_number_ - 1 + return reader_.line_number_ > 0 ? reader_.line_number_ - 1 : reader_.line_number_; } + size_t position() const { + return reader_.chars_read_; + } + template no_void_validator_tup_t get_next() { std::optional error; @@ -192,7 +214,7 @@ public: reader_.next_line_converter_.set_column_mapping(column_mappings, header_.size()); - if (line() == 1) { + if (line() == 0) { ignore_next(); } } @@ -351,20 +373,22 @@ public: template void try_convert_and_invoke(std::optional& value, Fun&& fun) { - if (!parser_.valid()) { - auto tuple_output = try_same(); - if (!parser_.valid()) { - return; - } - - if constexpr (!std::is_same_v) { - value = to_object(std::move(tuple_output)); - } else { - value = std::move(tuple_output); - } - - parser_.try_invoke(*value, std::forward(fun)); + if (parser_.valid()) { + return; } + + auto tuple_output = try_same(); + if (!parser_.valid()) { + return; + } + + if constexpr (!std::is_same_v) { + value = to_object(std::move(tuple_output)); + } else { + value = std::move(tuple_output); + } + + parser_.try_invoke(*value, std::forward(fun)); } template @@ -512,6 +536,19 @@ private: } } + void handle_error_null_buffer() { + constexpr static auto error_msg = " received null data buffer"; + + if constexpr (string_error) { + error_.clear(); + error_.append(file_name_).append(error_msg); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg}; + } else { + error_ = true; + } + } + void handle_error_file_not_open() { constexpr static auto error_msg = " could not be opened"; @@ -638,21 +675,30 @@ private: struct reader { reader(const std::string& file_name_, const std::string& delim) - : delim_{delim}, file_{fopen(file_name_.c_str(), "rb")} { + : delim_{delim}, file_{std::fopen(file_name_.c_str(), "rb")} { + } + + reader(const char* const buffer, size_t csv_data_size, + const std::string& delim) + : delim_{delim}, csv_data_buffer_{buffer}, + csv_data_size_{csv_data_size} { } reader(reader&& other) : buffer_{other.buffer_}, next_line_buffer_{other.next_line_buffer_}, - helper_buffer_{other.helper_buffer_}, converter_{std::move( - other.converter_)}, + helper_buffer_{other.helper_buffer_}, + converter_{std::move(other.converter_)}, next_line_converter_{std::move(other.next_line_converter_)}, buffer_size_{other.buffer_size_}, next_line_buffer_size_{other.next_line_buffer_size_}, - helper_size_{other.helper_size_}, delim_{std::move(other.delim_)}, - file_{other.file_}, crlf_{other.crlf_}, - line_number_{other.line_number_}, next_line_size_{ - other.next_line_size_} { + helper_buffer_size{other.helper_buffer_size}, + delim_{std::move(other.delim_)}, file_{other.file_}, + csv_data_buffer_{other.csv_data_buffer_}, + csv_data_size_{other.csv_data_size_}, + curr_char_{other.curr_char_}, crlf_{other.crlf_}, + line_number_{other.line_number_}, chars_read_{other.chars_read_}, + next_line_size_{other.next_line_size_} { other.buffer_ = nullptr; other.next_line_buffer_ = nullptr; other.helper_buffer_ = nullptr; @@ -668,29 +714,34 @@ private: next_line_converter_ = std::move(other.next_line_converter_); buffer_size_ = other.buffer_size_; next_line_buffer_size_ = other.next_line_buffer_size_; - helper_size_ = other.helper_size_; + helper_buffer_size = other.helper_buffer_size; delim_ = std::move(other.delim_); file_ = other.file_; + csv_data_buffer_ = other.csv_data_buffer_; + csv_data_size_ = other.csv_data_size_; + curr_char_ = other.curr_char_; crlf_ = other.crlf_; line_number_ = other.line_number_; + chars_read_ = other.chars_read_; next_line_size_ = other.next_line_size_; other.buffer_ = nullptr; other.next_line_buffer_ = nullptr; other.helper_buffer_ = nullptr; other.file_ = nullptr; + other.csv_data_buffer_ = nullptr; } return *this; } ~reader() { - free(buffer_); - free(next_line_buffer_); - free(helper_buffer_); + std::free(buffer_); + std::free(next_line_buffer_); + std::free(helper_buffer_); if (file_) { - fclose(file_); + std::fclose(file_); } } @@ -698,6 +749,42 @@ private: reader(const reader& other) = delete; reader& operator=(const reader& other) = delete; + ssize_t get_line_buffer(char** lineptr, size_t* n, + const char* const csv_data_buffer, + size_t csv_data_size, size_t& curr_char) { + if (curr_char >= csv_data_size) { + return -1; + } + + if (*lineptr == nullptr || *n < get_line_initial_buffer_size) { + auto new_lineptr = static_cast( + strict_realloc(*lineptr, get_line_initial_buffer_size)); + *lineptr = new_lineptr; + *n = get_line_initial_buffer_size; + } + + size_t line_used = 0; + while (curr_char <= csv_data_size) { + if (line_used + 1 >= *n) { + size_t new_n = *n * 2; + + char* new_lineptr = + static_cast(strict_realloc(*lineptr, new_n)); + *n = new_n; + *lineptr = new_lineptr; + } + + auto c = csv_data_buffer[curr_char++]; + (*lineptr)[line_used++] = c; + if (c == '\n') { + (*lineptr)[line_used] = '\0'; + return line_used; + } + } + + return (line_used != 0) ? line_used : -1; + } + // read next line each time in order to set eof_ bool read_next() { next_line_converter_.clear_error(); @@ -708,10 +795,23 @@ private: if (next_line_buffer_size_ > 0) { next_line_buffer_[0] = '\0'; } - ssize = get_line(&next_line_buffer_, &next_line_buffer_size_, - file_); + + chars_read_ = curr_char_; + if (file_) { + ssize = get_line_file(&next_line_buffer_, + &next_line_buffer_size_, file_); + curr_char_ = std::ftell(file_); + } else { + ssize = get_line_buffer(&next_line_buffer_, + &next_line_buffer_size_, + csv_data_buffer_, csv_data_size_, + curr_char_); + } if (ssize == -1) { + if (errno == ENOMEM) { + throw std::bad_alloc{}; + } return false; } @@ -821,6 +921,10 @@ private: } size_t remove_eol(char*& buffer, size_t ssize) { + if (buffer[ssize - 1] != '\n') { + return ssize; + } + size_t size = ssize - 1; if (ssize >= 2 && buffer[ssize - 2] == '\r') { crlf_ = true; @@ -834,14 +938,11 @@ private: } void realloc_concat(char*& first, size_t& first_size, - const char* const second, size_t second_size) { - // TODO make buffer_size an argument - next_line_buffer_size_ = first_size + second_size + 3; + size_t& buffer_size, const char* const second, + size_t second_size) { + buffer_size = first_size + second_size + 3; auto new_first = static_cast( - realloc(static_cast(first), next_line_buffer_size_)); - if (!first) { - throw std::bad_alloc{}; - } + strict_realloc(static_cast(first), buffer_size)); first = new_first; std::copy_n(second, second_size + 1, first + first_size); @@ -851,15 +952,25 @@ private: bool append_next_line_to_buffer(char*& buffer, size_t& size) { undo_remove_eol(buffer, size); - ssize_t next_ssize = - get_line(&helper_buffer_, &helper_size_, file_); + ssize_t next_ssize; + if (file_) { + next_ssize = + get_line_file(&helper_buffer_, &helper_buffer_size, file_); + } else { + next_ssize = + get_line_buffer(&helper_buffer_, &helper_buffer_size, + csv_data_buffer_, csv_data_size_, + curr_char_); + } + if (next_ssize == -1) { return false; } ++line_number_; size_t next_size = remove_eol(helper_buffer_, next_ssize); - realloc_concat(buffer, size, helper_buffer_, next_size); + realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_, + next_size); return true; } @@ -879,13 +990,18 @@ private: size_t buffer_size_{0}; size_t next_line_buffer_size_{0}; - size_t helper_size_{0}; + size_t helper_buffer_size{0}; std::string delim_; FILE* file_{nullptr}; + const char* csv_data_buffer_{nullptr}; + size_t csv_data_size_{0}; + size_t curr_char_{0}; + bool crlf_{false}; size_t line_number_{0}; + size_t chars_read_{0}; size_t next_line_size_{0}; }; diff --git a/include/ss/setup.hpp b/include/ss/setup.hpp index fd41da5..2f298fc 100644 --- a/include/ss/setup.hpp +++ b/include/ss/setup.hpp @@ -109,7 +109,7 @@ struct get_matcher { struct is_matcher : is_instance_of_matcher {}; static_assert(count_v <= 1, - "the same matcher cannot" + "the same matcher cannot " "be defined multiple times"); using type = std::conditional_t::value, T, typename get_matcher::type>; @@ -165,25 +165,25 @@ using get_multiline_t = typename get_multiline::type; // string_error //////////////// -class string_error; +class string_error {}; //////////////// // ignore_header //////////////// -class ignore_header; +class ignore_header {}; //////////////// // ignore_empty //////////////// -class ignore_empty; +class ignore_empty {}; //////////////// // throw_on_error //////////////// -class throw_on_error; +class throw_on_error {}; //////////////// // setup implementation diff --git a/include/ss/splitter.hpp b/include/ss/splitter.hpp index dd8f365..d974e83 100644 --- a/include/ss/splitter.hpp +++ b/include/ss/splitter.hpp @@ -199,7 +199,7 @@ private: }; bool match(const char* const curr, const std::string& delim) { - return strncmp(curr, delim.c_str(), delim.size()) == 0; + return std::strncmp(curr, delim.c_str(), delim.size()) == 0; }; size_t delimiter_size(char) { diff --git a/script/ci_install_lcov.sh b/script/ci_install_lcov.sh new file mode 100755 index 0000000..e1280cd --- /dev/null +++ b/script/ci_install_lcov.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +echo yes | cpan DateTime Capture::Tiny + +wget -qO- https://github.com/linux-test-project/lcov/releases/download/v2.0/lcov-2.0.tar.gz | tar xvz +(cd lcov-2.0 && make install) diff --git a/ssp.hpp b/ssp.hpp index 71d05cf..4cc906e 100644 --- a/ssp.hpp +++ b/ssp.hpp @@ -640,15 +640,24 @@ inline void assert_throw_on_error_not_defined() { "'throw_on_error' is enabled"); } +inline void* strict_realloc(void* ptr, size_t size) { + ptr = std::realloc(ptr, size); + if (!ptr) { + throw std::bad_alloc{}; + } + + return ptr; +} + #if __unix__ -inline ssize_t get_line(char** lineptr, size_t* n, FILE* stream) { +inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) { return getline(lineptr, n, stream); } #else -using ssize_t = int64_t; +using ssize_t = intptr_t; -ssize_t get_line(char** lineptr, size_t* n, FILE* fp) { +ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) { if (lineptr == nullptr || n == nullptr || fp == nullptr) { errno = EINVAL; return -1; @@ -658,36 +667,24 @@ ssize_t get_line(char** lineptr, size_t* n, FILE* fp) { if (*lineptr == nullptr || *n < sizeof(buff)) { size_t new_n = sizeof(buff); - auto new_lineptr = static_cast(realloc(*lineptr, new_n)); - if (new_lineptr == nullptr) { - errno = ENOMEM; - return -1; - } - - *lineptr = new_lineptr; + *lineptr = static_cast(strict_realloc(*lineptr, new_n)); *n = new_n; } (*lineptr)[0] = '\0'; - while (fgets(buff, sizeof(buff), fp) != nullptr) { - size_t line_used = strlen(*lineptr); - size_t buff_used = strlen(buff); + size_t line_used = 0; + while (std::fgets(buff, sizeof(buff), fp) != nullptr) { + line_used = std::strlen(*lineptr); + size_t buff_used = std::strlen(buff); if (*n <= buff_used + line_used) { size_t new_n = *n * 2; - - auto new_lineptr = static_cast(realloc(*lineptr, new_n)); - if (new_lineptr == nullptr) { - errno = ENOMEM; - return -1; - } - - *lineptr = new_lineptr; + *lineptr = static_cast(strict_realloc(*lineptr, new_n)); *n = new_n; } - memcpy(*lineptr + line_used, buff, buff_used); + std::memcpy(*lineptr + line_used, buff, buff_used); line_used += buff_used; (*lineptr)[line_used] = '\0'; @@ -696,7 +693,7 @@ ssize_t get_line(char** lineptr, size_t* n, FILE* fp) { } } - return -1; + return (line_used != 0) ? line_used : -1; } #endif @@ -810,7 +807,7 @@ struct get_matcher { struct is_matcher : is_instance_of_matcher {}; static_assert(count_v <= 1, - "the same matcher cannot" + "the same matcher cannot " "be defined multiple times"); using type = std::conditional_t::value, T, typename get_matcher::type>; @@ -866,25 +863,25 @@ using get_multiline_t = typename get_multiline::type; // string_error //////////////// -class string_error; +class string_error {}; //////////////// // ignore_header //////////////// -class ignore_header; +class ignore_header {}; //////////////// // ignore_empty //////////////// -class ignore_empty; +class ignore_empty {}; //////////////// // throw_on_error //////////////// -class throw_on_error; +class throw_on_error {}; //////////////// // setup implementation @@ -1185,7 +1182,7 @@ private: }; bool match(const char* const curr, const std::string& delim) { - return strncmp(curr, delim.c_str(), delim.size()) == 0; + return std::strncmp(curr, delim.c_str(), delim.size()) == 0; }; size_t delimiter_size(char) { @@ -1533,6 +1530,38 @@ std::enable_if_t, std::optional> to_num( #endif +//////////////// +// numeric_wrapper +//////////////// + +template +struct numeric_wrapper { + using type = T; + + numeric_wrapper() = default; + numeric_wrapper(numeric_wrapper&&) = default; + numeric_wrapper(const numeric_wrapper&) = default; + + numeric_wrapper& operator=(numeric_wrapper&&) = default; + numeric_wrapper& operator=(const numeric_wrapper&) = default; + + numeric_wrapper(T other) : value{other} { + } + + operator T() { + return value; + } + + operator T() const { + return value; + } + + T value; +}; + +using int8 = numeric_wrapper; +using uint8 = numeric_wrapper; + template std::enable_if_t, std::optional> to_num( const char* const begin, const char* const end) { @@ -1545,6 +1574,18 @@ std::enable_if_t, std::optional> to_num( return ret; } +template +std::enable_if_t, std::optional> to_num( + const char* const begin, const char* const end) { + T ret; + auto [ptr, ec] = std::from_chars(begin, end, ret.value); + + if (ec != std::errc() || ptr != end) { + return std::nullopt; + } + return ret; +} + //////////////// // extract //////////////// @@ -1559,7 +1600,8 @@ struct unsupported_type { template std::enable_if_t && !std::is_floating_point_v && !is_instance_of_v && - !is_instance_of_v, + !is_instance_of_v && + !is_instance_of_v, bool> extract(const char*, const char*, T&) { static_assert(error::unsupported_type::value, @@ -1568,7 +1610,9 @@ extract(const char*, const char*, T&) { } template -std::enable_if_t || std::is_floating_point_v, bool> +std::enable_if_t || std::is_floating_point_v || + is_instance_of_v, + bool> extract(const char* begin, const char* end, T& value) { auto optional_value = to_num(begin, end); if (!optional_value) { @@ -1625,9 +1669,9 @@ inline bool extract(const char* begin, const char* end, bool& value) { } } else { size_t size = end - begin; - if (size == 4 && strncmp(begin, "true", size) == 0) { + if (size == 4 && std::strncmp(begin, "true", size) == 0) { value = true; - } else if (size == 5 && strncmp(begin, "false", size) == 0) { + } else if (size == 5 && std::strncmp(begin, "false", size) == 0) { value = false; } else { return false; @@ -2181,6 +2225,23 @@ public: } } + parser(const char* const csv_data_buffer, size_t csv_data_size, + const std::string& delim = ss::default_delimiter) + : file_name_{"buffer line"}, + reader_{csv_data_buffer, csv_data_size, delim} { + if (csv_data_buffer) { + read_line(); + if constexpr (ignore_header) { + ignore_next(); + } else { + raw_header_ = reader_.get_buffer(); + } + } else { + handle_error_null_buffer(); + eof_ = true; + } + } + parser(parser&& other) = default; parser& operator=(parser&& other) = default; @@ -2217,10 +2278,14 @@ public: } size_t line() const { - return reader_.line_number_ > 1 ? reader_.line_number_ - 1 + return reader_.line_number_ > 0 ? reader_.line_number_ - 1 : reader_.line_number_; } + size_t position() const { + return reader_.chars_read_; + } + template no_void_validator_tup_t get_next() { std::optional error; @@ -2325,7 +2390,7 @@ public: reader_.next_line_converter_.set_column_mapping(column_mappings, header_.size()); - if (line() == 1) { + if (line() == 0) { ignore_next(); } } @@ -2484,20 +2549,22 @@ public: template void try_convert_and_invoke(std::optional& value, Fun&& fun) { - if (!parser_.valid()) { - auto tuple_output = try_same(); - if (!parser_.valid()) { - return; - } - - if constexpr (!std::is_same_v) { - value = to_object(std::move(tuple_output)); - } else { - value = std::move(tuple_output); - } - - parser_.try_invoke(*value, std::forward(fun)); + if (parser_.valid()) { + return; } + + auto tuple_output = try_same(); + if (!parser_.valid()) { + return; + } + + if constexpr (!std::is_same_v) { + value = to_object(std::move(tuple_output)); + } else { + value = std::move(tuple_output); + } + + parser_.try_invoke(*value, std::forward(fun)); } template @@ -2645,6 +2712,19 @@ private: } } + void handle_error_null_buffer() { + constexpr static auto error_msg = " received null data buffer"; + + if constexpr (string_error) { + error_.clear(); + error_.append(file_name_).append(error_msg); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg}; + } else { + error_ = true; + } + } + void handle_error_file_not_open() { constexpr static auto error_msg = " could not be opened"; @@ -2771,21 +2851,30 @@ private: struct reader { reader(const std::string& file_name_, const std::string& delim) - : delim_{delim}, file_{fopen(file_name_.c_str(), "rb")} { + : delim_{delim}, file_{std::fopen(file_name_.c_str(), "rb")} { + } + + reader(const char* const buffer, size_t csv_data_size, + const std::string& delim) + : delim_{delim}, csv_data_buffer_{buffer}, + csv_data_size_{csv_data_size} { } reader(reader&& other) : buffer_{other.buffer_}, next_line_buffer_{other.next_line_buffer_}, - helper_buffer_{other.helper_buffer_}, converter_{std::move( - other.converter_)}, + helper_buffer_{other.helper_buffer_}, + converter_{std::move(other.converter_)}, next_line_converter_{std::move(other.next_line_converter_)}, buffer_size_{other.buffer_size_}, next_line_buffer_size_{other.next_line_buffer_size_}, - helper_size_{other.helper_size_}, delim_{std::move(other.delim_)}, - file_{other.file_}, crlf_{other.crlf_}, - line_number_{other.line_number_}, next_line_size_{ - other.next_line_size_} { + helper_buffer_size{other.helper_buffer_size}, + delim_{std::move(other.delim_)}, file_{other.file_}, + csv_data_buffer_{other.csv_data_buffer_}, + csv_data_size_{other.csv_data_size_}, + curr_char_{other.curr_char_}, crlf_{other.crlf_}, + line_number_{other.line_number_}, chars_read_{other.chars_read_}, + next_line_size_{other.next_line_size_} { other.buffer_ = nullptr; other.next_line_buffer_ = nullptr; other.helper_buffer_ = nullptr; @@ -2801,29 +2890,34 @@ private: next_line_converter_ = std::move(other.next_line_converter_); buffer_size_ = other.buffer_size_; next_line_buffer_size_ = other.next_line_buffer_size_; - helper_size_ = other.helper_size_; + helper_buffer_size = other.helper_buffer_size; delim_ = std::move(other.delim_); file_ = other.file_; + csv_data_buffer_ = other.csv_data_buffer_; + csv_data_size_ = other.csv_data_size_; + curr_char_ = other.curr_char_; crlf_ = other.crlf_; line_number_ = other.line_number_; + chars_read_ = other.chars_read_; next_line_size_ = other.next_line_size_; other.buffer_ = nullptr; other.next_line_buffer_ = nullptr; other.helper_buffer_ = nullptr; other.file_ = nullptr; + other.csv_data_buffer_ = nullptr; } return *this; } ~reader() { - free(buffer_); - free(next_line_buffer_); - free(helper_buffer_); + std::free(buffer_); + std::free(next_line_buffer_); + std::free(helper_buffer_); if (file_) { - fclose(file_); + std::fclose(file_); } } @@ -2831,6 +2925,42 @@ private: reader(const reader& other) = delete; reader& operator=(const reader& other) = delete; + ssize_t get_line_buffer(char** lineptr, size_t* n, + const char* const csv_data_buffer, + size_t csv_data_size, size_t& curr_char) { + if (curr_char >= csv_data_size) { + return -1; + } + + if (*lineptr == nullptr || *n < get_line_initial_buffer_size) { + auto new_lineptr = static_cast( + strict_realloc(*lineptr, get_line_initial_buffer_size)); + *lineptr = new_lineptr; + *n = get_line_initial_buffer_size; + } + + size_t line_used = 0; + while (curr_char <= csv_data_size) { + if (line_used + 1 >= *n) { + size_t new_n = *n * 2; + + char* new_lineptr = + static_cast(strict_realloc(*lineptr, new_n)); + *n = new_n; + *lineptr = new_lineptr; + } + + auto c = csv_data_buffer[curr_char++]; + (*lineptr)[line_used++] = c; + if (c == '\n') { + (*lineptr)[line_used] = '\0'; + return line_used; + } + } + + return (line_used != 0) ? line_used : -1; + } + // read next line each time in order to set eof_ bool read_next() { next_line_converter_.clear_error(); @@ -2841,10 +2971,23 @@ private: if (next_line_buffer_size_ > 0) { next_line_buffer_[0] = '\0'; } - ssize = get_line(&next_line_buffer_, &next_line_buffer_size_, - file_); + + chars_read_ = curr_char_; + if (file_) { + ssize = get_line_file(&next_line_buffer_, + &next_line_buffer_size_, file_); + curr_char_ = std::ftell(file_); + } else { + ssize = get_line_buffer(&next_line_buffer_, + &next_line_buffer_size_, + csv_data_buffer_, csv_data_size_, + curr_char_); + } if (ssize == -1) { + if (errno == ENOMEM) { + throw std::bad_alloc{}; + } return false; } @@ -2954,6 +3097,10 @@ private: } size_t remove_eol(char*& buffer, size_t ssize) { + if (buffer[ssize - 1] != '\n') { + return ssize; + } + size_t size = ssize - 1; if (ssize >= 2 && buffer[ssize - 2] == '\r') { crlf_ = true; @@ -2967,14 +3114,11 @@ private: } void realloc_concat(char*& first, size_t& first_size, - const char* const second, size_t second_size) { - // TODO make buffer_size an argument - next_line_buffer_size_ = first_size + second_size + 3; + size_t& buffer_size, const char* const second, + size_t second_size) { + buffer_size = first_size + second_size + 3; auto new_first = static_cast( - realloc(static_cast(first), next_line_buffer_size_)); - if (!first) { - throw std::bad_alloc{}; - } + strict_realloc(static_cast(first), buffer_size)); first = new_first; std::copy_n(second, second_size + 1, first + first_size); @@ -2984,15 +3128,25 @@ private: bool append_next_line_to_buffer(char*& buffer, size_t& size) { undo_remove_eol(buffer, size); - ssize_t next_ssize = - get_line(&helper_buffer_, &helper_size_, file_); + ssize_t next_ssize; + if (file_) { + next_ssize = + get_line_file(&helper_buffer_, &helper_buffer_size, file_); + } else { + next_ssize = + get_line_buffer(&helper_buffer_, &helper_buffer_size, + csv_data_buffer_, csv_data_size_, + curr_char_); + } + if (next_ssize == -1) { return false; } ++line_number_; size_t next_size = remove_eol(helper_buffer_, next_ssize); - realloc_concat(buffer, size, helper_buffer_, next_size); + realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_, + next_size); return true; } @@ -3012,13 +3166,18 @@ private: size_t buffer_size_{0}; size_t next_line_buffer_size_{0}; - size_t helper_size_{0}; + size_t helper_buffer_size{0}; std::string delim_; FILE* file_{nullptr}; + const char* csv_data_buffer_{nullptr}; + size_t csv_data_size_{0}; + size_t curr_char_{0}; + bool crlf_{false}; size_t line_number_{0}; + size_t chars_read_{0}; size_t next_line_size_{0}; }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index de6b612..be399a1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,9 +32,11 @@ set(DOCTEST "${FETCHCONTENT_BASE_DIR}/doctest-src") enable_testing() -foreach(name IN ITEMS test_splitter test_parser test_converter test_extractions - test_parser2_1 test_parser2_2 test_parser2_3 - test_parser2_4 test_extractions_without_fast_float) +foreach(name IN ITEMS test_splitter test_parser1_1 test_parser1_2 + test_parser1_3 test_parser1_4 test_converter + test_extractions test_parser2_1 test_parser2_2 + test_parser2_3 test_parser2_4 test_parser2_5 + test_parser2_6 test_extractions_without_fast_float) add_executable("${name}" "${name}.cpp") target_link_libraries("${name}" PRIVATE ssp::ssp fast_float doctest::doctest) diff --git a/test/meson.build b/test/meson.build index 25bf963..f4e6639 100644 --- a/test/meson.build +++ b/test/meson.build @@ -2,7 +2,10 @@ doctest_dep = dependency('doctest') add_project_arguments('-DDOCTEST_CONFIG_IMPLEMENT_WITH_MAIN', language: 'cpp') tests = [ - 'parser', + 'parser1_1', + 'parser1_2', + 'parser1_3', + 'parser1_4', 'splitter', 'converter', 'extractions', @@ -10,6 +13,8 @@ tests = [ 'parser2_2', 'parser2_3', 'parser2_4', + 'parser2_5', + 'parser2_6', 'extractions_without_fast_float', ] diff --git a/test/test_converter.cpp b/test/test_converter.cpp index 6dd4562..659e78f 100644 --- a/test/test_converter.cpp +++ b/test/test_converter.cpp @@ -46,90 +46,88 @@ TEST_CASE("converter test split with exceptions") { } } -TEST_CASE("converter test valid conversions") { +TEST_CASE_TEMPLATE("converter test valid conversions", T, int, ss::uint8) { ss::converter c; { - auto tup = c.convert("5"); + auto tup = c.convert("5"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } { - auto tup = c.convert("5,junk"); + auto tup = c.convert("5,junk"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } { - auto tup = c.convert("junk,5"); + auto tup = c.convert("junk,5"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } { - auto tup = c.convert("5\njunk\njunk", "\n"); + auto tup = c.convert("5\njunk\njunk", "\n"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } { - auto tup = c.convert("junk 5 junk", " "); + auto tup = c.convert("junk 5 junk", " "); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } { - auto tup = c.convert("junk\tjunk\t5", "\t"); + auto tup = c.convert("junk\tjunk\t5", "\t"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } { auto tup = - c.convert>("junk\tjunk\t5", "\t"); + c.convert>("junk\tjunk\t5", "\t"); REQUIRE(c.valid()); REQUIRE(tup.has_value()); CHECK_EQ(tup, 5); } { - auto tup = c.convert("5,6.6,junk"); + auto tup = c.convert("5,6.6,junk"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); } { - auto tup = c.convert("5,junk,6.6"); + auto tup = c.convert("5,junk,6.6"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); } { - auto tup = c.convert("junk;5;6.6", ";"); + auto tup = c.convert("junk;5;6.6", ";"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); } { - auto tup = - c.convert, double>("junk;5;6.6", ";"); + auto tup = c.convert, double>("junk;5;6.6", ";"); REQUIRE(c.valid()); REQUIRE(std::get<0>(tup).has_value()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); } { auto tup = - c.convert, double>("junk;5.4;6.6", ";"); + c.convert, double>("junk;5.4;6.6", ";"); REQUIRE(c.valid()); REQUIRE_FALSE(std::get<0>(tup).has_value()); - CHECK_EQ(tup, std::make_tuple(std::optional{}, 6.6)); + CHECK_EQ(tup, std::make_tuple(std::optional{}, 6.6)); } { auto tup = - c.convert, double>("junk;5;6.6", - ";"); + c.convert, double>("junk;5;6.6", ";"); REQUIRE(c.valid()); - REQUIRE(std::holds_alternative(std::get<0>(tup))); - CHECK_EQ(tup, std::make_tuple(std::variant{5}, 6.6)); + REQUIRE(std::holds_alternative(std::get<0>(tup))); + CHECK_EQ(tup, std::make_tuple(std::variant{T(5)}, 6.6)); } { auto tup = - c.convert, double>("junk;5.5;6.6", - ";"); + c.convert, double>("junk;5.5;6.6", + ";"); REQUIRE(c.valid()); REQUIRE(std::holds_alternative(std::get<0>(tup))); - CHECK_EQ(tup, std::make_tuple(std::variant{5.5}, 6.6)); + CHECK_EQ(tup, std::make_tuple(std::variant{5.5}, 6.6)); } { auto tup = c.convert c; try { - auto tup = c.convert("5"); + auto tup = c.convert("5"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } catch (ss::exception& e) { @@ -152,7 +151,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("5,junk"); + auto tup = c.convert("5,junk"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } catch (ss::exception& e) { @@ -160,7 +159,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("junk,5"); + auto tup = c.convert("junk,5"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } catch (ss::exception& e) { @@ -168,7 +167,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("5\njunk\njunk", "\n"); + auto tup = c.convert("5\njunk\njunk", "\n"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } catch (ss::exception& e) { @@ -176,7 +175,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("junk 5 junk", " "); + auto tup = c.convert("junk 5 junk", " "); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } catch (ss::exception& e) { @@ -184,7 +183,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("junk\tjunk\t5", "\t"); + auto tup = c.convert("junk\tjunk\t5", "\t"); REQUIRE(c.valid()); CHECK_EQ(tup, 5); } catch (ss::exception& e) { @@ -193,7 +192,7 @@ TEST_CASE("converter test valid conversions with exceptions") { try { auto tup = - c.convert>("junk\tjunk\t5", "\t"); + c.convert>("junk\tjunk\t5", "\t"); REQUIRE(c.valid()); REQUIRE(tup.has_value()); CHECK_EQ(tup, 5); @@ -202,7 +201,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("5,6.6,junk"); + auto tup = c.convert("5,6.6,junk"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); } catch (ss::exception& e) { @@ -210,7 +209,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("5,junk,6.6"); + auto tup = c.convert("5,junk,6.6"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); } catch (ss::exception& e) { @@ -218,7 +217,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = c.convert("junk;5;6.6", ";"); + auto tup = c.convert("junk;5;6.6", ";"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); } catch (ss::exception& e) { @@ -226,8 +225,7 @@ TEST_CASE("converter test valid conversions with exceptions") { } try { - auto tup = - c.convert, double>("junk;5;6.6", ";"); + auto tup = c.convert, double>("junk;5;6.6", ";"); REQUIRE(c.valid()); REQUIRE(std::get<0>(tup).has_value()); CHECK_EQ(tup, std::make_tuple(5, 6.6)); @@ -237,32 +235,31 @@ TEST_CASE("converter test valid conversions with exceptions") { try { auto tup = - c.convert, double>("junk;5.4;6.6", ";"); + c.convert, double>("junk;5.4;6.6", ";"); REQUIRE(c.valid()); REQUIRE_FALSE(std::get<0>(tup).has_value()); - CHECK_EQ(tup, std::make_tuple(std::optional{}, 6.6)); + CHECK_EQ(tup, std::make_tuple(std::optional{}, 6.6)); } catch (ss::exception& e) { FAIL(std::string{e.what()}); } try { auto tup = - c.convert, double>("junk;5;6.6", - ";"); + c.convert, double>("junk;5;6.6", ";"); REQUIRE(c.valid()); - REQUIRE(std::holds_alternative(std::get<0>(tup))); - CHECK_EQ(tup, std::make_tuple(std::variant{5}, 6.6)); + REQUIRE(std::holds_alternative(std::get<0>(tup))); + CHECK_EQ(tup, std::make_tuple(std::variant{T(5)}, 6.6)); } catch (ss::exception& e) { FAIL(std::string{e.what()}); } try { auto tup = - c.convert, double>("junk;5.5;6.6", - ";"); + c.convert, double>("junk;5.5;6.6", + ";"); REQUIRE(c.valid()); REQUIRE(std::holds_alternative(std::get<0>(tup))); - CHECK_EQ(tup, std::make_tuple(std::variant{5.5}, 6.6)); + CHECK_EQ(tup, std::make_tuple(std::variant{5.5}, 6.6)); } catch (ss::exception& e) { FAIL(std::string{e.what()}); } @@ -278,110 +275,114 @@ TEST_CASE("converter test valid conversions with exceptions") { } } -TEST_CASE("converter test invalid conversions") { +TEST_CASE_TEMPLATE("converter test invalid conversions", T, int, ss::uint8) { ss::converter c; - c.convert(""); + c.convert(""); REQUIRE_FALSE(c.valid()); - c.convert("1", ""); + c.convert("1", ""); REQUIRE_FALSE(c.valid()); - c.convert("10", ""); + c.convert("10", ""); REQUIRE_FALSE(c.valid()); - c.convert(""); + c.convert(""); REQUIRE_FALSE(c.valid()); - c.convert(",junk"); + c.convert(",junk"); REQUIRE_FALSE(c.valid()); - c.convert("junk,"); + c.convert("junk,"); REQUIRE_FALSE(c.valid()); - c.convert("x"); + c.convert("x"); REQUIRE_FALSE(c.valid()); - c.convert("x"); + c.convert("x"); REQUIRE_FALSE(c.valid()); - c.convert("x,junk"); + c.convert("x,junk"); REQUIRE_FALSE(c.valid()); - c.convert("junk,x"); + c.convert("junk,x"); REQUIRE_FALSE(c.valid()); - c.convert, double>("junk;.5.5;6", ";"); + c.convert, double>("junk;.5.5;6", ";"); REQUIRE_FALSE(c.valid()); } -TEST_CASE("converter test invalid conversions with exceptions") { +TEST_CASE_TEMPLATE("converter test invalid conversions with exceptions", T, int, + ss::uint8) { ss::converter c; - REQUIRE_EXCEPTION(c.convert("")); - REQUIRE_EXCEPTION(c.convert("1", "")); - REQUIRE_EXCEPTION(c.convert("10", "")); - REQUIRE_EXCEPTION(c.convert("")); - REQUIRE_EXCEPTION(c.convert(",junk")); - REQUIRE_EXCEPTION(c.convert("junk,")); - REQUIRE_EXCEPTION(c.convert("x")); - REQUIRE_EXCEPTION(c.convert("x")); - REQUIRE_EXCEPTION(c.convert("x,junk")); - REQUIRE_EXCEPTION(c.convert("junk,x")); + REQUIRE_EXCEPTION(c.convert("")); + REQUIRE_EXCEPTION(c.convert("1", "")); + REQUIRE_EXCEPTION(c.convert("10", "")); + REQUIRE_EXCEPTION(c.convert("")); + REQUIRE_EXCEPTION(c.convert(",junk")); + REQUIRE_EXCEPTION(c.convert("junk,")); + REQUIRE_EXCEPTION(c.convert("x")); + REQUIRE_EXCEPTION(c.convert("x")); + REQUIRE_EXCEPTION(c.convert("x,junk")); + REQUIRE_EXCEPTION(c.convert("junk,x")); REQUIRE_EXCEPTION( - c.convert, double>("junk;.5.5;6", ";")); + c.convert, double>("junk;.5.5;6", ";")); } -TEST_CASE("converter test ss:ax restriction (all except)") { +TEST_CASE_TEMPLATE("converter test ss:ax restriction (all except)", T, int, + ss::uint8) { ss::converter c; - c.convert>("0"); + c.convert>("0"); REQUIRE_FALSE(c.valid()); - c.convert>("1"); + c.convert>("1"); REQUIRE_FALSE(c.valid()); - c.convert>("junk,c,1"); + c.convert>("junk,c,1"); REQUIRE_FALSE(c.valid()); - c.convert, char>("1,c"); + c.convert, char>("1,c"); REQUIRE_FALSE(c.valid()); { - int tup = c.convert>("3"); + T tup = c.convert>("3"); REQUIRE(c.valid()); CHECK_EQ(tup, 3); } { - std::tuple tup = c.convert>("c,3"); + std::tuple tup = c.convert>("c,3"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple('c', 3)); } { - std::tuple tup = c.convert, char>("3,c"); + std::tuple tup = c.convert, char>("3,c"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(3, 'c')); } } -TEST_CASE("converter test ss:ax restriction (all except) with exceptions") { +TEST_CASE_TEMPLATE( + "converter test ss:ax restriction (all except) with exceptions", T, int, + ss::uint8) { ss::converter c; - REQUIRE_EXCEPTION(c.convert>("0")); - REQUIRE_EXCEPTION(c.convert>("1")); - REQUIRE_EXCEPTION(c.convert>("junk,c,1")); - REQUIRE_EXCEPTION(c.convert, char>("1,c")); + REQUIRE_EXCEPTION(c.convert>("0")); + REQUIRE_EXCEPTION(c.convert>("1")); + REQUIRE_EXCEPTION(c.convert>("junk,c,1")); + REQUIRE_EXCEPTION(c.convert, char>("1,c")); try { { - int tup = c.convert>("3"); + T tup = c.convert>("3"); CHECK_EQ(tup, 3); } { - std::tuple tup = c.convert>("c,3"); + std::tuple tup = c.convert>("c,3"); CHECK_EQ(tup, std::make_tuple('c', 3)); } { - std::tuple tup = c.convert, char>("3,c"); + std::tuple tup = c.convert, char>("3,c"); CHECK_EQ(tup, std::make_tuple(3, 'c')); } } catch (ss::exception& e) { @@ -456,65 +457,68 @@ TEST_CASE("converter test ss:nx restriction (none except) with exceptions") { } } -TEST_CASE("converter test ss:ir restriction (in range)") { +TEST_CASE_TEMPLATE("converter test ss:ir restriction (in range)", T, int, + ss::uint8) { ss::converter c; - c.convert>("3"); + c.convert>("3"); REQUIRE_FALSE(c.valid()); - c.convert>("c,3"); + c.convert>("c,3"); REQUIRE_FALSE(c.valid()); - c.convert, char>("3,c"); + c.convert, char>("3,c"); REQUIRE_FALSE(c.valid()); { - auto tup = c.convert>("3"); + auto tup = c.convert>("3"); REQUIRE(c.valid()); CHECK_EQ(tup, 3); } { - auto tup = c.convert>("2"); + auto tup = c.convert>("2"); REQUIRE(c.valid()); CHECK_EQ(tup, 2); } { - auto tup = c.convert>("c,junk,1"); + auto tup = c.convert>("c,junk,1"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple('c', 1)); } { - auto tup = c.convert, char>("1,c"); + auto tup = c.convert, char>("1,c"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(1, 'c')); } } -TEST_CASE("converter test ss:ir restriction (in range) with exceptions") { +TEST_CASE_TEMPLATE( + "converter test ss:ir restriction (in range) with exceptions", T, int, + ss::uint8) { ss::converter c; - REQUIRE_EXCEPTION(c.convert>("3")); - REQUIRE_EXCEPTION(c.convert>("c,3")); - REQUIRE_EXCEPTION(c.convert, char>("3,c")); + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("c,3")); + REQUIRE_EXCEPTION(c.convert, char>("3,c")); try { { - auto tup = c.convert>("3"); + auto tup = c.convert>("3"); REQUIRE(c.valid()); CHECK_EQ(tup, 3); } { - auto tup = c.convert>("2"); + auto tup = c.convert>("2"); REQUIRE(c.valid()); CHECK_EQ(tup, 2); } { - auto tup = c.convert>("c,junk,1"); + auto tup = c.convert>("c,junk,1"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple('c', 1)); } { - auto tup = c.convert, char>("1,c"); + auto tup = c.convert, char>("1,c"); REQUIRE(c.valid()); CHECK_EQ(tup, std::make_tuple(1, 'c')); } @@ -978,4 +982,3 @@ TEST_CASE("converter test invalid split conversions with exceptions") { buff(R"(just,some,2,"strings\")"))); CHECK(c.unterminated_quote()); } - diff --git a/test/test_extractions.cpp b/test/test_extractions.cpp index 4cab853..88ec317 100644 --- a/test/test_extractions.cpp +++ b/test/test_extractions.cpp @@ -2,6 +2,32 @@ #include #include +namespace { + +template +struct numeric_limits : public std::numeric_limits {}; + +template +struct numeric_limits> : public std::numeric_limits { +}; + +template +struct is_signed : public std::is_signed {}; + +template <> +struct is_signed : public std::true_type {}; + +template +struct is_unsigned : public std::is_unsigned {}; + +template <> +struct is_unsigned : public std::true_type {}; + +} /* namespace */ + +static_assert(is_signed::value); +static_assert(is_unsigned::value); + TEST_CASE("testing extract functions for floating point values") { CHECK_FLOATING_CONVERSION(123.456, float); CHECK_FLOATING_CONVERSION(123.456, double); @@ -22,18 +48,18 @@ TEST_CASE("testing extract functions for floating point values") { #define CHECK_DECIMAL_CONVERSION(input, type) \ { \ std::string s = #input; \ - auto t = ss::to_num(s.c_str(), s.c_str() + s.size()); \ - REQUIRE(t.has_value()); \ - CHECK_EQ(t.value(), type(input)); \ + type value; \ + bool valid = ss::extract(s.c_str(), s.c_str() + s.size(), value); \ + REQUIRE(valid); \ + CHECK_EQ(value, type(input)); \ } \ - { \ - /* check negative too */ \ - if (std::is_signed_v) { \ - auto s = std::string("-") + #input; \ - auto t = ss::to_num(s.c_str(), s.c_str() + s.size()); \ - REQUIRE(t.has_value()); \ - CHECK_EQ(t.value(), type(-input)); \ - } \ + /* check negative too */ \ + if (is_signed::value) { \ + std::string s = std::string("-") + #input; \ + type value; \ + bool valid = ss::extract(s.c_str(), s.c_str() + s.size(), value); \ + REQUIRE(valid); \ + CHECK_EQ(value, type(-input)); \ } using us = unsigned short; @@ -43,6 +69,8 @@ using ll = long long; using ull = unsigned long long; TEST_CASE("extract test functions for decimal values") { + CHECK_DECIMAL_CONVERSION(12, ss::int8); + CHECK_DECIMAL_CONVERSION(12, ss::uint8); CHECK_DECIMAL_CONVERSION(1234, short); CHECK_DECIMAL_CONVERSION(1234, us); CHECK_DECIMAL_CONVERSION(1234, int); @@ -54,6 +82,9 @@ TEST_CASE("extract test functions for decimal values") { } TEST_CASE("extract test functions for numbers with invalid inputs") { + // negative unsigned value for numeric_wrapper + CHECK_INVALID_CONVERSION("-12", ss::uint8); + // negative unsigned value CHECK_INVALID_CONVERSION("-1234", ul); @@ -70,46 +101,38 @@ TEST_CASE("extract test functions for numbers with invalid inputs") { CHECK_INVALID_CONVERSION("", int); } -#define CHECK_OUT_OF_RANGE_CONVERSION(type) \ - { \ - std::string s = std::to_string(std::numeric_limits::max()); \ - auto t = ss::to_num(s.c_str(), s.c_str() + s.size()); \ - CHECK(t.has_value()); \ - for (auto& i : s) { \ - if (i != '9' && i != '.') { \ - i = '9'; \ - break; \ - } \ - } \ - t = ss::to_num(s.c_str(), s.c_str() + s.size()); \ - CHECK_FALSE(t.has_value()); \ - } \ - { \ - std::string s = std::to_string(std::numeric_limits::min()); \ - auto t = ss::to_num(s.c_str(), s.c_str() + s.size()); \ - CHECK(t.has_value()); \ - for (auto& i : s) { \ - if (std::is_signed_v && i != '9' && i != '.') { \ - i = '9'; \ - break; \ - } else if (std::is_unsigned_v) { \ - s = "-1"; \ - break; \ - } \ - } \ - t = ss::to_num(s.c_str(), s.c_str() + s.size()); \ - CHECK_FALSE(t.has_value()); \ +TEST_CASE_TEMPLATE( + "extract test functions for numbers with out of range inputs", T, short, us, + int, ui, long, ul, ll, ull, ss::uint8) { + { + std::string s = std::to_string(numeric_limits::max()); + auto t = ss::to_num(s.c_str(), s.c_str() + s.size()); + CHECK(t.has_value()); + for (auto& i : s) { + if (i != '9' && i != '.') { + i = '9'; + break; + } + } + t = ss::to_num(s.c_str(), s.c_str() + s.size()); + CHECK_FALSE(t.has_value()); + } + { + std::string s = std::to_string(numeric_limits::min()); + auto t = ss::to_num(s.c_str(), s.c_str() + s.size()); + CHECK(t.has_value()); + for (auto& i : s) { + if (is_signed::value && i != '9' && i != '.') { + i = '9'; + break; + } else if (is_unsigned::value) { + s = "-1"; + break; + } + } + t = ss::to_num(s.c_str(), s.c_str() + s.size()); + CHECK_FALSE(t.has_value()); } - -TEST_CASE("extract test functions for numbers with out of range inputs") { - CHECK_OUT_OF_RANGE_CONVERSION(short); - CHECK_OUT_OF_RANGE_CONVERSION(us); - CHECK_OUT_OF_RANGE_CONVERSION(int); - CHECK_OUT_OF_RANGE_CONVERSION(ui); - CHECK_OUT_OF_RANGE_CONVERSION(long); - CHECK_OUT_OF_RANGE_CONVERSION(ul); - CHECK_OUT_OF_RANGE_CONVERSION(ll); - CHECK_OUT_OF_RANGE_CONVERSION(ull); } TEST_CASE("extract test functions for boolean values") { @@ -142,12 +165,12 @@ TEST_CASE("extract test functions for char values") { } } -TEST_CASE("extract test functions for std::optional") { - for (const auto& [i, s] : - {std::pair, std::string>{1, "1"}, - {69, "69"}, - {-4, "-4"}}) { - std::optional v; +TEST_CASE_TEMPLATE("extract test functions for std::optional", T, int, + ss::int8) { + for (const auto& [i, s] : {std::pair, std::string>{1, "1"}, + {69, "69"}, + {-4, "-4"}}) { + std::optional v; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), v)); REQUIRE(v.has_value()); CHECK_EQ(*v, i); @@ -164,7 +187,7 @@ TEST_CASE("extract test functions for std::optional") { } for (const std::string s : {"aa", "xxx", ""}) { - std::optional v; + std::optional v; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), v)); CHECK_FALSE(v.has_value()); } @@ -176,56 +199,57 @@ TEST_CASE("extract test functions for std::optional") { } } -TEST_CASE("extract test functions for std::variant") { +TEST_CASE_TEMPLATE("extract test functions for std::variant", T, int, + ss::uint8) { { std::string s = "22"; { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); CHECK_NOT_VARIANT(var, double); CHECK_NOT_VARIANT(var, std::string); - REQUIRE_VARIANT(var, 22, int); + REQUIRE_VARIANT(var, 22, T); } { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, std::string); REQUIRE_VARIANT(var, 22, double); } { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, double); REQUIRE_VARIANT(var, "22", std::string); } { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - REQUIRE_VARIANT(var, 22, int); + REQUIRE_VARIANT(var, 22, T); } } { std::string s = "22.2"; { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, std::string); REQUIRE_VARIANT(var, 22.2, double); } { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, std::string); REQUIRE_VARIANT(var, 22.2, double); } { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, double); REQUIRE_VARIANT(var, "22.2", std::string); } @@ -233,45 +257,45 @@ TEST_CASE("extract test functions for std::variant") { { std::string s = "2.2.2"; { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, double); REQUIRE_VARIANT(var, "2.2.2", std::string); } { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, double); REQUIRE_VARIANT(var, "2.2.2", std::string); } { - std::variant var; + std::variant var; REQUIRE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); CHECK_NOT_VARIANT(var, double); REQUIRE_VARIANT(var, "2.2.2", std::string); } { - std::variant var; + std::variant var; REQUIRE_FALSE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - REQUIRE_VARIANT(var, int{}, int); + REQUIRE_VARIANT(var, T{}, T); CHECK_NOT_VARIANT(var, double); } { - std::variant var; + std::variant var; REQUIRE_FALSE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); REQUIRE_VARIANT(var, double{}, double); - CHECK_NOT_VARIANT(var, int); + CHECK_NOT_VARIANT(var, T); } { - std::variant var; + std::variant var; REQUIRE_FALSE(ss::extract(s.c_str(), s.c_str() + s.size(), var)); - REQUIRE_VARIANT(var, int{}, int); + REQUIRE_VARIANT(var, T{}, T); } } } diff --git a/test/test_helpers.hpp b/test/test_helpers.hpp index 15d20f6..188e731 100644 --- a/test/test_helpers.hpp +++ b/test/test_helpers.hpp @@ -1,7 +1,11 @@ #pragma once +#include #include #include +#include #include +#include +#include #include #include #include @@ -12,7 +16,30 @@ #include #endif +namespace ss { +template +class parser; +} /* ss */ + namespace { + +struct bool_error {}; + +template +struct config { + using BufferMode = T; + using ErrorMode = U; + + constexpr static auto ThrowOnError = std::is_same_v; + constexpr static auto StringError = std::is_same_v; +}; + +#define ParserOptionCombinations \ + config, config, \ + config, config, \ + config, \ + config + struct buffer { std::string data_; @@ -34,27 +61,34 @@ struct buffer { [[maybe_unused]] inline buffer buff; -std::string time_now_rand() { +[[maybe_unused]] std::string time_now_rand() { + std::srand(std::time(nullptr)); std::stringstream ss; auto t = std::time(nullptr); auto tm = *std::localtime(&t); ss << std::put_time(&tm, "%d%m%Y%H%M%S"); - srand(time(nullptr)); + std::srand(std::time(nullptr)); return ss.str() + std::to_string(rand()); } struct unique_file_name { static inline int i = 0; - const std::string name; + std::string name; - unique_file_name(const std::string& test) - : name{"random_" + test + "_" + std::to_string(i++) + "_" + - time_now_rand() + "_file.csv"} { + unique_file_name(const std::string& test) { + do { + name = "ssp_test_" + test + "_" + std::to_string(i++) + "_" + + time_now_rand() + "_file.csv"; + } while (std::filesystem::exists(name)); } ~unique_file_name() { - std::filesystem::remove(name); + try { + std::filesystem::remove(name); + } catch (const std::filesystem::filesystem_error& e) { + std::cerr << e.what() << std::endl; + } } }; @@ -112,8 +146,8 @@ struct unique_file_name { } template -std::vector> vector_combinations(const std::vector& v, - size_t n) { +[[maybe_unused]] std::vector> vector_combinations( + const std::vector& v, size_t n) { std::vector> ret; if (n <= 1) { for (const auto& i : v) { @@ -126,9 +160,68 @@ std::vector> vector_combinations(const std::vector& v, for (const auto& i : v) { for (auto j : inner_combinations) { j.insert(j.begin(), i); - ret.push_back(move(j)); + ret.push_back(std::move(j)); } } return ret; } + +[[maybe_unused]] std::string make_buffer(const std::string& file_name) { + std::ifstream in{file_name, std::ios::binary}; + std::string tmp; + std::string out; + + auto copy_if_whitespaces = [&] { + std::string matches = "\n\r\t "; + while (std::any_of(matches.begin(), matches.end(), + [&](auto c) { return in.peek() == c; })) { + if (in.peek() == '\r') { + out += "\r\n"; + in.ignore(2); + } else { + out += std::string{static_cast(in.peek())}; + in.ignore(1); + } + } + }; + + out.reserve(sizeof(out) + 1); + + copy_if_whitespaces(); + while (in >> tmp) { + out += tmp; + copy_if_whitespaces(); + } + return out; +} + +template +std::tuple, std::string> make_parser_impl( + const std::string& file_name, std::string delim = ss::default_delimiter) { + if (buffer_mode) { + auto buffer = make_buffer(file_name); + return {ss::parser{buffer.data(), buffer.size(), delim}, + std::move(buffer)}; + } else { + return {ss::parser{file_name, delim}, std::string{}}; + } +} + +template +[[maybe_unused]] std::enable_if_t< + !std::is_same_v, + std::tuple, std::string>> +make_parser(const std::string& file_name, + std::string delim = ss::default_delimiter) { + return make_parser_impl(file_name, delim); +} + +template +[[maybe_unused]] std::enable_if_t, + std::tuple, std::string>> +make_parser(const std::string& file_name, + std::string delim = ss::default_delimiter) { + return make_parser_impl(file_name, delim); +} + } /* namespace */ diff --git a/test/test_parser.cpp b/test/test_parser.cpp deleted file mode 100644 index e7089ff..0000000 --- a/test/test_parser.cpp +++ /dev/null @@ -1,1682 +0,0 @@ -#include "test_helpers.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { -[[maybe_unused]] void replace_all(std::string& s, const std::string& from, - const std::string& to) { - if (from.empty()) return; - size_t start_pos = 0; - while ((start_pos = s.find(from, start_pos)) != std::string::npos) { - s.replace(start_pos, from.length(), to); - start_pos += to.length(); - } -} - -template -void expect_error_on_command(ss::parser& p, - const std::function command) { - if (ss::setup::throw_on_error) { - try { - command(); - } catch (const std::exception& e) { - CHECK_FALSE(std::string{e.what()}.empty()); - } - } else { - command(); - CHECK(!p.valid()); - if constexpr (ss::setup::string_error) { - CHECK_FALSE(p.error_msg().empty()); - } - } -} - -void update_if_crlf(std::string& s) { -#ifdef _WIN32 - replace_all(s, "\r\n", "\n"); -#else - (void)(s); -#endif -} - -struct X { - constexpr static auto delim = ","; - constexpr static auto empty = "_EMPTY_"; - int i; - double d; - std::string s; - - std::string to_string() const { - if (s == empty) { - return ""; - } - - return std::to_string(i) - .append(delim) - .append(std::to_string(d)) - .append(delim) - .append(s); - } - auto tied() const { - return std::tie(i, d, s); - } -}; - -template -std::enable_if_t, bool> operator==(const T& lhs, - const T& rhs) { - return lhs.tied() == rhs.tied(); -} - -template -static void make_and_write(const std::string& file_name, - const std::vector& data, - const std::vector& header = {}) { - std::ofstream out{file_name}; - -#ifdef _WIN32 - std::vector new_lines = {"\n"}; -#else - std::vector new_lines = {"\n", "\r\n"}; -#endif - - for (const auto& i : header) { - if (&i != &header.front()) { - out << T::delim; - } - out << i; - } - - if (!header.empty()) { - out << new_lines.front(); - } - - for (size_t i = 0; i < data.size(); ++i) { - out << data[i].to_string() << new_lines[i % new_lines.size()]; - } -} -} /* namespace */ - -TEST_CASE("test file not found") { - unique_file_name f{"test_parser"}; - - { - ss::parser p{f.name, ","}; - CHECK_FALSE(p.valid()); - } - - { - ss::parser p{f.name, ","}; - CHECK_FALSE(p.valid()); - } - - try { - ss::parser p{f.name, ","}; - FAIL("Expected exception..."); - } catch (const std::exception& e) { - CHECK_FALSE(std::string{e.what()}.empty()); - } -} - -template -void test_various_cases() { - unique_file_name f{"test_parser"}; - std::vector data = {{1, 2, "x"}, {3, 4, "y"}, {5, 6, "z"}, - {7, 8, "u"}, {9, 10, "v"}, {11, 12, "w"}}; - make_and_write(f.name, data); - { - ss::parser p{f.name, ","}; - ss::parser p0{std::move(p)}; - p = std::move(p0); - std::vector i; - - ss::parser p2{f.name, ","}; - std::vector i2; - - auto move_rotate = [&] { - auto p1 = std::move(p); - p0 = std::move(p1); - p = std::move(p0); - }; - - while (!p.eof()) { - move_rotate(); - auto a = p.template get_next(); - i.emplace_back(ss::to_object(a)); - } - - for (const auto& a : p2.iterate()) { - i2.emplace_back(ss::to_object(a)); - } - - CHECK_EQ(i, data); - CHECK_EQ(i2, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - ss::parser p2{f.name, ","}; - std::vector i2; - - ss::parser p3{f.name, ","}; - std::vector i3; - - std::vector expected = {std::begin(data) + 1, std::end(data)}; - using tup = std::tuple; - - p.ignore_next(); - while (!p.eof()) { - auto a = p.get_next(); - i.emplace_back(ss::to_object(a)); - } - - p2.ignore_next(); - for (const auto& a : p2.iterate()) { - i2.emplace_back(ss::to_object(a)); - } - - p3.ignore_next(); - for (auto it = p3.iterate().begin(); it != p3.iterate().end(); - ++it) { - i3.emplace_back(ss::to_object(*it)); - } - - CHECK_EQ(i, expected); - CHECK_EQ(i2, expected); - CHECK_EQ(i3, expected); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - ss::parser p2{f.name, ","}; - std::vector i2; - - while (!p.eof()) { - i.push_back(p.get_object()); - } - - for (auto&& a : p2.iterate_object()) { - i2.push_back(std::move(a)); - } - - CHECK_EQ(i, data); - CHECK_EQ(i2, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - for (auto&& a : p.iterate_object()) { - i.push_back(std::move(a)); - } - - CHECK_EQ(i, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - ss::parser p2{f.name, ","}; - std::vector i2; - - using tup = std::tuple; - while (!p.eof()) { - i.push_back(p.get_object()); - } - - for (auto it = p2.iterate_object().begin(); - it != p2.iterate_object().end(); it++) { - i2.push_back({it->i, it->d, it->s}); - } - - CHECK_EQ(i, data); - CHECK_EQ(i2, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - using tup = std::tuple; - for (auto&& a : p.iterate_object()) { - i.push_back(std::move(a)); - } - - CHECK_EQ(i, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - while (!p.eof()) { - i.push_back(p.get_next()); - } - - CHECK_EQ(i, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - for (auto&& a : p.iterate()) { - i.push_back(std::move(a)); - } - - CHECK_EQ(i, data); - } - - { - constexpr int excluded = 3; - ss::parser p{f.name, ","}; - std::vector i; - - ss::parser p2{f.name, ","}; - std::vector i2; - - while (!p.eof()) { - auto a = - p.get_object, double, std::string>(); - if (p.valid()) { - i.push_back(a); - } - } - - for (auto&& a : p2.iterate_object, double, - std::string>()) { - if (p2.valid()) { - i2.push_back(std::move(a)); - } - } - - std::vector expected; - for (auto& x : data) { - if (x.i != excluded) { - expected.push_back(x); - } - } - - std::copy_if(data.begin(), data.end(), expected.begin(), - [&](const X& x) { return x.i != excluded; }); - CHECK_EQ(i, expected); - CHECK_EQ(i2, expected); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - ss::parser p2{f.name, ","}; - std::vector i2; - - while (!p.eof()) { - auto a = p.get_object, double, std::string>(); - if (p.valid()) { - i.push_back(a); - } - } - - for (auto&& a : - p2.iterate_object, double, std::string>()) { - if (p2.valid()) { - i2.push_back(std::move(a)); - } - } - - std::vector expected = {{3, 4, "y"}}; - CHECK_EQ(i, expected); - CHECK_EQ(i2, expected); - } - - { - unique_file_name empty_f{"test_parser"}; - std::vector empty_data = {}; - - make_and_write(empty_f.name, empty_data); - - ss::parser p{empty_f.name, ","}; - std::vector i; - - ss::parser p2{empty_f.name, ","}; - std::vector i2; - - while (!p.eof()) { - i.push_back(p.get_next()); - } - - for (auto&& a : p2.iterate()) { - i2.push_back(std::move(a)); - } - - CHECK(i.empty()); - CHECK(i2.empty()); - } -} - -TEST_CASE("parser test various cases") { - test_various_cases(); - test_various_cases(); - test_various_cases(); -} - -using test_tuple = std::tuple; -struct test_struct { - int i; - double d; - char c; - auto tied() { - return std::tie(i, d, c); - } -}; - -static inline void expect_test_struct(const test_struct&) { -} - -template -void test_composite_conversion() { - unique_file_name f{"test_parser"}; - { - std::ofstream out{f.name}; - for (auto& i : - {"10,a,11.1", "10,20,11.1", "junk", "10,11.1", "1,11.1,a", "junk", - "10,junk", "11,junk", "10,11.1,c", "10,20", "10,22.2,f"}) { - out << i << std::endl; - } - } - - ss::parser p{f.name, ","}; - auto fail = [] { FAIL(""); }; - auto expect_error = [](auto error) { CHECK(!error.empty()); }; - auto ignore_error = [] {}; - - REQUIRE(p.valid()); - REQUIRE_FALSE(p.eof()); - - { - constexpr static auto expectedData = std::tuple{10, 'a', 11.1}; - - auto [d1, d2, d3, d4] = - p.template try_next(fail) - .template or_else(fail) - .template or_else( - [&](auto&& data) { CHECK_EQ(data, expectedData); }) - .on_error(fail) - .template or_else(fail) - .values(); - - REQUIRE(p.valid()); - REQUIRE_FALSE(d1); - REQUIRE_FALSE(d2); - REQUIRE(d3); - REQUIRE_FALSE(d4); - CHECK_EQ(*d3, expectedData); - } - - { - REQUIRE(!p.eof()); - constexpr static auto expectedData = std::tuple{10, 20, 11.1}; - - auto [d1, d2, d3, d4] = - p.template try_next( - [&](auto& i1, auto i2, double d) { - CHECK_EQ(std::tie(i1, i2, d), expectedData); - }) - .on_error(fail) - .template or_object(fail) - .on_error(fail) - .template or_else(fail) - .on_error(fail) - .template or_else(fail) - .values(); - - REQUIRE(p.valid()); - REQUIRE(d1); - REQUIRE_FALSE(d2); - REQUIRE_FALSE(d3); - REQUIRE_FALSE(d4); - CHECK_EQ(*d1, expectedData); - } - - { - REQUIRE(!p.eof()); - - auto [d1, d2, d3, d4, d5] = - p.template try_object(fail) - .on_error(expect_error) - .template or_else(fail) - .template or_else(fail) - .template or_else(fail) - .template or_else(fail) - .values(); - - REQUIRE_FALSE(p.valid()); - REQUIRE_FALSE(d1); - REQUIRE_FALSE(d2); - REQUIRE_FALSE(d3); - REQUIRE_FALSE(d4); - REQUIRE_FALSE(d5); - } - - { - REQUIRE(!p.eof()); - - auto [d1, d2] = - p.template try_next([](auto& i, auto& d) { - REQUIRE_EQ(std::tie(i, d), std::tuple{10, 11.1}); - }) - .template or_else([](auto&, auto&) { FAIL(""); }) - .values(); - - REQUIRE(p.valid()); - REQUIRE(d1); - REQUIRE_FALSE(d2); - } - - { - REQUIRE(!p.eof()); - - auto [d1, d2] = - p.template try_next([](auto&, auto&) { FAIL(""); }) - .template or_else(expect_test_struct) - .values(); - - REQUIRE(p.valid()); - REQUIRE_FALSE(d1); - REQUIRE(d2); - CHECK_EQ(d2->tied(), std::tuple{1, 11.1, 'a'}); - } - - { - REQUIRE(!p.eof()); - - auto [d1, d2, d3, d4, d5] = - p.template try_next(fail) - .template or_object() - .template or_else(expect_test_struct) - .template or_else(fail) - .template or_else>(fail) - .on_error(ignore_error) - .on_error(expect_error) - .values(); - - REQUIRE_FALSE(p.valid()); - REQUIRE_FALSE(d1); - REQUIRE_FALSE(d2); - REQUIRE_FALSE(d3); - REQUIRE_FALSE(d4); - REQUIRE_FALSE(d5); - } - - { - REQUIRE(!p.eof()); - - auto [d1, d2] = - p.template try_next>() - .on_error(ignore_error) - .on_error(fail) - .template or_else>(fail) - .on_error(ignore_error) - .on_error(fail) - .on_error(ignore_error) - .values(); - - REQUIRE(p.valid()); - REQUIRE(d1); - REQUIRE_FALSE(d2); - CHECK_EQ(*d1, std::tuple{10, std::nullopt}); - } - - { - REQUIRE_FALSE(p.eof()); - - auto [d1, d2] = - p.template try_next>() - .on_error(fail) - .template or_else>(fail) - .on_error(fail) - .values(); - - REQUIRE(p.valid()); - REQUIRE(d1); - REQUIRE_FALSE(d2); - CHECK_EQ(*d1, std::tuple{11, std::variant{"junk"}}); - } - - { - REQUIRE(!p.eof()); - - auto [d1, d2] = p.template try_object() - .template or_else(fail) - .values(); - REQUIRE(p.valid()); - REQUIRE(d1); - REQUIRE_FALSE(d2); - CHECK_EQ(d1->tied(), std::tuple{10, 11.1, 'c'}); - } - - { - REQUIRE_FALSE(p.eof()); - - auto [d1, d2, d3, d4] = - p.template try_next([] { return false; }) - .template or_else([](auto&) { return false; }) - .template or_else() - .template or_else(fail) - .values(); - - REQUIRE(p.valid()); - REQUIRE_FALSE(d1); - REQUIRE_FALSE(d2); - REQUIRE(d3); - REQUIRE_FALSE(d4); - CHECK_EQ(d3.value(), std::tuple{10, 20}); - } - - { - REQUIRE(!p.eof()); - - auto [d1, d2, d3, d4] = - p.template try_object( - [] { return false; }) - .template or_else([](auto&) { return false; }) - .template or_object() - .template or_else(fail) - .values(); - - REQUIRE(p.valid()); - REQUIRE_FALSE(d1); - REQUIRE_FALSE(d2); - REQUIRE(d3); - REQUIRE_FALSE(d4); - CHECK_EQ(d3->tied(), std::tuple{10, 22.2, 'f'}); - } - - CHECK(p.eof()); -} - -// various scenarios -TEST_CASE("parser test composite conversion") { - test_composite_conversion(); -} - -struct my_string { - char* data{nullptr}; - - my_string() = default; - - ~my_string() { - delete[] data; - } - - // make sure no object is copied - my_string(const my_string&) = delete; - my_string& operator=(const my_string&) = delete; - - my_string(my_string&& other) : data{other.data} { - other.data = nullptr; - } - - my_string& operator=(my_string&& other) { - data = other.data; - return *this; - } -}; - -template <> -inline bool ss::extract(const char* begin, const char* end, my_string& s) { - size_t size = end - begin; - s.data = new char[size + 1]; - strncpy(s.data, begin, size); - s.data[size] = '\0'; - return true; -} - -struct xyz { - my_string x; - my_string y; - my_string z; - auto tied() { - return std::tie(x, y, z); - } -}; - -template -void test_moving_of_parsed_composite_values() { - // to compile is enough - return; - ss::parser p{"", ""}; - p.template try_next() - .template or_else( - [](auto&&) {}) - .template or_else([](auto&) {}) - .template or_else([](auto&&) {}) - .template or_object([](auto&&) {}) - .template or_else>( - [](auto&, auto&, auto&) {}); -} - -TEST_CASE("parser test the moving of parsed composite values") { - test_moving_of_parsed_composite_values(); - test_moving_of_parsed_composite_values(); -} - -TEST_CASE("parser test error mode") { - unique_file_name f{"test_parser"}; - { - std::ofstream out{f.name}; - out << "junk" << std::endl; - out << "junk" << std::endl; - } - - ss::parser p(f.name, ","); - - REQUIRE_FALSE(p.eof()); - p.get_next(); - CHECK_FALSE(p.valid()); - CHECK_FALSE(p.error_msg().empty()); -} - -TEST_CASE("parser throw on error mode") { - unique_file_name f{"test_parser"}; - { - std::ofstream out{f.name}; - out << "junk" << std::endl; - out << "junk" << std::endl; - } - - ss::parser p(f.name, ","); - - REQUIRE_FALSE(p.eof()); - try { - p.get_next(); - FAIL("Expected exception..."); - } catch (const std::exception& e) { - CHECK_FALSE(std::string{e.what()}.empty()); - } -} - -static inline std::string no_quote(const std::string& s) { - if (!s.empty() && s[0] == '"') { - return {std::next(begin(s)), std::prev(end(s))}; - } - return s; -} - -template -void test_quote_multiline() { - unique_file_name f{"test_parser"}; - std::vector data = {{1, 2, "\"x\r\nx\nx\""}, - {3, 4, "\"y\ny\r\ny\""}, - {5, 6, "\"z\nz\""}, - {7, 8, "\"u\"\"\""}, - {9, 10, "v"}, - {11, 12, "\"w\n\""}}; - for (auto& [_, __, s] : data) { - update_if_crlf(s); - } - - make_and_write(f.name, data); - for (auto& [_, __, s] : data) { - s = no_quote(s); - if (s[0] == 'u') { - s = "u\""; - } - } - - ss::parser, Ts...> p{f.name, ","}; - std::vector i; - - while (!p.eof()) { - auto a = p.template get_next(); - i.emplace_back(ss::to_object(a)); - } - - for (auto& [_, __, s] : i) { - update_if_crlf(s); - } - CHECK_EQ(i, data); - - ss::parser, Ts...> p_no_multiline{f.name, ","}; - while (!p.eof()) { - auto command = [&] { - p_no_multiline.template get_next(); - }; - expect_error_on_command(p_no_multiline, command); - } -} - -TEST_CASE("parser test csv on multiple lines with quotes") { - test_quote_multiline(); - test_quote_multiline(); - test_quote_multiline(); -} - -static inline std::string no_escape(std::string& s) { - s.erase(std::remove(begin(s), end(s), '\\'), end(s)); - return s; -} - -template -void test_escape_multiline() { - unique_file_name f{"test_parser"}; - std::vector data = {{1, 2, "x\\\nx\\\r\nx"}, - {5, 6, "z\\\nz\\\nz"}, - {7, 8, "u"}, - {3, 4, "y\\\ny\\\ny"}, - {9, 10, "v\\\\"}, - {11, 12, "w\\\n"}}; - for (auto& [_, __, s] : data) { - update_if_crlf(s); - } - - make_and_write(f.name, data); - for (auto& [_, __, s] : data) { - s = no_escape(s); - if (s == "v") { - s = "v\\"; - } - } - - ss::parser, Ts...> p{f.name, ","}; - std::vector i; - - while (!p.eof()) { - auto a = p.template get_next(); - i.emplace_back(ss::to_object(a)); - } - - for (auto& [_, __, s] : i) { - update_if_crlf(s); - } - CHECK_EQ(i, data); - - ss::parser, Ts...> p_no_multiline{f.name, ","}; - while (!p.eof()) { - auto command = [&] { - auto a = - p_no_multiline.template get_next(); - }; - expect_error_on_command(p_no_multiline, command); - } -} - -TEST_CASE("parser test csv on multiple lines with escapes") { - test_escape_multiline(); - test_escape_multiline(); - test_escape_multiline(); -} - -template -void test_quote_escape_multiline() { - unique_file_name f{"test_parser"}; - { - std::ofstream out{f.name}; - out << "1,2,\"just\\\n\nstrings\"" << std::endl; -#ifndef _WIN32 - out << "3,4,\"just\r\nsome\\\r\n\n\\\nstrings\"" << std::endl; - out << "5,6,\"just\\\n\\\r\n\r\n\nstrings" << std::endl; -#else - out << "3,4,\"just\nsome\\\n\n\\\nstrings\"" << std::endl; - out << "5,6,\"just\\\n\\\n\n\nstrings" << std::endl; -#endif - out << "7,8,\"just strings\"" << std::endl; - out << "9,10,just strings" << std::endl; - } - size_t bad_lines = 1; - auto num_errors = 0; - - ss::parser, ss::quote<'"'>, Ts...> p{ - f.name}; - std::vector i; - - while (!p.eof()) { - try { - auto a = p.template get_next(); - if (p.valid()) { - i.emplace_back(ss::to_object(a)); - } else { - ++num_errors; - } - } catch (const std::exception& e) { - ++num_errors; - } - } - - CHECK(bad_lines == num_errors); - - std::vector data = {{1, 2, "just\n\nstrings"}, -#ifndef _WIN32 - {3, 4, "just\r\nsome\r\n\n\nstrings"}, -#else - {3, 4, "just\nsome\n\n\nstrings"}, -#endif - {9, 10, "just strings"}}; - - for (auto& [_, __, s] : i) { - update_if_crlf(s); - } - CHECK_EQ(i, data); -} - -TEST_CASE("parser test csv on multiple lines with quotes and escapes") { - test_quote_escape_multiline(); - test_quote_escape_multiline(); - test_quote_escape_multiline(); -} - -template -void test_multiline_restricted() { - unique_file_name f{"test_parser"}; - { - std::ofstream out{f.name}; - out << "1,2,\"just\n\nstrings\"" << std::endl; -#ifndef _WIN32 - out << "3,4,\"ju\n\r\n\nnk\"" << std::endl; - out << "5,6,just\\\n\\\r\nstrings" << std::endl; -#else - out << "3,4,\"ju\n\n\nnk\"" << std::endl; - out << "5,6,just\\\n\\\nstrings" << std::endl; -#endif - out << "7,8,ju\\\n\\\n\\\nnk" << std::endl; - out << "9,10,\"just\\\n\nstrings\"" << std::endl; - out << "11,12,\"ju\\\n|\n\n\n\n\nk\"" << std::endl; - out << "13,14,\"ju\\\n\\\n15,16\"\\\n\\\\\n\nnk\"" << std::endl; - out << "17,18,\"ju\\\n\\\n\\\n\\\\\n\nnk\"" << std::endl; - out << "19,20,just strings" << std::endl; - } - auto bad_lines = 15; - auto num_errors = 0; - - ss::parser, ss::quote<'"'>, ss::escape<'\\'>, - Ts...> - p{f.name, ","}; - std::vector i; - - while (!p.eof()) { - try { - auto a = p.template get_next(); - if (p.valid()) { - i.emplace_back(ss::to_object(a)); - } else { - ++num_errors; - } - } catch (const std::exception& e) { - ++num_errors; - } - } - - CHECK(bad_lines == num_errors); - - std::vector data = {{1, 2, "just\n\nstrings"}, -#ifndef _WIN32 - {5, 6, "just\n\r\nstrings"}, -#else - {5, 6, "just\n\nstrings"}, -#endif - {9, 10, "just\n\nstrings"}, - {19, 20, "just strings"}}; - - for (auto& [_, __, s] : i) { - update_if_crlf(s); - } - - if (i.size() != data.size()) { - CHECK_EQ(i.size(), data.size()); - } - - CHECK_EQ(i, data); -} - -TEST_CASE("parser test multiline restricted") { - test_multiline_restricted(); - test_multiline_restricted(); - test_multiline_restricted(); -} - -template -void test_unterminated_line_impl(const std::vector& lines, - size_t bad_line) { - unique_file_name f{"test_parser"}; - std::ofstream out{f.name}; - for (const auto& line : lines) { - out << line << std::endl; - } - out.close(); - - ss::parser p{f.name}; - size_t line = 0; - while (!p.eof()) { - auto command = [&] { p.template get_next(); }; - - if (line == bad_line) { - expect_error_on_command(p, command); - break; - } else { - command(); - CHECK(p.valid()); - ++line; - } - } -} - -template -void test_unterminated_line(const std::vector& lines, - size_t bad_line) { - test_unterminated_line_impl(lines, bad_line); - test_unterminated_line_impl(lines, bad_line); - test_unterminated_line_impl(lines, bad_line); -} - -TEST_CASE("parser test csv on multiline with errors") { - using multiline = ss::multiline_restricted<3>; - using escape = ss::escape<'\\'>; - using quote = ss::quote<'"'>; - - // unterminated escape - { - const std::vector lines{"1,2,just\\"}; - test_unterminated_line(lines, 0); - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"1,2,just\\", "9,8,second"}; - test_unterminated_line(lines, 0); - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"9,8,first", "1,2,just\\"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,first", "1,2,just\\", - "3,4,third"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,first", - "1,2,just\\\nstrings\\", - "3,4,th\\\nird"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,first", "3,4,second", - "1,2,just\\"}; - test_unterminated_line(lines, 2); - test_unterminated_line(lines, 2); - } - - { - const std::vector lines{"9,8,\\first", "3,4,second", - "1,2,jus\\t\\"}; - test_unterminated_line(lines, 2); - test_unterminated_line(lines, 2); - } - - // unterminated quote - { - const std::vector lines{"1,2,\"just"}; - test_unterminated_line(lines, 0); - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"1,2,\"just", "9,8,second"}; - test_unterminated_line(lines, 0); - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"9,8,first", "1,2,\"just"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,first", "1,2,\"just", - "3,4,th\\,ird"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,first", "3,4,second", - "1,2,\"just"}; - test_unterminated_line(lines, 2); - test_unterminated_line(lines, 2); - } - - { - const std::vector lines{"9,8,\"first\"", - "\"3\",4,\"sec,ond\"", - "1,2,\"ju\"\"st"}; - test_unterminated_line(lines, 2); - test_unterminated_line(lines, 2); - } - - // unterminated quote and escape - { - const std::vector lines{"1,2,\"just\\"}; - test_unterminated_line(lines, 0); - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"1,2,\"just\\\n\\"}; - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"1,2,\"just\n\\"}; - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"9,8,first", "1,2,\"just\n\\"}; - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,first", "1,2,\"just\n\\", - "4,3,thrid"}; - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,f\\\nirst", "1,2,\"just\n\\", - "4,3,thrid"}; - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,\"f\ni\nrst\"", - "1,2,\"just\n\\", "4,3,thrid"}; - test_unterminated_line(lines, 1); - } - - // multiline limmit reached escape - { - const std::vector lines{"1,2,\\\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 0); - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"9,8,first", - "1,2,\\\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,fi\\\nrs\\\nt", - "1,2,\\\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,first", - "1,2,\\\n\\\n\\\n\\\njust", - "4,3,third"}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - // multiline limmit reached quote - { - const std::vector lines{"1,2,\"\n\n\n\n\njust\""}; - test_unterminated_line(lines, 0); - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"9,8,first", - "1,2,\"\n\n\n\n\njust\""}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,\"fir\nst\"", - "1,2,\"\n\n\n\n\njust\""}; - test_unterminated_line(lines, 1); - test_unterminated_line(lines, 1); - } - - // multiline limmit reached quote and escape - { - const std::vector lines{"1,2,\"\\\n\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 0); - } - - { - const std::vector lines{"9,8,first", - "1,2,\"\\\n\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,fi\\\nrst", - "1,2,\"\\\n\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,\"fi\nrst\"", - "1,2,\"\\\n\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 1); - } - - { - const std::vector lines{"9,8,\"fi\nr\\\nst\"", - "1,2,\"\\\n\n\\\n\\\n\\\njust"}; - test_unterminated_line(lines, 1); - } -} - -template -struct has_type; - -template -struct has_type> - : std::disjunction...> {}; - -static inline void check_size(size_t size1, size_t size2) { - CHECK_EQ(size1, size2); -} - -template -static void test_fields_impl(const std::string file_name, - const std::vector& data, - const std::vector& fields) { - using CaseType = std::tuple; - - ss::parser p{file_name, ","}; - CHECK_FALSE(p.field_exists("Unknown")); - p.use_fields(fields); - std::vector i; - - for (const auto& a : p.template iterate()) { - i.push_back(a); - } - - check_size(i.size(), data.size()); - for (size_t j = 0; j < i.size(); ++j) { - if constexpr (has_type::value) { - CHECK_EQ(std::get(i[j]), data[j].i); - } - if constexpr (has_type::value) { - CHECK_EQ(std::get(i[j]), data[j].d); - } - if constexpr (has_type::value) { - CHECK_EQ(std::get(i[j]), data[j].s); - } - } -} - -template -static void test_fields(const std::string file_name, const std::vector& data, - const std::vector& fields) { - test_fields_impl, Ts...>(file_name, data, fields); - test_fields_impl, Ts...>(file_name, data, - fields); - test_fields_impl, Ts...>(file_name, data, - fields); -} - -TEST_CASE("parser test various cases with header") { - unique_file_name f{"test_parser"}; - constexpr static auto Int = "Int"; - constexpr static auto Dbl = "Double"; - constexpr static auto Str = "String"; - using str = std::string; - - std::vector header{Int, Dbl, Str}; - - std::vector data = {{1, 2, "x"}, {3, 4, "y"}, {5, 6, "z"}, - {7, 8, "u"}, {9, 10, "v"}, {11, 12, "w"}}; - - make_and_write(f.name, data, header); - const auto& o = f.name; - const auto& d = data; - - { - ss::parser p{f.name, ","}; - std::vector i; - - for (const auto& a : p.iterate()) { - i.emplace_back(ss::to_object(a)); - } - - CHECK_NE(i, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - p.ignore_next(); - for (const auto& a : p.iterate()) { - i.emplace_back(ss::to_object(a)); - } - - CHECK_EQ(i, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - for (const auto& a : p.iterate()) { - i.emplace_back(ss::to_object(a)); - } - - CHECK_EQ(i, data); - } - - { - ss::parser p{f.name, ","}; - p.use_fields(Int, Dbl, Str); - CHECK_FALSE(p.valid()); - } - - { - ss::parser p{f.name, ","}; - CHECK_FALSE(p.field_exists("Unknown")); - - p.use_fields(Int, "Unknown"); - CHECK_FALSE(p.valid()); - } - - { - ss::parser p{f.name, ","}; - p.use_fields(Int, Int); - CHECK_FALSE(p.valid()); - } - - { - ss::parser p{f.name, ","}; - p.use_fields(Int, Dbl); - - { - auto [int_, double_] = p.get_next(); - CHECK_EQ(int_, data[0].i); - CHECK_EQ(double_, data[0].d); - } - - p.use_fields(Dbl, Int); - - { - auto [double_, int_] = p.get_next(); - CHECK_EQ(int_, data[1].i); - CHECK_EQ(double_, data[1].d); - } - - p.use_fields(Str); - - { - auto string_ = p.get_next(); - CHECK_EQ(string_, data[2].s); - } - - p.use_fields(Str, Int, Dbl); - - { - auto [string_, int_, double_] = - p.get_next(); - CHECK_EQ(double_, data[3].d); - CHECK_EQ(int_, data[3].i); - CHECK_EQ(string_, data[3].s); - } - } - - /* python used to generate permutations - import itertools - - header = {'str': 'Str', - 'double': 'Dbl', - 'int': 'Int'} - - keys = ['str', 'int', 'double'] - - for r in range (1, 3): - combinations = list(itertools.permutations(keys, r = r)) - - for combination in combinations: - template_params = [] - arg_params = [] - for type in combination: - template_params.append(type) - arg_params.append(header[type]) - call = 'testFields<' + ', '.join(template_params) + \ - '>(o, d, {' + ', '.join(arg_params) + '});' - print(call) - */ - - test_fields(o, d, {Str}); - test_fields(o, d, {Int}); - test_fields(o, d, {Dbl}); - test_fields(o, d, {Str, Int}); - test_fields(o, d, {Str, Dbl}); - test_fields(o, d, {Int, Str}); - test_fields(o, d, {Int, Dbl}); - test_fields(o, d, {Dbl, Str}); - test_fields(o, d, {Dbl, Int}); - test_fields(o, d, {Str, Int, Dbl}); - test_fields(o, d, {Str, Dbl, Int}); - test_fields(o, d, {Int, Str, Dbl}); - test_fields(o, d, {Int, Dbl, Str}); - test_fields(o, d, {Dbl, Str, Int}); - test_fields(o, d, {Dbl, Int, Str}); -} - -template -void test_invalid_fields_impl(const std::vector& lines, - const std::vector& fields) { - unique_file_name f{"test_parser"}; - std::ofstream out{f.name}; - for (const auto& line : lines) { - out << line << std::endl; - } - out.close(); - - { - // No fields specified - ss::parser p{f.name, ","}; - auto command = [&] { p.use_fields(); }; - expect_error_on_command(p, command); - } - - { - // Unknown field - ss::parser p{f.name, ","}; - auto command = [&] { p.use_fields("Unknown"); }; - expect_error_on_command(p, command); - } - - { - // Field used multiple times - ss::parser p{f.name, ","}; - auto command = [&] { p.use_fields(fields.at(0), fields.at(0)); }; - if (!fields.empty()) { - expect_error_on_command(p, command); - } - } - - { - // Mapping out of range - ss::parser p{f.name, ","}; - auto command = [&] { - p.use_fields(fields.at(0)); - p.template get_next(); - }; - if (!fields.empty()) { - expect_error_on_command(p, command); - } - } - - { - // Invalid header - ss::parser p{f.name, ","}; - auto command = [&] { p.use_fields(fields); }; - - if (!fields.empty()) { - // Pass if there are no duplicates, fail otherwise - if (std::unordered_set{fields.begin(), fields.end()} - .size() != fields.size()) { - expect_error_on_command(p, command); - } else { - command(); - CHECK(p.valid()); - if (!p.valid()) { - if constexpr (ss::setup::string_error) { - std::cout << p.error_msg() << std::endl; - } - } - } - } - } -} - -template -void test_invalid_fields(const std::vector& lines, - const std::vector& fields) { - test_invalid_fields_impl(lines, fields); - test_invalid_fields_impl(lines, fields); - test_invalid_fields_impl(lines, fields); -} - -TEST_CASE("parser test invalid header fields usage") { - test_invalid_fields({}, {}); - - test_invalid_fields({"Int"}, {"Int"}); - test_invalid_fields({"Int", "1"}, {"Int"}); - test_invalid_fields({"Int", "1", "2"}, {"Int"}); - - test_invalid_fields({"Int,String"}, {"Int", "String"}); - test_invalid_fields({"Int,String", "1,hi"}, {"Int", "String"}); - test_invalid_fields({"Int,String", "2,hello"}, {"Int", "String"}); - - test_invalid_fields({"Int,String,Double"}, {"Int", "String", "Double"}); - test_invalid_fields({"Int,String,Double", "1,hi,2.34"}, - {"Int", "String", "Double"}); - test_invalid_fields({"Int,String,Double", "1,hi,2.34", "2,hello,3.45"}, - {"Int", "String", "Double"}); - - test_invalid_fields({"Int,Int,Int"}, {"Int", "Int", "Int"}); - test_invalid_fields({"Int,Int,Int", "1,2,3"}, {"Int", "Int", "Int"}); - - test_invalid_fields({"Int,String,Int"}, {"Int", "String", "Int"}); - test_invalid_fields({"Int,String,Int", "1,hi,3"}, {"Int", "String", "Int"}); -} - -template -void test_invalid_rows_with_header() { - unique_file_name f{"test_parser"}; - { - std::ofstream out{f.name}; - out << "Int,String,Double" << std::endl; - out << "1,line1,2.34" << std::endl; - out << "2,line2" << std::endl; - out << "3,line3,67.8" << std::endl; - out << "4,line4,67.8,9" << std::endl; - out << "5,line5,9.10" << std::endl; - out << "six,line6,10.11" << std::endl; - } - - { - ss::parser p{f.name}; - - p.use_fields("Int", "String", "Double"); - using data = std::tuple; - std::vector i; - - CHECK(p.valid()); - - while (!p.eof()) { - try { - const auto& t = p.template get_next(); - if (p.valid()) { - i.push_back(t); - } - } catch (const ss::exception&) { - continue; - } - } - - std::vector expected = {{1, "line1", 2.34}, - {3, "line3", 67.8}, - {5, "line5", 9.10}}; - CHECK_EQ(i, expected); - } - - { - ss::parser p{f.name}; - - p.use_fields("Double", "Int"); - using data = std::tuple; - std::vector i; - - CHECK(p.valid()); - - while (!p.eof()) { - try { - const auto& t = p.template get_next(); - if (p.valid()) { - i.push_back(t); - } - } catch (const ss::exception&) { - continue; - } - } - - std::vector expected = {{2.34, 1}, {67.8, 3}, {9.10, 5}}; - CHECK_EQ(i, expected); - } - - { - ss::parser p{f.name}; - - p.use_fields("String", "Double"); - using data = std::tuple; - std::vector i; - - CHECK(p.valid()); - - while (!p.eof()) { - try { - const auto& t = p.template get_next(); - if (p.valid()) { - i.push_back(t); - } - } catch (const ss::exception&) { - continue; - } - } - - std::vector expected = {{"line1", 2.34}, - {"line3", 67.8}, - {"line5", 9.10}, - {"line6", 10.11}}; - CHECK_EQ(i, expected); - } -} - -TEST_CASE("parser test invalid rows with header") { - test_invalid_rows_with_header(); - test_invalid_rows_with_header(); - test_invalid_rows_with_header(); -} - -template -void test_ignore_empty_impl(const std::vector& data) { - unique_file_name f{"test_parser"}; - make_and_write(f.name, data); - - std::vector expected; - for (const auto& d : data) { - if (d.s != X::empty) { - expected.push_back(d); - } - } - - { - ss::parser p{f.name, ","}; - - std::vector i; - for (const auto& a : p.template iterate()) { - i.push_back(a); - } - - CHECK_EQ(i, expected); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - size_t n = 0; - while (!p.eof()) { - try { - ++n; - const auto& a = p.template get_next(); - if (data.at(n - 1).s == X::empty) { - CHECK_FALSE(p.valid()); - continue; - } - i.push_back(a); - } catch (...) { - CHECK_EQ(data.at(n - 1).s, X::empty); - } - } - - CHECK_EQ(i, expected); - } -} - -template -void test_ignore_empty(const std::vector& data) { - test_ignore_empty_impl(data); - test_ignore_empty_impl(data); - test_ignore_empty_impl(data); -} - -TEST_CASE("parser test various cases with empty lines") { - test_ignore_empty({{1, 2, "x"}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, "w"}}); - - test_ignore_empty( - {{1, 2, X::empty}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, "w"}}); - - test_ignore_empty( - {{1, 2, "x"}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, X::empty}}); - - test_ignore_empty( - {{1, 2, "x"}, {5, 6, X::empty}, {9, 10, "v"}, {11, 12, "w"}}); - - test_ignore_empty( - {{1, 2, X::empty}, {5, 6, X::empty}, {9, 10, "v"}, {11, 12, "w"}}); - - test_ignore_empty( - {{1, 2, X::empty}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, X::empty}}); - - test_ignore_empty( - {{1, 2, "x"}, {3, 4, "y"}, {9, 10, X::empty}, {11, 12, X::empty}}); - - test_ignore_empty( - {{1, 2, X::empty}, {3, 4, "y"}, {9, 10, X::empty}, {11, 12, X::empty}}); - - test_ignore_empty({{1, 2, X::empty}, - {3, 4, X::empty}, - {9, 10, X::empty}, - {11, 12, X::empty}}); - - test_ignore_empty( - {{1, 2, "x"}, {3, 4, X::empty}, {9, 10, X::empty}, {11, 12, X::empty}}); - - test_ignore_empty( - {{1, 2, X::empty}, {3, 4, X::empty}, {9, 10, X::empty}, {11, 12, "w"}}); - - test_ignore_empty({{11, 12, X::empty}}); - - test_ignore_empty({}); -} diff --git a/test/test_parser1.hpp b/test/test_parser1.hpp new file mode 100644 index 0000000..90267c8 --- /dev/null +++ b/test/test_parser1.hpp @@ -0,0 +1,112 @@ +#pragma once + +#include "test_helpers.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +[[maybe_unused]] void replace_all(std::string& s, const std::string& from, + const std::string& to) { + if (from.empty()) return; + size_t start_pos = 0; + while ((start_pos = s.find(from, start_pos)) != std::string::npos) { + s.replace(start_pos, from.length(), to); + start_pos += to.length(); + } +} + +template +void expect_error_on_command(ss::parser& p, + const std::function command) { + if (ss::setup::throw_on_error) { + try { + command(); + } catch (const std::exception& e) { + CHECK_FALSE(std::string{e.what()}.empty()); + } + } else { + command(); + CHECK(!p.valid()); + if constexpr (ss::setup::string_error) { + CHECK_FALSE(p.error_msg().empty()); + } + } +} + +[[maybe_unused]] void update_if_crlf(std::string& s) { +#ifdef _WIN32 + replace_all(s, "\r\n", "\n"); +#else + (void)(s); +#endif +} + +struct X { + constexpr static auto delim = ","; + constexpr static auto empty = "_EMPTY_"; + int i; + double d; + std::string s; + + std::string to_string() const { + if (s == empty) { + return ""; + } + + return std::to_string(i) + .append(delim) + .append(std::to_string(d)) + .append(delim) + .append(s); + } + auto tied() const { + return std::tie(i, d, s); + } +}; + +template +std::enable_if_t, bool> operator==(const T& lhs, + const T& rhs) { + return lhs.tied() == rhs.tied(); +} + +template +static void make_and_write(const std::string& file_name, + const std::vector& data, + const std::vector& header = {}, + bool new_line_eof = true) { + std::ofstream out{file_name}; + +#ifdef _WIN32 + std::vector new_lines = {"\n"}; +#else + std::vector new_lines = {"\n", "\r\n"}; +#endif + + for (const auto& i : header) { + if (&i != &header.front()) { + out << T::delim; + } + out << i; + } + + if (!header.empty()) { + out << new_lines.front(); + } + + for (size_t i = 0; i < data.size(); ++i) { + out << data[i].to_string(); + if (new_line_eof || i + 1 < data.size()) { + out << new_lines[i % new_lines.size()]; + } + } +} + +} /* namespace */ diff --git a/test/test_parser1_1.cpp b/test/test_parser1_1.cpp new file mode 100644 index 0000000..650393b --- /dev/null +++ b/test/test_parser1_1.cpp @@ -0,0 +1,674 @@ +#include "test_parser1.hpp" + +TEST_CASE("test file not found") { + unique_file_name f{"file_not_found"}; + + { + ss::parser p{f.name, ","}; + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{f.name, ","}; + CHECK_FALSE(p.valid()); + CHECK_FALSE(p.error_msg().empty()); + } + + try { + ss::parser p{f.name, ","}; + FAIL("Expected exception..."); + } catch (const std::exception& e) { + CHECK_FALSE(std::string{e.what()}.empty()); + } +} + +TEST_CASE("test null buffer") { + { + ss::parser p{nullptr, 10, ","}; + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{nullptr, 10, ","}; + CHECK_FALSE(p.valid()); + CHECK_FALSE(p.error_msg().empty()); + } + + try { + ss::parser p{nullptr, 10, ","}; + FAIL("Expected exception..."); + } catch (const std::exception& e) { + CHECK_FALSE(std::string{e.what()}.empty()); + } +} + +struct Y { + constexpr static auto delim = ","; + std::string s1; + std::string s2; + std::string s3; + + std::string to_string() const { + return std::string{} + .append(s1) + .append(delim) + .append(s2) + .append(delim) + .append(s3); + } + + auto tied() const { + return std::tie(s1, s2, s3); + } +}; + +TEST_CASE_TEMPLATE("test position method", T, ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"position_method"}; + std::vector data = {{"1", "21", "x"}, {"321", "4", "y"}, + {"54", "6", "zz"}, {"7", "876", "uuuu"}, + {"910", "10", "v"}, {"10", "321", "ww"}}; + make_and_write(f.name, data); + + auto [p, buff] = make_parser(f.name); + auto data_at = [&buff = buff, &f = f](auto n) { + if (!buff.empty()) { + return buff[n]; + } else { + auto file = std::fopen(f.name.c_str(), "r"); + std::fseek(file, n, SEEK_SET); + return static_cast(std::fgetc(file)); + } + }; + + while (!p.eof()) { + auto curr_char = p.position(); + const auto& [s1, s2, s3] = + p.template get_next(); + + auto s = s1 + "," + s2 + "," + s3; + + for (size_t i = 0; i < s1.size(); ++i) { + CHECK_EQ(data_at(curr_char + i), s[i]); + } + + auto last_char = data_at(curr_char + s.size()); + CHECK((last_char == '\n' || last_char == '\r')); + } +} + +TEST_CASE_TEMPLATE("test line method", T, ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"line_method"}; + std::vector data = {{"1", "21", "x"}, {"321", "4", "y"}, + {"54", "6", "zz"}, {"7", "876", "uuuu"}, + {"910", "10", "v"}, {"10", "321", "ww"}}; + make_and_write(f.name, data); + + auto [p, buff] = make_parser(f.name); + + auto expected_line = 0; + CHECK_EQ(p.line(), expected_line); + + while (!p.eof()) { + auto _ = p.template get_next(); + ++expected_line; + CHECK_EQ(p.line(), expected_line); + } + + CHECK_EQ(p.line(), data.size()); +} + +TEST_CASE_TEMPLATE("parser test various valid cases", T, + ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"various_valid_cases"}; + std::vector data = {{1, 2, "x"}, {3, 4, "y"}, {5, 6, "z"}, + {7, 8, "u"}, {9, 10, "v"}, {11, 12, "w"}}; + make_and_write(f.name, data); + auto csv_data_buffer = make_buffer(f.name); + { + auto [p, _] = make_parser(f.name, ","); + ss::parser p0{std::move(p)}; + p = std::move(p0); + std::vector i; + + auto [p2, __] = make_parser(f.name, ","); + std::vector i2; + + auto move_rotate = [&p = p, &p0 = p0] { + auto p1 = std::move(p); + p0 = std::move(p1); + p = std::move(p0); + }; + + while (!p.eof()) { + move_rotate(); + auto a = p.template get_next(); + i.emplace_back(ss::to_object(a)); + } + + for (const auto& a : p2.template iterate()) { + i2.emplace_back(ss::to_object(a)); + } + + CHECK_EQ(i, data); + CHECK_EQ(i2, data); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + auto [p2, __] = make_parser(f.name, ","); + std::vector i2; + + auto [p3, ___] = make_parser(f.name, ","); + std::vector i3; + + std::vector expected = {std::begin(data) + 1, std::end(data)}; + using tup = std::tuple; + + p.ignore_next(); + while (!p.eof()) { + auto a = p.template get_next(); + i.emplace_back(ss::to_object(a)); + } + + p2.ignore_next(); + for (const auto& a : p2.template iterate()) { + i2.emplace_back(ss::to_object(a)); + } + + p3.ignore_next(); + for (auto it = p3.template iterate().begin(); + it != p3.template iterate().end(); ++it) { + i3.emplace_back(ss::to_object(*it)); + } + + CHECK_EQ(i, expected); + CHECK_EQ(i2, expected); + CHECK_EQ(i3, expected); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + auto [p2, __] = make_parser(f.name, ","); + std::vector i2; + + while (!p.eof()) { + i.push_back(p.template get_object()); + } + + for (auto&& a : + p2.template iterate_object()) { + i2.push_back(std::move(a)); + } + + CHECK_EQ(i, data); + CHECK_EQ(i2, data); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + for (auto&& a : + p.template iterate_object()) { + i.push_back(std::move(a)); + } + + CHECK_EQ(i, data); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + auto [p2, __] = make_parser(f.name, ","); + std::vector i2; + + using tup = std::tuple; + while (!p.eof()) { + i.push_back(p.template get_object()); + } + + for (auto it = p2.template iterate_object().begin(); + it != p2.template iterate_object().end(); it++) { + i2.push_back({it->i, it->d, it->s}); + } + + CHECK_EQ(i, data); + CHECK_EQ(i2, data); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + using tup = std::tuple; + for (auto&& a : p.template iterate_object()) { + i.push_back(std::move(a)); + } + + CHECK_EQ(i, data); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + while (!p.eof()) { + i.push_back(p.template get_next()); + } + + CHECK_EQ(i, data); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + for (auto&& a : p.template iterate()) { + i.push_back(std::move(a)); + } + + CHECK_EQ(i, data); + } + + { + constexpr int excluded = 3; + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + auto [p2, __] = make_parser(f.name, ","); + std::vector i2; + + while (!p.eof()) { + try { + auto a = p.template get_object, double, + std::string>(); + if (p.valid()) { + i.push_back(a); + } + } catch (...) { + // ignore + }; + } + + if (!T::ThrowOnError) { + for (auto&& a : p2.template iterate_object, + double, std::string>()) { + if (p2.valid()) { + i2.push_back(std::move(a)); + } + } + } + + std::vector expected; + for (auto& x : data) { + if (x.i != excluded) { + expected.push_back(x); + } + } + + std::copy_if(data.begin(), data.end(), expected.begin(), + [&](const X& x) { return x.i != excluded; }); + CHECK_EQ(i, expected); + + if (!T::ThrowOnError) { + CHECK_EQ(i2, expected); + } + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + + auto [p2, __] = make_parser(f.name, ","); + std::vector i2; + + while (!p.eof()) { + try { + auto a = p.template get_object, double, + std::string>(); + if (p.valid()) { + i.push_back(a); + } + } catch (...) { + // ignore + } + } + + if (!T::ThrowOnError) { + for (auto&& a : p2.template iterate_object, + double, std::string>()) { + if (p2.valid()) { + i2.push_back(std::move(a)); + } + } + } + + std::vector expected = {{3, 4, "y"}}; + CHECK_EQ(i, expected); + if (!T::ThrowOnError) { + CHECK_EQ(i2, expected); + } + } + + { + unique_file_name empty_f{"various_valid_cases"}; + std::vector empty_data = {}; + + make_and_write(empty_f.name, empty_data); + + auto [p, _] = make_parser(empty_f.name, ","); + std::vector i; + + auto [p2, __] = make_parser(empty_f.name, ","); + std::vector i2; + + while (!p.eof()) { + i.push_back(p.template get_next()); + } + + for (auto&& a : p2.template iterate()) { + i2.push_back(std::move(a)); + } + + CHECK(i.empty()); + CHECK(i2.empty()); + } +} + +using test_tuple = std::tuple; +struct test_struct { + int i; + double d; + char c; + auto tied() { + return std::tie(i, d, c); + } +}; + +static inline void expect_test_struct(const test_struct&) { +} + +TEST_CASE_TEMPLATE("parser test composite conversion", BufferMode, + std::true_type, std::false_type) { + constexpr auto buffer_mode = BufferMode::value; + unique_file_name f{"composite_conversion"}; + { + std::ofstream out{f.name}; + for (auto& i : + {"10,a,11.1", "10,20,11.1", "junk", "10,11.1", "1,11.1,a", "junk", + "10,junk", "11,junk", "10,11.1,c", "10,20", "10,22.2,f"}) { + out << i << std::endl; + } + } + + auto [p, _] = make_parser(f.name, ","); + auto fail = [] { FAIL(""); }; + auto expect_error = [](auto error) { CHECK(!error.empty()); }; + auto ignore_error = [] {}; + + REQUIRE(p.valid()); + REQUIRE_FALSE(p.eof()); + + { + constexpr static auto expectedData = std::tuple{10, 'a', 11.1}; + + auto [d1, d2, d3, d4] = + p.template try_next(fail) + .template or_else(fail) + .template or_else( + [&](auto&& data) { CHECK_EQ(data, expectedData); }) + .on_error(fail) + .template or_else(fail) + .values(); + + REQUIRE(p.valid()); + REQUIRE_FALSE(d1); + REQUIRE_FALSE(d2); + REQUIRE(d3); + REQUIRE_FALSE(d4); + CHECK_EQ(*d3, expectedData); + } + + { + REQUIRE(!p.eof()); + constexpr static auto expectedData = std::tuple{10, 20, 11.1}; + + auto [d1, d2, d3, d4] = + p.template try_next( + [&](auto& i1, auto i2, double d) { + CHECK_EQ(std::tie(i1, i2, d), expectedData); + }) + .on_error(fail) + .template or_object(fail) + .on_error(fail) + .template or_else(fail) + .on_error(fail) + .template or_else(fail) + .values(); + + REQUIRE(p.valid()); + REQUIRE(d1); + REQUIRE_FALSE(d2); + REQUIRE_FALSE(d3); + REQUIRE_FALSE(d4); + CHECK_EQ(*d1, expectedData); + } + + { + REQUIRE(!p.eof()); + + auto [d1, d2, d3, d4, d5] = + p.template try_object(fail) + .on_error(expect_error) + .template or_else(fail) + .template or_else(fail) + .template or_else(fail) + .template or_else(fail) + .values(); + + REQUIRE_FALSE(p.valid()); + REQUIRE_FALSE(d1); + REQUIRE_FALSE(d2); + REQUIRE_FALSE(d3); + REQUIRE_FALSE(d4); + REQUIRE_FALSE(d5); + } + + { + REQUIRE(!p.eof()); + + auto [d1, d2] = + p.template try_next([](auto& i, auto& d) { + REQUIRE_EQ(std::tie(i, d), std::tuple{10, 11.1}); + }) + .template or_else([](auto&, auto&) { FAIL(""); }) + .values(); + + REQUIRE(p.valid()); + REQUIRE(d1); + REQUIRE_FALSE(d2); + } + + { + REQUIRE(!p.eof()); + + auto [d1, d2] = + p.template try_next([](auto&, auto&) { FAIL(""); }) + .template or_else(expect_test_struct) + .values(); + + REQUIRE(p.valid()); + REQUIRE_FALSE(d1); + REQUIRE(d2); + CHECK_EQ(d2->tied(), std::tuple{1, 11.1, 'a'}); + } + + { + REQUIRE(!p.eof()); + + auto [d1, d2, d3, d4, d5] = + p.template try_next(fail) + .template or_object() + .template or_else(expect_test_struct) + .template or_else(fail) + .template or_else>(fail) + .on_error(ignore_error) + .on_error(expect_error) + .values(); + + REQUIRE_FALSE(p.valid()); + REQUIRE_FALSE(d1); + REQUIRE_FALSE(d2); + REQUIRE_FALSE(d3); + REQUIRE_FALSE(d4); + REQUIRE_FALSE(d5); + } + + { + REQUIRE(!p.eof()); + + auto [d1, d2] = + p.template try_next>() + .on_error(ignore_error) + .on_error(fail) + .template or_else>(fail) + .on_error(ignore_error) + .on_error(fail) + .on_error(ignore_error) + .values(); + + REQUIRE(p.valid()); + REQUIRE(d1); + REQUIRE_FALSE(d2); + CHECK_EQ(*d1, std::tuple{10, std::nullopt}); + } + + { + REQUIRE_FALSE(p.eof()); + + auto [d1, d2] = + p.template try_next>() + .on_error(fail) + .template or_else>(fail) + .on_error(fail) + .values(); + + REQUIRE(p.valid()); + REQUIRE(d1); + REQUIRE_FALSE(d2); + CHECK_EQ(*d1, std::tuple{11, std::variant{"junk"}}); + } + + { + REQUIRE(!p.eof()); + + auto [d1, d2] = p.template try_object() + .template or_else(fail) + .values(); + REQUIRE(p.valid()); + REQUIRE(d1); + REQUIRE_FALSE(d2); + CHECK_EQ(d1->tied(), std::tuple{10, 11.1, 'c'}); + } + + { + REQUIRE_FALSE(p.eof()); + + auto [d1, d2, d3, d4] = + p.template try_next([] { return false; }) + .template or_else([](auto&) { return false; }) + .template or_else() + .template or_else(fail) + .values(); + + REQUIRE(p.valid()); + REQUIRE_FALSE(d1); + REQUIRE_FALSE(d2); + REQUIRE(d3); + REQUIRE_FALSE(d4); + CHECK_EQ(d3.value(), std::tuple{10, 20}); + } + + { + REQUIRE(!p.eof()); + + auto [d1, d2, d3, d4] = + p.template try_object( + [] { return false; }) + .template or_else([](auto&) { return false; }) + .template or_object() + .template or_else(fail) + .values(); + + REQUIRE(p.valid()); + REQUIRE_FALSE(d1); + REQUIRE_FALSE(d2); + REQUIRE(d3); + REQUIRE_FALSE(d4); + CHECK_EQ(d3->tied(), std::tuple{10, 22.2, 'f'}); + } + + CHECK(p.eof()); +} + +template +void test_no_new_line_at_eof_impl(const std::vector& data) { + unique_file_name f{"no_new_line_at_eof"}; + make_and_write(f.name, data, {}, false); + + auto [p, _] = make_parser(f.name); + std::vector parsed_data; + + for (const auto& el : p.template iterate()) { + parsed_data.push_back(el); + } + + CHECK_EQ(data, parsed_data); +} + +template +void test_no_new_line_at_eof() { + test_no_new_line_at_eof_impl({}); + test_no_new_line_at_eof_impl({{1, 2, "X"}}); + test_no_new_line_at_eof_impl({{1, 2, "X"}, {}}); + test_no_new_line_at_eof_impl( + {{1, 2, "X"}, {3, 4, "YY"}}); + test_no_new_line_at_eof_impl( + {{1, 2, "X"}, {3, 4, "YY"}, {}}); + test_no_new_line_at_eof_impl( + {{1, 2, "X"}, {3, 4, "YY"}, {5, 6, "ZZZ"}, {7, 8, "UUU"}}); + + for (size_t i = 0; i < 2 * ss::get_line_initial_buffer_size; ++i) { + test_no_new_line_at_eof_impl( + {{1, 2, std::string(i, 'X')}}); + + for (size_t j = 0; j < 2 * ss::get_line_initial_buffer_size; j += 13) { + + test_no_new_line_at_eof_impl( + {{1, 2, std::string(i, 'X')}, {3, 4, std::string(j, 'Y')}}); + + test_no_new_line_at_eof_impl( + {{1, 2, std::string(j, 'X')}, {3, 4, std::string(i, 'Y')}}); + } + } +} + +TEST_CASE_TEMPLATE("test no new line at end of data", T, + ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + test_no_new_line_at_eof(); +} diff --git a/test/test_parser1_2.cpp b/test/test_parser1_2.cpp new file mode 100644 index 0000000..eaf9516 --- /dev/null +++ b/test/test_parser1_2.cpp @@ -0,0 +1,262 @@ +#include "test_parser1.hpp" + +struct my_string { + char* data{nullptr}; + + my_string() = default; + + ~my_string() { + delete[] data; + } + + // make sure no object is copied + my_string(const my_string&) = delete; + my_string& operator=(const my_string&) = delete; + + my_string(my_string&& other) : data{other.data} { + other.data = nullptr; + } + + my_string& operator=(my_string&& other) { + data = other.data; + return *this; + } +}; + +template <> +inline bool ss::extract(const char* begin, const char* end, my_string& s) { + size_t size = end - begin; + s.data = new char[size + 1]; + strncpy(s.data, begin, size); + s.data[size] = '\0'; + return true; +} + +struct xyz { + my_string x; + my_string y; + my_string z; + auto tied() { + return std::tie(x, y, z); + } +}; + +TEST_CASE_TEMPLATE("test moving of parsed composite values", T, + config, config, + config, + config) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + // to compile is enough + return; + auto [p, _] = make_parser("", ""); + p.template try_next() + .template or_else( + [](auto&&) {}) + .template or_else([](auto&) {}) + .template or_else([](auto&&) {}) + .template or_object([](auto&&) {}) + .template or_else>( + [](auto&, auto&, auto&) {}); +} + +TEST_CASE_TEMPLATE("parser test string error mode", BufferMode, std::true_type, + std::false_type) { + unique_file_name f{"string_error"}; + { + std::ofstream out{f.name}; + out << "junk" << std::endl; + out << "junk" << std::endl; + } + + auto [p, _] = make_parser(f.name, ","); + + REQUIRE_FALSE(p.eof()); + p.template get_next(); + CHECK_FALSE(p.valid()); + CHECK_FALSE(p.error_msg().empty()); +} + +TEST_CASE_TEMPLATE("parser throw on error mode", BufferMode, std::true_type, + std::false_type) { + unique_file_name f{"throw_on_error"}; + { + std::ofstream out{f.name}; + out << "junk" << std::endl; + out << "junk" << std::endl; + } + + auto [p, _] = + make_parser(f.name, ","); + + REQUIRE_FALSE(p.eof()); + try { + p.template get_next(); + FAIL("Expected exception..."); + } catch (const std::exception& e) { + CHECK_FALSE(std::string{e.what()}.empty()); + } +} + +static inline std::string no_quote(const std::string& s) { + if (!s.empty() && s[0] == '"') { + return {std::next(begin(s)), std::prev(end(s))}; + } + return s; +} + +TEST_CASE_TEMPLATE("test quote multiline", T, ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"quote_multiline"}; + std::vector data = {{1, 2, "\"x\r\nx\nx\""}, + {3, 4, "\"y\ny\r\ny\""}, + {5, 6, "\"z\nz\""}, + {7, 8, "\"u\"\"\""}, + {9, 10, "v"}, + {11, 12, "\"w\n\""}}; + for (auto& [_, __, s] : data) { + update_if_crlf(s); + } + + make_and_write(f.name, data); + for (auto& [_, __, s] : data) { + s = no_quote(s); + if (s[0] == 'u') { + s = "u\""; + } + } + + auto [p, _] = make_parser>(f.name, ","); + + std::vector i; + + while (!p.eof()) { + auto a = p.template get_next(); + i.emplace_back(ss::to_object(a)); + } + + for (auto& [_, __, s] : i) { + update_if_crlf(s); + } + CHECK_EQ(i, data); + + auto [p_no_multiline, __] = + make_parser>(f.name, ","); + while (!p.eof()) { + auto command = [&p_no_multiline = p_no_multiline] { + p_no_multiline.template get_next(); + }; + expect_error_on_command(p_no_multiline, command); + } +} + +static inline std::string no_escape(std::string& s) { + s.erase(std::remove(begin(s), end(s), '\\'), end(s)); + return s; +} + +TEST_CASE_TEMPLATE("test escape multiline", T, ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"escape_multiline"}; + std::vector data = {{1, 2, "x\\\nx\\\r\nx"}, + {5, 6, "z\\\nz\\\nz"}, + {7, 8, "u"}, + {3, 4, "y\\\ny\\\ny"}, + {9, 10, "v\\\\"}, + {11, 12, "w\\\n"}}; + for (auto& [_, __, s] : data) { + update_if_crlf(s); + } + + make_and_write(f.name, data); + for (auto& [_, __, s] : data) { + s = no_escape(s); + if (s == "v") { + s = "v\\"; + } + } + + auto [p, _] = make_parser>(f.name, ","); + std::vector i; + + while (!p.eof()) { + auto a = p.template get_next(); + i.emplace_back(ss::to_object(a)); + } + + for (auto& [_, __, s] : i) { + update_if_crlf(s); + } + CHECK_EQ(i, data); + + auto [p_no_multiline, __] = + make_parser>(f.name, ","); + while (!p.eof()) { + auto command = [&p_no_multiline = p_no_multiline] { + auto a = + p_no_multiline.template get_next(); + }; + expect_error_on_command(p_no_multiline, command); + } +} + +TEST_CASE_TEMPLATE("test quote escape multiline", T, ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"quote_escape_multiline"}; + { + std::ofstream out{f.name}; + out << "1,2,\"just\\\n\nstrings\"" << std::endl; +#ifndef _WIN32 + out << "3,4,\"just\r\nsome\\\r\n\n\\\nstrings\"" << std::endl; + out << "5,6,\"just\\\n\\\r\n\r\n\nstrings" << std::endl; +#else + out << "3,4,\"just\nsome\\\n\n\\\nstrings\"" << std::endl; + out << "5,6,\"just\\\n\\\n\n\nstrings" << std::endl; +#endif + out << "7,8,\"just strings\"" << std::endl; + out << "9,10,just strings" << std::endl; + } + size_t bad_lines = 1; + auto num_errors = 0; + + auto [p, _] = make_parser, ss::quote<'"'>>(f.name); + std::vector i; + + while (!p.eof()) { + try { + auto a = p.template get_next(); + if (p.valid()) { + i.emplace_back(ss::to_object(a)); + } else { + ++num_errors; + } + } catch (const std::exception& e) { + ++num_errors; + } + } + + CHECK(bad_lines == num_errors); + + std::vector data = {{1, 2, "just\n\nstrings"}, +#ifndef _WIN32 + {3, 4, "just\r\nsome\r\n\n\nstrings"}, +#else + {3, 4, "just\nsome\n\n\nstrings"}, +#endif + {9, 10, "just strings"}}; + + for (auto& [_, __, s] : i) { + update_if_crlf(s); + } + CHECK_EQ(i, data); +} diff --git a/test/test_parser1_3.cpp b/test/test_parser1_3.cpp new file mode 100644 index 0000000..07147b6 --- /dev/null +++ b/test/test_parser1_3.cpp @@ -0,0 +1,313 @@ +#include "test_parser1.hpp" + +TEST_CASE_TEMPLATE("test multiline restricted", T, ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"multiline_restricted"}; + { + std::ofstream out{f.name}; + out << "1,2,\"just\n\nstrings\"" << std::endl; +#ifndef _WIN32 + out << "3,4,\"ju\n\r\n\nnk\"" << std::endl; + out << "5,6,just\\\n\\\r\nstrings" << std::endl; +#else + out << "3,4,\"ju\n\n\nnk\"" << std::endl; + out << "5,6,just\\\n\\\nstrings" << std::endl; +#endif + out << "7,8,ju\\\n\\\n\\\nnk" << std::endl; + out << "9,10,\"just\\\n\nstrings\"" << std::endl; + out << "11,12,\"ju\\\n|\n\n\n\n\nk\"" << std::endl; + out << "13,14,\"ju\\\n\\\n15,16\"\\\n\\\\\n\nnk\"" << std::endl; + out << "17,18,\"ju\\\n\\\n\\\n\\\\\n\nnk\"" << std::endl; + out << "19,20,just strings" << std::endl; + } + auto bad_lines = 15; + auto num_errors = 0; + + auto [p, _] = + make_parser, + ss::quote<'"'>, ss::escape<'\\'>>(f.name, ","); + std::vector i; + + while (!p.eof()) { + try { + auto a = p.template get_next(); + if (p.valid()) { + i.emplace_back(ss::to_object(a)); + } else { + ++num_errors; + } + } catch (const std::exception& e) { + ++num_errors; + } + } + + CHECK(bad_lines == num_errors); + + std::vector data = {{1, 2, "just\n\nstrings"}, +#ifndef _WIN32 + {5, 6, "just\n\r\nstrings"}, +#else + {5, 6, "just\n\nstrings"}, +#endif + {9, 10, "just\n\nstrings"}, + {19, 20, "just strings"}}; + + for (auto& [_, __, s] : i) { + update_if_crlf(s); + } + + if (i.size() != data.size()) { + CHECK_EQ(i.size(), data.size()); + } + + CHECK_EQ(i, data); +} + +template +void test_unterminated_line(const std::vector& lines, + size_t bad_line) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"unterminated_line"}; + std::ofstream out{f.name}; + for (const auto& line : lines) { + out << line << std::endl; + } + out.close(); + + auto [p, _] = make_parser(f.name); + size_t line = 0; + while (!p.eof()) { + auto command = [&p = p] { + p.template get_next(); + }; + + if (line == bad_line) { + expect_error_on_command(p, command); + break; + } else { + command(); + CHECK(p.valid()); + ++line; + } + } +} + +TEST_CASE_TEMPLATE("parser test csv on multiline with errors", T, + ParserOptionCombinations) { + using multiline = ss::multiline_restricted<3>; + using escape = ss::escape<'\\'>; + using quote = ss::quote<'"'>; + + // unterminated escape + { + const std::vector lines{"1,2,just\\"}; + test_unterminated_line(lines, 0); + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"1,2,just\\", "9,8,second"}; + test_unterminated_line(lines, 0); + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"9,8,first", "1,2,just\\"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,first", "1,2,just\\", + "3,4,third"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,first", + "1,2,just\\\nstrings\\", + "3,4,th\\\nird"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,first", "3,4,second", + "1,2,just\\"}; + test_unterminated_line(lines, 2); + test_unterminated_line(lines, 2); + } + + { + const std::vector lines{"9,8,\\first", "3,4,second", + "1,2,jus\\t\\"}; + test_unterminated_line(lines, 2); + test_unterminated_line(lines, 2); + } + + // unterminated quote + { + const std::vector lines{"1,2,\"just"}; + test_unterminated_line(lines, 0); + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"1,2,\"just", "9,8,second"}; + test_unterminated_line(lines, 0); + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"9,8,first", "1,2,\"just"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,first", "1,2,\"just", + "3,4,th\\,ird"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,first", "3,4,second", + "1,2,\"just"}; + test_unterminated_line(lines, 2); + test_unterminated_line(lines, 2); + } + + { + const std::vector lines{"9,8,\"first\"", + "\"3\",4,\"sec,ond\"", + "1,2,\"ju\"\"st"}; + test_unterminated_line(lines, 2); + test_unterminated_line(lines, 2); + } + + // unterminated quote and escape + { + const std::vector lines{"1,2,\"just\\"}; + test_unterminated_line(lines, 0); + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"1,2,\"just\\\n\\"}; + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"1,2,\"just\n\\"}; + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"9,8,first", "1,2,\"just\n\\"}; + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,first", "1,2,\"just\n\\", + "4,3,thrid"}; + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,f\\\nirst", "1,2,\"just\n\\", + "4,3,thrid"}; + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,\"f\ni\nrst\"", + "1,2,\"just\n\\", "4,3,thrid"}; + test_unterminated_line(lines, 1); + } + + // multiline limmit reached escape + { + const std::vector lines{"1,2,\\\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 0); + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"9,8,first", + "1,2,\\\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,fi\\\nrs\\\nt", + "1,2,\\\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,first", + "1,2,\\\n\\\n\\\n\\\njust", + "4,3,third"}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + // multiline limmit reached quote + { + const std::vector lines{"1,2,\"\n\n\n\n\njust\""}; + test_unterminated_line(lines, 0); + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"9,8,first", + "1,2,\"\n\n\n\n\njust\""}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,\"fir\nst\"", + "1,2,\"\n\n\n\n\njust\""}; + test_unterminated_line(lines, 1); + test_unterminated_line(lines, 1); + } + + // multiline limmit reached quote and escape + { + const std::vector lines{"1,2,\"\\\n\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 0); + } + + { + const std::vector lines{"9,8,first", + "1,2,\"\\\n\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,fi\\\nrst", + "1,2,\"\\\n\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,\"fi\nrst\"", + "1,2,\"\\\n\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 1); + } + + { + const std::vector lines{"9,8,\"fi\nr\\\nst\"", + "1,2,\"\\\n\n\\\n\\\n\\\njust"}; + test_unterminated_line(lines, 1); + } +} diff --git a/test/test_parser1_4.cpp b/test/test_parser1_4.cpp new file mode 100644 index 0000000..993e74b --- /dev/null +++ b/test/test_parser1_4.cpp @@ -0,0 +1,471 @@ +#include "test_parser1.hpp" + +template +struct has_type; + +template +struct has_type> + : std::disjunction...> {}; + +template +static void test_fields(const std::string file_name, const std::vector& data, + const std::vector& fields) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + using CaseType = std::tuple; + + auto [p, _] = make_parser(file_name, ","); + CHECK_FALSE(p.field_exists("Unknown")); + p.use_fields(fields); + std::vector i; + + for (const auto& a : p.template iterate()) { + i.push_back(a); + } + + CHECK_EQ(i.size(), data.size()); + for (size_t j = 0; j < i.size(); ++j) { + if constexpr (has_type::value) { + CHECK_EQ(std::get(i[j]), data[j].i); + } + if constexpr (has_type::value) { + CHECK_EQ(std::get(i[j]), data[j].d); + } + if constexpr (has_type::value) { + CHECK_EQ(std::get(i[j]), data[j].s); + } + } +} + +TEST_CASE_TEMPLATE("test various cases with header", T, + ParserOptionCombinations) { + unique_file_name f{"various_cases_with_header"}; + constexpr static auto Int = "Int"; + constexpr static auto Dbl = "Double"; + constexpr static auto Str = "String"; + using str = std::string; + + std::vector header{Int, Dbl, Str}; + + std::vector data = {{1, 2, "x"}, {3, 4, "y"}, {5, 6, "z"}, + {7, 8, "u"}, {9, 10, "v"}, {11, 12, "w"}}; + + make_and_write(f.name, data, header); + const auto& o = f.name; + const auto& d = data; + + { + ss::parser p{f.name, ","}; + std::vector i; + + for (const auto& a : p.iterate()) { + i.emplace_back(ss::to_object(a)); + } + + CHECK_NE(i, data); + } + + { + ss::parser p{f.name, ","}; + std::vector i; + + p.ignore_next(); + for (const auto& a : p.iterate()) { + i.emplace_back(ss::to_object(a)); + } + + CHECK_EQ(i, data); + } + + { + ss::parser p{f.name, ","}; + std::vector i; + + for (const auto& a : p.iterate()) { + i.emplace_back(ss::to_object(a)); + } + + CHECK_EQ(i, data); + } + + { + ss::parser p{f.name, ","}; + p.use_fields(Int, Dbl, Str); + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{f.name, ","}; + CHECK_FALSE(p.field_exists("Unknown")); + + p.use_fields(Int, "Unknown"); + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{f.name, ","}; + p.use_fields(Int, Int); + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{f.name, ","}; + p.use_fields(Int, Dbl); + + { + auto [int_, double_] = p.get_next(); + CHECK_EQ(int_, data[0].i); + CHECK_EQ(double_, data[0].d); + } + + p.use_fields(Dbl, Int); + + { + auto [double_, int_] = p.get_next(); + CHECK_EQ(int_, data[1].i); + CHECK_EQ(double_, data[1].d); + } + + p.use_fields(Str); + + { + auto string_ = p.get_next(); + CHECK_EQ(string_, data[2].s); + } + + p.use_fields(Str, Int, Dbl); + + { + auto [string_, int_, double_] = + p.get_next(); + CHECK_EQ(double_, data[3].d); + CHECK_EQ(int_, data[3].i); + CHECK_EQ(string_, data[3].s); + } + } + + /* python used to generate permutations + import itertools + + header = {'str': 'Str', + 'double': 'Dbl', + 'int': 'Int'} + + keys = ['str', 'int', 'double'] + + for r in range (1, 3): + combinations = list(itertools.permutations(keys, r = r)) + + for combination in combinations: + template_params = [] + arg_params = [] + for type in combination: + template_params.append(type) + arg_params.append(header[type]) + call = 'testFields<' + ', '.join(template_params) + \ + '>(o, d, {' + ', '.join(arg_params) + '});' + print(call) + */ + + test_fields(o, d, {Str}); + test_fields(o, d, {Int}); + test_fields(o, d, {Dbl}); + test_fields(o, d, {Str, Int}); + test_fields(o, d, {Str, Dbl}); + test_fields(o, d, {Int, Str}); + test_fields(o, d, {Int, Dbl}); + test_fields(o, d, {Dbl, Str}); + test_fields(o, d, {Dbl, Int}); + test_fields(o, d, {Str, Int, Dbl}); + test_fields(o, d, {Str, Dbl, Int}); + test_fields(o, d, {Int, Str, Dbl}); + test_fields(o, d, {Int, Dbl, Str}); + test_fields(o, d, {Dbl, Str, Int}); + test_fields(o, d, {Dbl, Int, Str}); +} + +template +void test_invalid_fields(const std::vector& lines, + const std::vector& fields) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"invalid_fields"}; + { + std::ofstream out{f.name}; + for (const auto& line : lines) { + out << line << std::endl; + } + } + + { + // No fields specified + auto [p, _] = make_parser(f.name, ","); + auto command = [&p = p] { p.use_fields(); }; + expect_error_on_command(p, command); + } + + { + // Unknown field + auto [p, _] = make_parser(f.name, ","); + auto command = [&p = p] { p.use_fields("Unknown"); }; + expect_error_on_command(p, command); + } + + { + // Field used multiple times + auto [p, _] = make_parser(f.name, ","); + auto command = [&p = p, &fields = fields] { + p.use_fields(fields.at(0), fields.at(0)); + }; + if (!fields.empty()) { + expect_error_on_command(p, command); + } + } + + { + // Mapping out of range + auto [p, _] = make_parser(f.name, ","); + auto command = [&p = p, &fields = fields] { + p.use_fields(fields.at(0)); + p.template get_next(); + }; + if (!fields.empty()) { + expect_error_on_command(p, command); + } + } + + { + // Invalid header + auto [p, _] = make_parser(f.name, ","); + auto command = [&p = p, &fields = fields] { p.use_fields(fields); }; + + if (!fields.empty()) { + // Pass if there are no duplicates, fail otherwise + if (std::unordered_set{fields.begin(), fields.end()} + .size() != fields.size()) { + expect_error_on_command(p, command); + } else { + command(); + CHECK(p.valid()); + if (!p.valid()) { + if constexpr (T::StringError) { + std::cout << p.error_msg() << std::endl; + } + } + } + } + } +} + +TEST_CASE_TEMPLATE("test invalid fheader fields usage", T, + ParserOptionCombinations) { + test_invalid_fields({}, {}); + + test_invalid_fields({"Int"}, {"Int"}); + test_invalid_fields({"Int", "1"}, {"Int"}); + test_invalid_fields({"Int", "1", "2"}, {"Int"}); + + test_invalid_fields({"Int,String"}, {"Int", "String"}); + test_invalid_fields({"Int,String", "1,hi"}, {"Int", "String"}); + test_invalid_fields({"Int,String", "2,hello"}, {"Int", "String"}); + + test_invalid_fields({"Int,String,Double"}, {"Int", "String", "Double"}); + test_invalid_fields({"Int,String,Double", "1,hi,2.34"}, + {"Int", "String", "Double"}); + test_invalid_fields({"Int,String,Double", "1,hi,2.34", "2,hello,3.45"}, + {"Int", "String", "Double"}); + + test_invalid_fields({"Int,Int,Int"}, {"Int", "Int", "Int"}); + test_invalid_fields({"Int,Int,Int", "1,2,3"}, {"Int", "Int", "Int"}); + + test_invalid_fields({"Int,String,Int"}, {"Int", "String", "Int"}); + test_invalid_fields({"Int,String,Int", "1,hi,3"}, + {"Int", "String", "Int"}); +} + +TEST_CASE_TEMPLATE("test invalid rows with header", T, + ParserOptionCombinations) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"invalid rows with header"}; + { + std::ofstream out{f.name}; + out << "Int,String,Double" << std::endl; + out << "1,line1,2.34" << std::endl; + out << "2,line2" << std::endl; + out << "3,line3,67.8" << std::endl; + out << "4,line4,67.8,9" << std::endl; + out << "5,line5,9.10" << std::endl; + out << "six,line6,10.11" << std::endl; + } + + { + auto [p, _] = make_parser(f.name); + + p.use_fields("Int", "String", "Double"); + using data = std::tuple; + std::vector i; + + CHECK(p.valid()); + + while (!p.eof()) { + try { + const auto& t = p.template get_next(); + if (p.valid()) { + i.push_back(t); + } + } catch (const ss::exception&) { + continue; + } + } + + std::vector expected = {{1, "line1", 2.34}, + {3, "line3", 67.8}, + {5, "line5", 9.10}}; + CHECK_EQ(i, expected); + } + + { + auto [p, _] = make_parser(f.name); + + p.use_fields("Double", "Int"); + using data = std::tuple; + std::vector i; + + CHECK(p.valid()); + + while (!p.eof()) { + try { + const auto& t = p.template get_next(); + if (p.valid()) { + i.push_back(t); + } + } catch (const ss::exception&) { + continue; + } + } + + std::vector expected = {{2.34, 1}, {67.8, 3}, {9.10, 5}}; + CHECK_EQ(i, expected); + } + + { + auto [p, _] = make_parser(f.name); + + p.use_fields("String", "Double"); + using data = std::tuple; + std::vector i; + + CHECK(p.valid()); + + while (!p.eof()) { + try { + const auto& t = p.template get_next(); + if (p.valid()) { + i.push_back(t); + } + } catch (const ss::exception&) { + continue; + } + } + + std::vector expected = {{"line1", 2.34}, + {"line3", 67.8}, + {"line5", 9.10}, + {"line6", 10.11}}; + CHECK_EQ(i, expected); + } +} + +template +void test_ignore_empty(const std::vector& data) { + constexpr auto buffer_mode = T::BufferMode::value; + using ErrorMode = typename T::ErrorMode; + + unique_file_name f{"ignore_empty"}; + make_and_write(f.name, data); + + std::vector expected; + for (const auto& d : data) { + if (d.s != X::empty) { + expected.push_back(d); + } + } + + { + auto [p, _] = + make_parser(f.name, ","); + + std::vector i; + for (const auto& a : p.template iterate()) { + i.push_back(a); + } + + CHECK_EQ(i, expected); + } + + { + auto [p, _] = make_parser(f.name, ","); + std::vector i; + size_t n = 0; + while (!p.eof()) { + try { + ++n; + const auto& a = p.template get_next(); + if (data.at(n - 1).s == X::empty) { + CHECK_FALSE(p.valid()); + continue; + } + i.push_back(a); + } catch (...) { + CHECK_EQ(data.at(n - 1).s, X::empty); + } + } + + CHECK_EQ(i, expected); + } +} + +TEST_CASE_TEMPLATE("test various cases with empty lines", T, + ParserOptionCombinations) { + test_ignore_empty( + {{1, 2, "x"}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, "w"}}); + + test_ignore_empty( + {{1, 2, X::empty}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, "w"}}); + + test_ignore_empty( + {{1, 2, "x"}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, X::empty}}); + + test_ignore_empty( + {{1, 2, "x"}, {5, 6, X::empty}, {9, 10, "v"}, {11, 12, "w"}}); + + test_ignore_empty( + {{1, 2, X::empty}, {5, 6, X::empty}, {9, 10, "v"}, {11, 12, "w"}}); + + test_ignore_empty( + {{1, 2, X::empty}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, X::empty}}); + + test_ignore_empty( + {{1, 2, "x"}, {3, 4, "y"}, {9, 10, X::empty}, {11, 12, X::empty}}); + + test_ignore_empty( + {{1, 2, X::empty}, {3, 4, "y"}, {9, 10, X::empty}, {11, 12, X::empty}}); + + test_ignore_empty({{1, 2, X::empty}, + {3, 4, X::empty}, + {9, 10, X::empty}, + {11, 12, X::empty}}); + + test_ignore_empty( + {{1, 2, "x"}, {3, 4, X::empty}, {9, 10, X::empty}, {11, 12, X::empty}}); + + test_ignore_empty( + {{1, 2, X::empty}, {3, 4, X::empty}, {9, 10, X::empty}, {11, 12, "w"}}); + + test_ignore_empty({{11, 12, X::empty}}); + + test_ignore_empty({}); +} diff --git a/test/test_parser2.hpp b/test/test_parser2.hpp index 262cb92..6048a33 100644 --- a/test/test_parser2.hpp +++ b/test/test_parser2.hpp @@ -121,7 +121,7 @@ column make_column(const std::string& input_header, } [[maybe_unused]] void replace_all2(std::string& s, const std::string& old_value, - const std::string& new_value) { + const std::string& new_value) { for (size_t i = 0; i < 999; ++i) { size_t pos = s.find(old_value); if (pos == std::string::npos) { @@ -257,9 +257,12 @@ std::vector generate_csv_data(const std::vector& data, } [[maybe_unused]] void write_to_file(const std::vector& data, - const std::string& delim, const std::string& file_name) { + const std::string& delim, + const std::string& file_name, + bool add_new_line = true) { std::ofstream out{file_name, std::ios_base::app}; std::string line; + for (size_t i = 0; i < data.size(); ++i) { line += data[i]; if (i != data.size() - 1) { @@ -267,7 +270,10 @@ std::vector generate_csv_data(const std::vector& data, } } - out << line << std::endl; + out << line; + if (add_new_line) { + out << std::endl; + } } #define CHECK_EQ_CRLF(V1, V2) \ @@ -299,7 +305,7 @@ std::vector generate_csv_data(const std::vector& data, CHECK(V1 == V2); \ } -template +template void test_data_combinations(const std::vector& input_data, const std::string& delim, bool include_header) { using setup = ss::setup; @@ -308,7 +314,7 @@ void test_data_combinations(const std::vector& input_data, return; } - unique_file_name f{"test_parser2" + std::string{SEGMENT_NAME}}; + unique_file_name f{"parser_data_combinations" + std::string{SEGMENT_NAME}}; std::vector> expected_data; std::vector header; std::vector field_header; @@ -329,7 +335,11 @@ void test_data_combinations(const std::vector& input_data, if (include_header) { auto header_data = generate_csv_data(field_header, delim); - write_to_file(header_data, delim, f.name); + if (input_data.size() == 0 && rand() % 10 == 0) { + write_to_file(header_data, delim, f.name, false); + } else { + write_to_file(header_data, delim, f.name); + } } std::vector layout; @@ -354,15 +364,12 @@ void test_data_combinations(const std::vector& input_data, expected_data.push_back(raw_data); auto data = generate_csv_data(raw_data, delim); - write_to_file(data, delim, f.name); - /* - std::cout << "[."; - for (const auto& el : data) { - std::cout << el << '.'; + if (i + 1 == n && rand() % 10 == 0) { + write_to_file(data, delim, f.name, false); + } else { + write_to_file(data, delim, f.name); } - std::cout << "]" << std::endl; - */ } auto layout_combinations = include_header && !setup::ignore_header @@ -388,7 +395,7 @@ void test_data_combinations(const std::vector& input_data, } for (const auto& layout : unique_layout_combinations) { - ss::parser p{f.name, delim}; + auto [p, _] = make_parser(f.name, delim); if (include_header && !setup::ignore_header) { std::vector fields; @@ -409,7 +416,7 @@ void test_data_combinations(const std::vector& input_data, REQUIRE(p.valid()); } - auto check_error = [&p] { + auto check_error = [&p = p] { CHECK(p.valid()); if (!p.valid()) { if constexpr (setup::string_error) { @@ -426,7 +433,6 @@ void test_data_combinations(const std::vector& input_data, auto s0 = p.template get_next(); if (i < n) { check_error(); - // std::cout << s0 << std::endl; CHECK_EQ_CRLF(s0, expected_data[i][layout[0]].value); } else { CHECK(p.eof()); @@ -439,7 +445,6 @@ void test_data_combinations(const std::vector& input_data, p.template get_next(); if (i < n) { check_error(); - // std::cout << s0 << ' ' << s1 << std::endl; CHECK_EQ_CRLF(s0, expected_data[i][layout[0]].value); CHECK_EQ_CRLF(s1, expected_data[i][layout[1]].value); } else { @@ -454,8 +459,6 @@ void test_data_combinations(const std::vector& input_data, std::string>(); if (i < n) { check_error(); - // std::cout << s0 << ' ' << s1 << ' ' << s2 << - // std::endl; CHECK_EQ_CRLF(s0, expected_data[i][layout[0]].value); CHECK_EQ_CRLF(s1, expected_data[i][layout[1]].value); CHECK_EQ_CRLF(s2, expected_data[i][layout[2]].value); @@ -471,10 +474,6 @@ void test_data_combinations(const std::vector& input_data, std::string, std::string>(); if (i < n) { check_error(); - /* - std::cout << s0 << ' ' << s1 << ' ' << s2 << ' ' << s3 - << std::endl; - */ CHECK_EQ_CRLF(s0, expected_data[i][layout[0]].value); CHECK_EQ_CRLF(s1, expected_data[i][layout[1]].value); CHECK_EQ_CRLF(s2, expected_data[i][layout[2]].value); @@ -492,9 +491,6 @@ void test_data_combinations(const std::vector& input_data, std::string>(); if (i < n) { check_error(); - // std::cout << s0 << ' ' << s1 << ' ' << s2 << ' ' << - // s3 - // << ' ' << s4 << std::endl; CHECK_EQ_CRLF(s0, expected_data[i][layout[0]].value); CHECK_EQ_CRLF(s1, expected_data[i][layout[1]].value); CHECK_EQ_CRLF(s2, expected_data[i][layout[2]].value); @@ -570,8 +566,14 @@ void test_option_combinations() { {columns0, columns1, columns2, columns3, columns4, columns5, columns6, columns7}) { try { - test_data_combinations(columns, delimiter, false); - test_data_combinations(columns, delimiter, true); + test_data_combinations(columns, delimiter, + false); + test_data_combinations(columns, delimiter, + true); + test_data_combinations(columns, delimiter, + false); + test_data_combinations(columns, delimiter, + true); } catch (std::exception& e) { std::cout << typeid(ss::parser).name() << std::endl; FAIL_CHECK(std::string{e.what()}); @@ -616,6 +618,7 @@ void test_option_combinations3() { } /* namespace */ +// Tests split into multiple compilation units #if 0 TEST_CASE("parser test various cases version 2 segment 1") { @@ -627,25 +630,33 @@ TEST_CASE("parser test various cases version 2 segment 1") { using multiline_r = ss::multiline_restricted<10>; using trimr = ss::trim_right<' '>; using triml = ss::trim_left<' '>; + using trim = ss::trim<' '>; // segment 1 test_option_combinations3<>(); test_option_combinations3(); - test_option_combinations3(); // segment 2 + test_option_combinations3(); test_option_combinations3(); + + // segment 3 test_option_combinations3(); test_option_combinations3(); - // segment 3 + // segment 4 test_option_combinations3(); test_option_combinations3(); - // segment 4 + // segment 5 test_option_combinations(); test_option_combinations(); + + // segment 6 + test_option_combinations3(); + test_option_combinations3(); #else + test_option_combinations3(); #endif } diff --git a/test/test_parser2_1.cpp b/test/test_parser2_1.cpp index 6f1ac7a..55ec45a 100644 --- a/test/test_parser2_1.cpp +++ b/test/test_parser2_1.cpp @@ -3,12 +3,10 @@ TEST_CASE("parser test various cases version 2 segment 1") { #ifdef CMAKE_GITHUB_CI - using quote = ss::quote<'"'>; using escape = ss::escape<'\\'>; test_option_combinations3<>(); test_option_combinations3(); - test_option_combinations3(); #endif } diff --git a/test/test_parser2_2.cpp b/test/test_parser2_2.cpp index 5b5cc2c..ec76aa9 100644 --- a/test/test_parser2_2.cpp +++ b/test/test_parser2_2.cpp @@ -5,11 +5,9 @@ TEST_CASE("parser test various cases version 2 segment 2") { #ifdef CMAKE_GITHUB_CI using quote = ss::quote<'"'>; using escape = ss::escape<'\\'>; - using multiline = ss::multiline; + test_option_combinations3(); test_option_combinations3(); - test_option_combinations3(); - test_option_combinations3(); #endif } diff --git a/test/test_parser2_3.cpp b/test/test_parser2_3.cpp index 776f49a..aab905c 100644 --- a/test/test_parser2_3.cpp +++ b/test/test_parser2_3.cpp @@ -6,10 +6,9 @@ TEST_CASE("parser test various cases version 2 segment 3") { using quote = ss::quote<'"'>; using escape = ss::escape<'\\'>; using multiline = ss::multiline; - using multiline_r = ss::multiline_restricted<10>; - test_option_combinations3(); - test_option_combinations3(); + test_option_combinations3(); + test_option_combinations3(); #endif } diff --git a/test/test_parser2_4.cpp b/test/test_parser2_4.cpp index 4658337..ea78aaf 100644 --- a/test/test_parser2_4.cpp +++ b/test/test_parser2_4.cpp @@ -2,18 +2,14 @@ #include "test_parser2.hpp" TEST_CASE("parser test various cases version 2 segment 4") { +#ifdef CMAKE_GITHUB_CI using quote = ss::quote<'"'>; using escape = ss::escape<'\\'>; using multiline = ss::multiline; + using multiline_r = ss::multiline_restricted<10>; -#ifdef CMAKE_GITHUB_CI - using trimr = ss::trim_right<' '>; - using triml = ss::trim_left<' '>; - - test_option_combinations(); - test_option_combinations(); -#else test_option_combinations3(); + test_option_combinations3(); #endif } diff --git a/test/test_parser2_5.cpp b/test/test_parser2_5.cpp new file mode 100644 index 0000000..3f345b0 --- /dev/null +++ b/test/test_parser2_5.cpp @@ -0,0 +1,16 @@ +#define SEGMENT_NAME "segment5" +#include "test_parser2.hpp" + +TEST_CASE("parser test various cases version 2 segment 5") { +#ifdef CMAKE_GITHUB_CI + using quote = ss::quote<'"'>; + using escape = ss::escape<'\\'>; + using multiline = ss::multiline; + using trimr = ss::trim_right<' '>; + using triml = ss::trim_left<' '>; + + test_option_combinations(); + test_option_combinations(); +#endif +} + diff --git a/test/test_parser2_6.cpp b/test/test_parser2_6.cpp new file mode 100644 index 0000000..d6ec30f --- /dev/null +++ b/test/test_parser2_6.cpp @@ -0,0 +1,11 @@ +#define SEGMENT_NAME "segment6" +#include "test_parser2.hpp" + +TEST_CASE("parser test various cases version 2 segment 6") { + using quote = ss::quote<'"'>; + using escape = ss::escape<'\\'>; + using multiline = ss::multiline; + + test_option_combinations3(); +} +