From eeac30651a41c53861d1ce75dd92f6164b7e5bc3 Mon Sep 17 00:00:00 2001 From: ado Date: Thu, 29 Jun 2023 23:41:03 +0200 Subject: [PATCH 01/64] Implement throw_on_error functionality, rename some template parameters --- include/ss/converter.hpp | 21 +++++++++++---- include/ss/exception.hpp | 30 +++++++++++++++++++++ include/ss/parser.hpp | 45 ++++++++++++++++++------------- include/ss/setup.hpp | 58 +++++++++++++++++++++++++--------------- include/ss/splitter.hpp | 23 ++++++++++------ 5 files changed, 124 insertions(+), 53 deletions(-) create mode 100644 include/ss/exception.hpp diff --git a/include/ss/converter.hpp b/include/ss/converter.hpp index 3650576..a9e3156 100644 --- a/include/ss/converter.hpp +++ b/include/ss/converter.hpp @@ -1,4 +1,5 @@ #pragma once +#include "exception.hpp" #include "extract.hpp" #include "function_traits.hpp" #include "restrictions.hpp" @@ -95,11 +96,12 @@ constexpr bool tied_class_v = tied_class::value; // converter //////////////// -template +template class converter { - using line_ptr_type = typename splitter::line_ptr_type; + using line_ptr_type = typename splitter::line_ptr_type; - constexpr static auto string_error = setup::string_error; + constexpr static auto string_error = setup::string_error; + constexpr static auto throw_on_error = setup::throw_on_error; constexpr static auto default_delimiter = ","; using error_type = std::conditional_t; @@ -229,6 +231,7 @@ private: if constexpr (string_error) { error_.clear(); error_.append(splitter_.error_msg()); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -239,6 +242,7 @@ private: error_.clear(); splitter_.set_error_unterminated_escape(); error_.append(splitter_.error_msg()); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -247,7 +251,8 @@ private: void set_error_multiline_limit_reached() { if constexpr (string_error) { error_.clear(); - error_.append("multiline limit reached."); + error_.append("multiline limit reached"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -258,6 +263,7 @@ private: error_.clear(); error_.append("invalid conversion for parameter ") .append(error_sufix(msg, pos)); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -268,6 +274,7 @@ private: if constexpr (string_error) { error_.clear(); error_.append(error).append(" ").append(error_sufix(msg, pos)); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -280,6 +287,7 @@ private: .append(std::to_string(expected_pos)) .append(", got: ") .append(std::to_string(pos)); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -295,6 +303,7 @@ private: .append(std::to_string(mapping_size)) .append(", got: ") .append(std::to_string(argument_size)); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -304,6 +313,7 @@ private: if constexpr (string_error) { error_.clear(); error_.append("received empty mapping"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -317,6 +327,7 @@ private: .append(std::to_string(maximum_index)) .append(", greater then number of columns: ") .append(std::to_string(number_of_columnts)); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -472,7 +483,7 @@ private: //////////////// error_type error_{}; - splitter splitter_; + splitter splitter_; template friend class parser; diff --git a/include/ss/exception.hpp b/include/ss/exception.hpp new file mode 100644 index 0000000..4122a1f --- /dev/null +++ b/include/ss/exception.hpp @@ -0,0 +1,30 @@ +#pragma once +#include +#include + +namespace ss { + +//////////////// +// exception +//////////////// + +class exception : public std::exception { + std::string msg_; + +public: + exception(const std::string& msg): msg_{msg} { + } + + virtual char const* what() const noexcept { + return msg_.c_str(); + } +}; + +template +void throw_if_throw_on_error(const std::string& msg) { + if constexpr (throw_on_error) { + throw ss::exception(msg); + } +} + +} /* ss */ diff --git a/include/ss/parser.hpp b/include/ss/parser.hpp index d22cf45..28062ef 100644 --- a/include/ss/parser.hpp +++ b/include/ss/parser.hpp @@ -2,6 +2,7 @@ #include "common.hpp" #include "converter.hpp" +#include "exception.hpp" #include "extract.hpp" #include "restrictions.hpp" #include @@ -12,22 +13,23 @@ namespace ss { -template +template class parser { - constexpr static auto string_error = setup::string_error; + constexpr static auto string_error = setup::string_error; + constexpr static auto throw_on_error = setup::throw_on_error; - using multiline = typename setup::multiline; + using multiline = typename setup::multiline; using error_type = std::conditional_t; constexpr static bool escaped_multiline_enabled = - multiline::enabled && setup::escape::enabled; + multiline::enabled && setup::escape::enabled; constexpr static bool quoted_multiline_enabled = - multiline::enabled && setup::quote::enabled; + multiline::enabled && setup::quote::enabled; - constexpr static bool ignore_header = setup::ignore_header; + constexpr static bool ignore_header = setup::ignore_header; - constexpr static bool ignore_empty = setup::ignore_empty; + constexpr static bool ignore_empty = setup::ignore_empty; public: parser(const std::string& file_name, @@ -152,11 +154,11 @@ public: struct iterable { struct iterator { using value = std::conditional_t>; + no_void_validator_tup_t>; iterator() : parser_{nullptr} { } - iterator(parser* parser) : parser_{parser} { + iterator(parser* parser) : parser_{parser} { } value& operator*() { @@ -197,10 +199,10 @@ public: private: value value_; - parser* parser_; + parser* parser_; }; - iterable(parser* parser) : parser_{parser} { + iterable(parser* parser) : parser_{parser} { } iterator begin() { @@ -211,7 +213,7 @@ public: } private: - parser* parser_; + parser* parser_; }; template @@ -423,7 +425,8 @@ private: void set_error_failed_check() { if constexpr (string_error) { - error_.append(file_name_).append(" failed check."); + error_.append(file_name_).append(" failed check"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -431,7 +434,8 @@ private: void set_error_file_not_open() { if constexpr (string_error) { - error_.append(file_name_).append(" could not be opened."); + error_.append(file_name_).append(" could not be opened"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -439,7 +443,8 @@ private: void set_error_eof_reached() { if constexpr (string_error) { - error_.append(file_name_).append(" reached end of file."); + error_.append(file_name_).append(" reached end of file"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -455,6 +460,7 @@ private: .append(": \"") .append(reader_.buffer_) .append("\""); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -467,6 +473,7 @@ private: .append("the header row is ignored within the setup, it cannot " "be used") .append("\""); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -477,6 +484,7 @@ private: error_.append(file_name_) .append(": header does not contain given field: ") .append(field); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -487,6 +495,7 @@ private: error_.append(file_name_) .append(": given field used multiple times: ") .append(field); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -650,7 +659,7 @@ private: const char* curr; for (curr = next_line_buffer_ + size - 1; curr >= next_line_buffer_ && - setup::escape::match(*curr); + setup::escape::match(*curr); --curr) { } return (next_line_buffer_ - curr + size) % 2 == 0; @@ -729,8 +738,8 @@ private: char* next_line_buffer_{nullptr}; char* helper_buffer_{nullptr}; - converter converter_; - converter next_line_converter_; + converter converter_; + converter next_line_converter_; size_t size_{0}; size_t next_line_size_{0}; diff --git a/include/ss/setup.hpp b/include/ss/setup.hpp index 62da91b..7fd8ce4 100644 --- a/include/ss/setup.hpp +++ b/include/ss/setup.hpp @@ -179,20 +179,26 @@ class ignore_header; class ignore_empty; +//////////////// +// throw_on_error +//////////////// + +class throw_on_error; + //////////////// // setup implementation //////////////// -template +template struct setup { private: - template + template struct is_matcher - : std::disjunction, - is_instance_of_matcher_t, - is_instance_of_matcher_t, - is_instance_of_matcher_t, - is_instance_of_matcher_t> {}; + : std::disjunction, + is_instance_of_matcher_t, + is_instance_of_matcher_t, + is_instance_of_matcher_t, + is_instance_of_matcher_t> {}; template struct is_string_error : std::is_same {}; @@ -203,39 +209,47 @@ private: template struct is_ignore_empty : std::is_same {}; - constexpr static auto count_matcher = count_v; + template + struct is_throw_on_error : std::is_same {}; + + constexpr static auto count_matcher = count_v; constexpr static auto count_multiline = - count_v; + count_v; - constexpr static auto count_string_error = count_v; + constexpr static auto count_string_error = count_v; constexpr static auto count_ignore_header = - count_v; + count_v; - constexpr static auto count_ignore_empty = count_v; + constexpr static auto count_throw_on_error = + count_v; + + constexpr static auto count_ignore_empty = count_v; constexpr static auto number_of_valid_setup_types = count_matcher + count_multiline + count_string_error + - count_ignore_header + count_ignore_empty; + count_ignore_header + count_ignore_empty + count_throw_on_error; - using trim_left_only = get_matcher_t; - using trim_right_only = get_matcher_t; - using trim_all = get_matcher_t; + using trim_left_only = get_matcher_t; + using trim_right_only = get_matcher_t; + using trim_all = get_matcher_t; public: - using quote = get_matcher_t; - using escape = get_matcher_t; + using quote = get_matcher_t; + using escape = get_matcher_t; using trim_left = std::conditional_t; using trim_right = std::conditional_t; - using multiline = get_multiline_t; + using multiline = get_multiline_t; constexpr static bool string_error = (count_string_error == 1); constexpr static bool ignore_header = (count_ignore_header == 1); constexpr static bool ignore_empty = (count_ignore_empty == 1); + constexpr static bool throw_on_error = (count_throw_on_error == 1); + // TODO set string_error if throw_on_error is defined private: #define ASSERT_MSG "cannot have the same match character in multiple matchers" @@ -264,11 +278,11 @@ private: static_assert(count_string_error <= 1, "string_error defined multiple times"); - static_assert(number_of_valid_setup_types == sizeof...(Ts), + static_assert(number_of_valid_setup_types == sizeof...(Options), "one or multiple invalid setup parameters defined"); }; -template -struct setup> : setup {}; +template +struct setup> : setup {}; } /* ss */ diff --git a/include/ss/splitter.hpp b/include/ss/splitter.hpp index dc2e0e7..02c8a21 100644 --- a/include/ss/splitter.hpp +++ b/include/ss/splitter.hpp @@ -1,5 +1,6 @@ #pragma once #include "common.hpp" +#include "exception.hpp" #include "setup.hpp" #include "type_traits.hpp" #include @@ -11,16 +12,17 @@ namespace ss { -template +template class splitter { private: - using quote = typename setup::quote; - using trim_left = typename setup::trim_left; - using trim_right = typename setup::trim_right; - using escape = typename setup::escape; - using multiline = typename setup::multiline; + using quote = typename setup::quote; + using trim_left = typename setup::trim_left; + using trim_right = typename setup::trim_right; + using escape = typename setup::escape; + using multiline = typename setup::multiline; - constexpr static auto string_error = setup::string_error; + constexpr static auto string_error = setup::string_error; + constexpr static auto throw_on_error = setup::throw_on_error; constexpr static auto is_const_line = !quote::enabled && !escape::enabled; using error_type = std::conditional_t; @@ -121,7 +123,8 @@ private: void set_error_empty_delimiter() { if constexpr (string_error) { error_.clear(); - error_.append("empt delimiter"); + error_.append("empty delimiter"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -131,6 +134,7 @@ private: if constexpr (string_error) { error_.clear(); error_.append("mismatched quote at position: " + std::to_string(n)); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -140,6 +144,7 @@ private: if constexpr (string_error) { error_.clear(); error_.append("unterminated escape at the end of the line"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -150,6 +155,7 @@ private: if constexpr (string_error) { error_.clear(); error_.append("unterminated quote"); + throw_if_throw_on_error(error_); } else { error_ = true; } @@ -161,6 +167,7 @@ private: error_.clear(); error_.append("invalid resplit, new line must be longer" "than the end of the last slice"); + throw_if_throw_on_error(error_); } else { error_ = true; } From f3225b8b1658f77f8fad82165c2a01d4ac5853a2 Mon Sep 17 00:00:00 2001 From: ado Date: Sun, 9 Jul 2023 12:54:39 +0200 Subject: [PATCH 02/64] Add splitter tests with throw_on_error --- include/ss/splitter.hpp | 1 + test/test_splitter.cpp | 375 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 356 insertions(+), 20 deletions(-) diff --git a/include/ss/splitter.hpp b/include/ss/splitter.hpp index 02c8a21..027a7ff 100644 --- a/include/ss/splitter.hpp +++ b/include/ss/splitter.hpp @@ -150,6 +150,7 @@ private: } } + // TODO handle this efficiently void set_error_unterminated_quote() { unterminated_quote_ = true; if constexpr (string_error) { diff --git a/test/test_splitter.cpp b/test/test_splitter.cpp index 14caded..ecf2a22 100644 --- a/test/test_splitter.cpp +++ b/test/test_splitter.cpp @@ -5,15 +5,15 @@ #include namespace { -constexpr static auto combinations_size_default = 4; -size_t combinations_size = combinations_size_default; +constexpr static auto num_combinations_default = 4; +size_t num_combinations = num_combinations_default; -struct set_combinations_size { - set_combinations_size(size_t size) { - combinations_size = size; +struct set_num_combinations { + set_num_combinations(size_t size) { + num_combinations = size; } - ~set_combinations_size() { - combinations_size = combinations_size_default; + ~set_num_combinations() { + num_combinations = num_combinations_default; } }; @@ -153,7 +153,7 @@ make_combinations(const std::vector& input, const std::string& delim) { std::vector lines; std::vector> expectations; - for (size_t i = 0; i < combinations_size; ++i) { + for (size_t i = 0; i < num_combinations; ++i) { auto l = combinations(input, delim, i); lines.reserve(lines.size() + l.size()); lines.insert(lines.end(), l.begin(), l.end()); @@ -176,6 +176,8 @@ template void test_combinations(matches_type& matches, std::vector delims) { ss::splitter s; + ss::splitter st; + std::vector inputs; std::vector outputs; for (const auto& [cases, e] : matches) { @@ -194,6 +196,13 @@ void test_combinations(matches_type& matches, std::vector delims) { auto vec = s.split(buff(lines[i].c_str()), delim); CHECK(s.valid()); CHECK_EQ(words(vec), expectations[i]); + + try { + auto vec = st.split(buff(lines[i].c_str()), delim); + CHECK_EQ(words(vec), expectations[i]); + } catch (ss::exception& e) { + FAIL(e.what()); + } } } } @@ -253,7 +262,7 @@ TEST_CASE("splitter test with quote") { } TEST_CASE("splitter test with trim") { - auto guard = set_combinations_size(3); + auto guard = set_num_combinations(3); case_type case1 = spaced({R"(x)"}, " "); case_type case2 = spaced({R"(yy)"}, " "); case_type case3 = spaced({R"(y y)"}, " "); @@ -320,7 +329,7 @@ TEST_CASE("splitter test with escape") { } TEST_CASE("splitter test with quote and trim") { - auto guard = set_combinations_size(3); + auto guard = set_num_combinations(3); case_type case1 = spaced({R"("""")"}, " "); case_type case2 = spaced({R"("x""x")", R"(x"x)"}, " "); case_type case3 = spaced({R"("")", R"()"}, " "); @@ -438,7 +447,7 @@ TEST_CASE("splitter test with escape and trim") { } TEST_CASE("splitter test with quote and escape and trim") { - auto guard = set_combinations_size(3); + auto guard = set_num_combinations(3); case_type case1 = spaced({R"("\"")", R"(\")", R"("""")"}, " "); case_type case2 = spaced({R"("x\"x")", R"(x\"x)", R"(x"x)", R"("x""x")"}, " "); @@ -509,6 +518,18 @@ TEST_CASE("splitter test error mode") { CHECK_FALSE(s.valid()); CHECK_FALSE(s.unterminated_quote()); CHECK_FALSE(s.error_msg().empty()); + + try { + // TODO remove ss::string_error + ss::splitter s; + s.split(buff("just,some,strings"), ""); + FAIL("expected exception"); + } catch (ss::exception& e) { + CHECK_EQ(std::string{e.what()}, s.error_msg()); + CHECK_FALSE(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + CHECK_FALSE(s.error_msg().empty()); + } } { @@ -523,10 +544,16 @@ TEST_CASE("splitter test error mode") { template auto expect_unterminated_quote(Splitter& s, const std::string& line) { - auto vec = s.split(buff(line.c_str())); - CHECK_FALSE(s.valid()); - CHECK(s.unterminated_quote()); - return vec; + try { + auto vec = s.split(buff(line.c_str())); + CHECK_FALSE(s.valid()); + CHECK(s.unterminated_quote()); + return vec; + } catch (ss::exception& e) { + // TODO check if this is ok + FAIL(e.what()); + return decltype(s.split(buff(line.c_str()))){}; + } } namespace ss { @@ -550,7 +577,9 @@ TEST_CASE("splitter test resplit unterminated quote") { { ss::converter, ss::multiline, ss::escape<'\\'>> c; auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"("x)"); + CHECK_EQ(vec.size(), 1); REQUIRE(s.unterminated_quote()); @@ -770,6 +799,270 @@ TEST_CASE("splitter test resplit unterminated quote") { } } +TEST_CASE("splitter test resplit unterminated quote with exceptions") { + + try { + ss::converter, ss::multiline, ss::escape<'\\'>, + ss::throw_on_error> + c; + auto& s = c.splitter; + + auto vec = expect_unterminated_quote(s, R"("x)"); + + CHECK_EQ(vec.size(), 1); + + REQUIRE(s.unterminated_quote()); + + { + auto new_linet = + buff.append_overwrite_last(R"(a\x)", c.size_shifted()); + + vec = c.resplit(new_linet, strlen(new_linet)); + + CHECK(s.unterminated_quote()); + CHECK_EQ(vec.size(), 1); + } + + { + auto new_linet = + buff.append_overwrite_last(R"(")", c.size_shifted()); + + vec = c.resplit(new_linet, strlen(new_linet)); + REQUIRE(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + REQUIRE_EQ(vec.size(), 1); + CHECK_EQ(words(vec)[0], "xax"); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::multiline, ss::throw_on_error> c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, "\"just"); + CHECK_EQ(vec.size(), 1); + + auto new_line = buff.append(R"(",strings)"); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + std::vector expected{"just", "strings"}; + CHECK_EQ(words(vec), expected); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::multiline, ss::throw_on_error> c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, "just,some,\"random"); + std::vector expected{"just", "some", "just,some,\""}; + CHECK_EQ(words(vec), expected); + + auto new_line = buff.append(R"(",strings)"); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"just", "some", "random", "strings"}; + CHECK_EQ(words(vec), expected); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::multiline, ss::throw_on_error> c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"("just","some","ran"")"); + std::vector expected{"just", "some", R"("just","some",")"}; + CHECK_EQ(words(vec), expected); + + auto new_line = + buff.append_overwrite_last(R"(,dom","strings")", c.size_shifted()); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"just", "some", "ran\",dom", "strings"}; + CHECK_EQ(words(vec), expected); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::multiline, ss::throw_on_error> c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"("just","some","ran)"); + std::vector expected{"just", "some", R"("just","some",")"}; + CHECK_EQ(words(vec), expected); + REQUIRE(s.unterminated_quote()); + + { + auto new_line = buff.append(R"(,dom)"); + vec = c.resplit(new_line, strlen(new_line)); + CHECK_FALSE(s.valid()); + CHECK(s.unterminated_quote()); + CHECK_EQ(words(vec), expected); + } + + { + auto new_line = buff.append(R"(",strings)"); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"just", "some", "ran,dom", "strings"}; + CHECK_EQ(words(vec), expected); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::escape<'\\'>, ss::multiline, + ss::throw_on_error> + c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"("just\"some","ra)"); + std::vector expected{"just\"some"}; + auto w = words(vec); + w.pop_back(); + CHECK_EQ(w, expected); + REQUIRE(s.unterminated_quote()); + { + auto new_line = buff.append(R"(n,dom",str\"ings)"); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"just\"some", "ran,dom", "str\"ings"}; + CHECK_EQ(words(vec), expected); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::escape<'\\'>, ss::multiline, + ss::throw_on_error> + c; + auto& s = c.splitter; + auto vec = + expect_unterminated_quote(s, "3,4," + "\"just0\\\n1\\\n22\\\n33333x\\\n4"); + + std::vector expected{"3", "4"}; + auto w = words(vec); + w.pop_back(); + CHECK_EQ(w, expected); + REQUIRE(s.unterminated_quote()); + { + auto new_line = + buff.append_overwrite_last("\nx5strings\"", c.size_shifted()); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"3", "4", "just0\n1\n22\n33333x\n4\nx5strings"}; + CHECK_EQ(words(vec), expected); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::escape<'\\'>, ss::multiline, + ss::throw_on_error> + c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"("just\"some","ra"")"); + std::vector expected{"just\"some"}; + auto w = words(vec); + w.pop_back(); + CHECK_EQ(w, expected); + REQUIRE(s.unterminated_quote()); + { + auto new_line = buff.append_overwrite_last(R"(n,dom",str\"ings)", + c.size_shifted()); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"just\"some", "ra\"n,dom", "str\"ings"}; + CHECK_EQ(words(vec), expected); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::escape<'\\'>, ss::multiline, + ss::throw_on_error> + c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"("just\"some","r\a\a\\\a\")"); + std::vector expected{"just\"some"}; + auto w = words(vec); + w.pop_back(); + CHECK_EQ(w, expected); + REQUIRE(s.unterminated_quote()); + { + auto new_line = buff.append_overwrite_last(R"(n,dom",str\"ings)", + c.size_shifted()); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"just\"some", "raa\\a\"n,dom", "str\"ings"}; + CHECK_EQ(words(vec), expected); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::trim<' '>, ss::multiline, + ss::throw_on_error> + c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"( "just" ,some, "ra )"); + std::vector expected{"just", "some"}; + auto w = words(vec); + w.pop_back(); + CHECK_EQ(w, expected); + REQUIRE(s.unterminated_quote()); + { + auto new_line = buff.append(R"( n,dom" , strings )"); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"just", "some", "ra n,dom", "strings"}; + CHECK_EQ(words(vec), expected); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, ss::trim<' '>, ss::escape<'\\'>, + ss::multiline> + c; + auto& s = c.splitter; + auto vec = expect_unterminated_quote(s, R"( "ju\"st" ,some, "ra \")"); + std::vector expected{"ju\"st", "some"}; + auto w = words(vec); + w.pop_back(); + CHECK_EQ(w, expected); + REQUIRE(s.unterminated_quote()); + { + auto new_line = + buff.append_overwrite_last(R"( n,dom" , strings )", + c.size_shifted()); + vec = c.resplit(new_line, strlen(new_line)); + CHECK(s.valid()); + CHECK_FALSE(s.unterminated_quote()); + expected = {"ju\"st", "some", "ra \" n,dom", "strings"}; + CHECK_EQ(words(vec), expected); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("splitter test invalid splits") { ss::converter, ss::trim<' '>, ss::escape<'\\'>> @@ -808,14 +1101,55 @@ TEST_CASE("splitter test invalid splits") { // invalid resplit char new_line[] = "some"; - auto a = c.resplit(new_line, strlen(new_line)); + c.resplit(new_line, strlen(new_line)); CHECK_FALSE(s.valid()); CHECK_FALSE(s.unterminated_quote()); CHECK_FALSE(s.error_msg().empty()); } +#define CHECK_EXCEPTION(CLASS, OPERATION) \ + try { \ + OPERATION; \ + } catch (ss::exception & e) { \ + CHECK_FALSE(CLASS.valid()); \ + CHECK_FALSE(CLASS.error_msg().empty()); \ + CHECK_EQ(CLASS.error_msg(), std::string{e.what()}); \ + } + +TEST_CASE("splitter test invalid splits with exceptions") { + ss::converter, ss::trim<' '>, + ss::escape<'\\'>, ss::throw_on_error> + c; + auto& s = c.splitter; + + // empty delimiter + CHECK_EXCEPTION(s, s.split(buff("some,random,strings"), "")); + CHECK_FALSE(s.unterminated_quote()); + + // mismatched delimiter + CHECK_EXCEPTION(s, s.split(buff(R"(some,"random,"strings")"))); + CHECK_FALSE(s.unterminated_quote()); + + // unterminated escape + CHECK_EXCEPTION(s, s.split(buff(R"(some,random,strings\)"))); + CHECK_FALSE(s.unterminated_quote()); + + // unterminated escape + CHECK_EXCEPTION(s, s.split(buff(R"(some,random,"strings\)"))); + CHECK_FALSE(s.unterminated_quote()); + + // unterminated quote + CHECK_EXCEPTION(s, s.split(buff("some,random,\"strings"))); + CHECK(s.unterminated_quote()); + + // invalid resplit + char new_line[] = "some"; + CHECK_EXCEPTION(s, c.resplit(new_line, strlen(new_line))); + CHECK_FALSE(s.unterminated_quote()); +} + TEST_CASE("splitter test with trim_left") { - auto guard = set_combinations_size(3); + auto guard = set_num_combinations(3); case_type case1 = spaced_left({R"(x )"}, " "); case_type case2 = spaced_left({R"(yy )"}, " "); case_type case3 = spaced_left({R"(y y )"}, " "); @@ -845,7 +1179,7 @@ TEST_CASE("splitter test with trim_left") { } TEST_CASE("splitter test with trim_right") { - auto guard = set_combinations_size(3); + auto guard = set_num_combinations(3); case_type case1 = spaced_right({R"( x)"}, " "); case_type case2 = spaced_right({R"( yy)"}, " "); case_type case3 = spaced_right({R"( y y)"}, " "); @@ -876,7 +1210,7 @@ TEST_CASE("splitter test with trim_right") { } TEST_CASE("splitter test with trim_right and trim_left") { - auto guard = set_combinations_size(3); + auto guard = set_num_combinations(3); case_type case1 = spaced_right({R"(-x)"}, "-"); case_type case2 = spaced_left({R"(yy_)"}, "_"); case_type case3 = spaced_right({R"(-y y)"}, "-"); @@ -894,7 +1228,7 @@ TEST_CASE("splitter test with trim_right and trim_left") { } TEST_CASE("splitter test with quote and escape, trim_left and trim_right") { - auto guard = set_combinations_size(3); + auto guard = set_num_combinations(3); case_type case1 = spaced_left({R"("\"")", R"(\")", R"("""")"}, "_"); case_type case2 = spaced_left({R"("x\"x")", R"(x\"x)", R"(x"x)", R"("x""x")"}, "_"); @@ -914,3 +1248,4 @@ TEST_CASE("splitter test with quote and escape, trim_left and trim_right") { ss::trim_right<'-'>>(p, delims); } } + From 41b89d1d3db427745c0cc5feb24a9d7a9462bf03 Mon Sep 17 00:00:00 2001 From: ado Date: Sun, 9 Jul 2023 17:11:52 +0200 Subject: [PATCH 03/64] Add converter tests with throw_on_error --- include/ss/converter.hpp | 12 +- include/ss/setup.hpp | 1 + include/ss/splitter.hpp | 1 + test/test_converter.cpp | 504 ++++++++++++++++++++++++++++++++++++++- test/test_helpers.hpp | 10 + test/test_splitter.cpp | 26 +- 6 files changed, 527 insertions(+), 27 deletions(-) diff --git a/include/ss/converter.hpp b/include/ss/converter.hpp index a9e3156..ea911fd 100644 --- a/include/ss/converter.hpp +++ b/include/ss/converter.hpp @@ -121,7 +121,12 @@ public: no_void_validator_tup_t convert( line_ptr_type line, const std::string& delim = default_delimiter) { split(line, delim); - return convert(splitter_.split_data_); + if (splitter_.valid()) { + return convert(splitter_.split_data_); + } else { + set_error_bad_split(); + return {}; + } } // parses already split line, returns 'T' object with extracted values @@ -227,7 +232,7 @@ private: return error; } - void set_error_unterminated_quote() { + void set_error_bad_split() { if constexpr (string_error) { error_.clear(); error_.append(splitter_.error_msg()); @@ -340,10 +345,11 @@ private: template no_void_validator_tup_t convert_impl(const split_data& elems) { clear_error(); + // TODO check if this is needed using return_type = no_void_validator_tup_t; if (!splitter_.valid()) { - set_error_unterminated_quote(); + set_error_bad_split(); no_void_validator_tup_t ret{}; return ret; } diff --git a/include/ss/setup.hpp b/include/ss/setup.hpp index 7fd8ce4..e80da45 100644 --- a/include/ss/setup.hpp +++ b/include/ss/setup.hpp @@ -250,6 +250,7 @@ public: constexpr static bool ignore_empty = (count_ignore_empty == 1); constexpr static bool throw_on_error = (count_throw_on_error == 1); // TODO set string_error if throw_on_error is defined + // TODO throw_on_error should be unique private: #define ASSERT_MSG "cannot have the same match character in multiple matchers" diff --git a/include/ss/splitter.hpp b/include/ss/splitter.hpp index 027a7ff..af04d9d 100644 --- a/include/ss/splitter.hpp +++ b/include/ss/splitter.hpp @@ -163,6 +163,7 @@ private: } void set_error_invalid_resplit() { + // TODO check this unterminated_quote_ = false; if constexpr (string_error) { error_.clear(); diff --git a/test/test_converter.cpp b/test/test_converter.cpp index 24a9987..2dbdbca 100644 --- a/test/test_converter.cpp +++ b/test/test_converter.cpp @@ -22,6 +22,30 @@ TEST_CASE("converter test split") { } } +TEST_CASE("converter test split with exceptions") { + ss::converter c; + try { + for (const auto& [s, expected, delim] : + // clang-format off + {std::make_tuple("a,b,c,d", std::vector{"a", "b", "c", "d"}, ","), + {"", {}, " "}, + {" x x x x | x ", {" x x x x ", " x "}, "|"}, + {"a::b::c::d", {"a", "b", "c", "d"}, "::"}, + {"x\t-\ty", {"x", "y"}, "\t-\t"}, + {"x", {"x"}, ","}} // clang-format on + ) { + auto split = c.split(s, delim); + CHECK_EQ(split.size(), expected.size()); + for (size_t i = 0; i < split.size(); ++i) { + auto s = std::string(split[i].first, split[i].second); + CHECK_EQ(s, expected[i]); + } + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("converter test valid conversions") { ss::converter c; @@ -116,12 +140,153 @@ TEST_CASE("converter test valid conversions") { } } +TEST_CASE("converter test valid conversions with exceptions") { + ss::converter c; + + try { + auto tup = c.convert("5"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 5); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("5,junk"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 5); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("junk,5"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 5); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("5\njunk\njunk", "\n"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 5); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("junk 5 junk", " "); + REQUIRE(c.valid()); + CHECK_EQ(tup, 5); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("junk\tjunk\t5", "\t"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 5); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = + c.convert>("junk\tjunk\t5", "\t"); + REQUIRE(c.valid()); + REQUIRE(tup.has_value()); + CHECK_EQ(tup, 5); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("5,6.6,junk"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(5, 6.6)); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("5,junk,6.6"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(5, 6.6)); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("junk;5;6.6", ";"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(5, 6.6)); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = + c.convert, double>("junk;5;6.6", ";"); + REQUIRE(c.valid()); + REQUIRE(std::get<0>(tup).has_value()); + CHECK_EQ(tup, std::make_tuple(5, 6.6)); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = + c.convert, double>("junk;5.4;6.6", ";"); + REQUIRE(c.valid()); + REQUIRE_FALSE(std::get<0>(tup).has_value()); + CHECK_EQ(tup, std::make_tuple(std::optional{}, 6.6)); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = + c.convert, double>("junk;5;6.6", + ";"); + REQUIRE(c.valid()); + REQUIRE(std::holds_alternative(std::get<0>(tup))); + CHECK_EQ(tup, std::make_tuple(std::variant{5}, 6.6)); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = + c.convert, double>("junk;5.5;6.6", + ";"); + REQUIRE(c.valid()); + REQUIRE(std::holds_alternative(std::get<0>(tup))); + CHECK_EQ(tup, std::make_tuple(std::variant{5.5}, 6.6)); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + auto tup = c.convert("junk;s1;6.6;s2", ";"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(std::string_view{"s1"}, 6.6, + std::string_view{"s2"})); + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("converter test invalid conversions") { ss::converter c; c.convert(""); REQUIRE_FALSE(c.valid()); + c.convert("1", ""); + REQUIRE_FALSE(c.valid()); + c.convert("10", ""); REQUIRE_FALSE(c.valid()); @@ -150,6 +315,24 @@ TEST_CASE("converter test invalid conversions") { REQUIRE_FALSE(c.valid()); } +TEST_CASE("converter test invalid conversions with exceptions") { + // TODO remove ss::string_error + ss::converter c; + + REQUIRE_EXCEPTION(c.convert("")); + REQUIRE_EXCEPTION(c.convert("1", "")); + REQUIRE_EXCEPTION(c.convert("10", "")); + REQUIRE_EXCEPTION(c.convert("")); + REQUIRE_EXCEPTION(c.convert(",junk")); + REQUIRE_EXCEPTION(c.convert("junk,")); + REQUIRE_EXCEPTION(c.convert("x")); + REQUIRE_EXCEPTION(c.convert("x")); + REQUIRE_EXCEPTION(c.convert("x,junk")); + REQUIRE_EXCEPTION(c.convert("junk,x")); + REQUIRE_EXCEPTION( + c.convert, double>("junk;.5.5;6", ";")); +} + TEST_CASE("converter test ss:ax restriction (all except)") { ss::converter c; @@ -181,6 +364,33 @@ TEST_CASE("converter test ss:ax restriction (all except)") { } } +TEST_CASE("converter test ss:ax restriction (all except) with exceptions") { + // TODO remove ss::string_error + ss::converter c; + + REQUIRE_EXCEPTION(c.convert>("0")); + REQUIRE_EXCEPTION(c.convert>("1")); + REQUIRE_EXCEPTION(c.convert>("junk,c,1")); + REQUIRE_EXCEPTION(c.convert, char>("1,c")); + + try { + { + int tup = c.convert>("3"); + CHECK_EQ(tup, 3); + } + { + std::tuple tup = c.convert>("c,3"); + CHECK_EQ(tup, std::make_tuple('c', 3)); + } + { + std::tuple tup = c.convert, char>("3,c"); + CHECK_EQ(tup, std::make_tuple(3, 'c')); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("converter test ss:nx restriction (none except)") { ss::converter c; @@ -215,6 +425,40 @@ TEST_CASE("converter test ss:nx restriction (none except)") { } } +TEST_CASE("converter test ss:nx restriction (none except) with exceptions") { + // TODO remove ss::string_error + ss::converter c; + + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("c,3")); + REQUIRE_EXCEPTION(c.convert, char>("3,c")); + + try { + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + { + auto tup = c.convert>("2"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 2); + } + { + auto tup = c.convert>("c,junk,1"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple('c', 1)); + } + { + auto tup = c.convert, char>("1,c"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(1, 'c')); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("converter test ss:ir restriction (in range)") { ss::converter c; @@ -249,6 +493,40 @@ TEST_CASE("converter test ss:ir restriction (in range)") { } } +TEST_CASE("converter test ss:ir restriction (in range) with exceptions") { + // TODO remove ss::string_error + ss::converter c; + + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("c,3")); + REQUIRE_EXCEPTION(c.convert, char>("3,c")); + + try { + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + { + auto tup = c.convert>("2"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 2); + } + { + auto tup = c.convert>("c,junk,1"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple('c', 1)); + } + { + auto tup = c.convert, char>("1,c"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(1, 'c')); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("converter test ss:oor restriction (out of range)") { ss::converter c; @@ -283,6 +561,38 @@ TEST_CASE("converter test ss:oor restriction (out of range)") { } } +TEST_CASE("converter test ss:oor restriction (out of range) with exceptions") { + // TODO remove ss::string_error + ss::converter c; + + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("2")); + REQUIRE_EXCEPTION(c.convert, void>("c,1,junk")); + REQUIRE_EXCEPTION(c.convert, char>("1,c")); + + try { + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + + { + auto tup = c.convert>("c,junk,3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple('c', 3)); + } + + { + auto tup = c.convert, char>("3,c"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(3, 'c')); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + const std::vector extracted_vector = {1, 2, 3}; // custom extract @@ -331,6 +641,38 @@ TEST_CASE("converter test ss:ne restriction (not empty)") { } } +TEST_CASE("converter test ss:ne restriction (not empty) with exceptions") { + // TODO remove ss::string_error + ss::converter c; + + REQUIRE_EXCEPTION(c.convert>("")); + REQUIRE_EXCEPTION(c.convert>("3,")); + REQUIRE_EXCEPTION(c.convert, int>(",3")); + REQUIRE_EXCEPTION(c.convert, int>("junk,,3")); + REQUIRE_EXCEPTION(c.convert>>("")); + + try { + { + auto tup = c.convert>("s"); + REQUIRE(c.valid()); + CHECK_EQ(tup, "s"); + } + { + auto tup = + c.convert, ss::ne>("1,s"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple(1, "s")); + } + { + auto tup = c.convert>>("{1 2 3}"); + REQUIRE(c.valid()); + CHECK_EQ(tup, extracted_vector); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE( "converter test ss:lt ss::lte ss::gt ss::gte restriction (in range)") { ss::converter c; @@ -390,6 +732,59 @@ TEST_CASE( } } +TEST_CASE("converter test ss:lt ss::lte ss::gt ss::gte restriction (in range) " + "with exception") { + // TODO remove ss::string_error + ss::converter c; + + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("3")); + REQUIRE_EXCEPTION(c.convert>("3")); + + try { + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + + { + auto tup = c.convert>("3"); + REQUIRE(c.valid()); + CHECK_EQ(tup, 3); + } + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("converter test error mode") { ss::converter c; c.convert("junk"); @@ -397,6 +792,12 @@ TEST_CASE("converter test error mode") { CHECK_FALSE(c.error_msg().empty()); } +TEST_CASE("converter test throw on error mode") { + // TODO remove ss::string_error + ss::converter c; + REQUIRE_EXCEPTION(c.convert("junk")); +} + TEST_CASE("converter test converter with quotes spacing and escaping") { { ss::converter c; @@ -444,6 +845,67 @@ TEST_CASE("converter test converter with quotes spacing and escaping") { } } +TEST_CASE("converter test converter with quotes spacing and escaping with " + "exceptions") { + // TODO remove ss::string_error on all below + try { + ss::converter c; + + auto tup = c.convert( + R"("just","some","strings")"); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple("\"just\"", "\"some\"", "\"strings\"")); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter> c; + + auto tup = c.convert( + buff(R"("just",some,"12.3","a")")); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple("just", "some", 12.3, 'a')); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter> c; + + auto tup = c.convert( + buff(R"( just , some , 12.3 ,a )")); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple("just", "some", 12.3, 'a')); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter> c; + + auto tup = + c.convert(buff(R"(ju\,st,strings)")); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple("ju,st", "strings")); + } catch (ss::exception& e) { + FAIL(e.what()); + } + + try { + ss::converter, + ss::trim<' '>, ss::quote<'"'>> + c; + + auto tup = c.convert( + buff(R"( ju\,st , "so,me" , 12.34 , "str""ings")")); + REQUIRE(c.valid()); + CHECK_EQ(tup, std::make_tuple("ju,st", "so,me", 12.34, "str\"ings")); + } catch (ss::exception& e) { + FAIL(e.what()); + } +} + TEST_CASE("converter test invalid split conversions") { ss::converter, ss::trim<' '>, ss::quote<'"'>> @@ -451,7 +913,7 @@ TEST_CASE("converter test invalid split conversions") { { // mismatched quote - auto tup = c.convert( + c.convert( buff(R"( "just , some , "12.3","a" )")); CHECK_FALSE(c.valid()); CHECK_FALSE(c.unterminated_quote()); @@ -460,7 +922,7 @@ TEST_CASE("converter test invalid split conversions") { { // unterminated quote - auto tup = c.convert( + c.convert( buff(R"( ju\,st , "so,me" , 12.34 , "str""ings)")); CHECK_FALSE(c.valid()); CHECK(c.unterminated_quote()); @@ -469,7 +931,7 @@ TEST_CASE("converter test invalid split conversions") { { // unterminated escape - auto tup = c.convert( + c.convert( buff(R"(just,some,2,strings\)")); CHECK_FALSE(c.valid()); CHECK_FALSE(c.unterminated_quote()); @@ -478,7 +940,7 @@ TEST_CASE("converter test invalid split conversions") { { // unterminated escape while quoting - auto tup = c.convert( + c.convert( buff(R"(just,some,2,"strings\)")); CHECK_FALSE(c.valid()); CHECK_FALSE(c.unterminated_quote()); @@ -487,10 +949,42 @@ TEST_CASE("converter test invalid split conversions") { { // unterminated escaped quote - auto tup = c.convert( + c.convert( buff(R"(just,some,2,"strings\")")); CHECK_FALSE(c.valid()); CHECK(c.unterminated_quote()); CHECK_FALSE(c.error_msg().empty()); } } + +TEST_CASE("converter test invalid split conversions with exceptions") { + // TODO remove ss::string_error + ss::converter, ss::trim<' '>, + ss::quote<'"'>, ss::throw_on_error> + c; + + // mismatched quote + REQUIRE_EXCEPTION(c.convert( + buff(R"( "just , some , "12.3","a" )"))); + CHECK_FALSE(c.unterminated_quote()); + + // unterminated quote + REQUIRE_EXCEPTION(c.convert( + buff(R"( ju\,st , "so,me" , 12.34 , "str""ings)"))); + CHECK(c.unterminated_quote()); + + // unterminated escape + REQUIRE_EXCEPTION(c.convert( + buff(R"(just,some,2,strings\)"))); + CHECK_FALSE(c.unterminated_quote()); + + // unterminated escape while quoting + REQUIRE_EXCEPTION(c.convert( + buff(R"(just,some,2,"strings\)"))); + CHECK_FALSE(c.unterminated_quote()); + + // unterminated escaped quote + REQUIRE_EXCEPTION(c.convert( + buff(R"(just,some,2,"strings\")"))); + CHECK(c.unterminated_quote()); +} diff --git a/test/test_helpers.hpp b/test/test_helpers.hpp index 06ea703..8cab8bf 100644 --- a/test/test_helpers.hpp +++ b/test/test_helpers.hpp @@ -79,3 +79,13 @@ struct buffer { } #define CHECK_NOT_VARIANT(var, type) CHECK(!std::holds_alternative(var)); +// TODO remove +#include + +#define REQUIRE_EXCEPTION(...) \ + try { \ + __VA_ARGS__; \ + FAIL("Expected exception"); \ + } catch (ss::exception & e) { \ + CHECK_FALSE(std::string{e.what()}.empty()); \ + } diff --git a/test/test_splitter.cpp b/test/test_splitter.cpp index ecf2a22..4a75f6b 100644 --- a/test/test_splitter.cpp +++ b/test/test_splitter.cpp @@ -525,10 +525,8 @@ TEST_CASE("splitter test error mode") { s.split(buff("just,some,strings"), ""); FAIL("expected exception"); } catch (ss::exception& e) { - CHECK_EQ(std::string{e.what()}, s.error_msg()); - CHECK_FALSE(s.valid()); + CHECK_FALSE(std::string{e.what()}.empty()); CHECK_FALSE(s.unterminated_quote()); - CHECK_FALSE(s.error_msg().empty()); } } @@ -1107,15 +1105,6 @@ TEST_CASE("splitter test invalid splits") { CHECK_FALSE(s.error_msg().empty()); } -#define CHECK_EXCEPTION(CLASS, OPERATION) \ - try { \ - OPERATION; \ - } catch (ss::exception & e) { \ - CHECK_FALSE(CLASS.valid()); \ - CHECK_FALSE(CLASS.error_msg().empty()); \ - CHECK_EQ(CLASS.error_msg(), std::string{e.what()}); \ - } - TEST_CASE("splitter test invalid splits with exceptions") { ss::converter, ss::trim<' '>, ss::escape<'\\'>, ss::throw_on_error> @@ -1123,28 +1112,28 @@ TEST_CASE("splitter test invalid splits with exceptions") { auto& s = c.splitter; // empty delimiter - CHECK_EXCEPTION(s, s.split(buff("some,random,strings"), "")); + REQUIRE_EXCEPTION(s.split(buff("some,random,strings"), "")); CHECK_FALSE(s.unterminated_quote()); // mismatched delimiter - CHECK_EXCEPTION(s, s.split(buff(R"(some,"random,"strings")"))); + REQUIRE_EXCEPTION(s.split(buff(R"(some,"random,"strings")"))); CHECK_FALSE(s.unterminated_quote()); // unterminated escape - CHECK_EXCEPTION(s, s.split(buff(R"(some,random,strings\)"))); + REQUIRE_EXCEPTION(s.split(buff(R"(some,random,strings\)"))); CHECK_FALSE(s.unterminated_quote()); // unterminated escape - CHECK_EXCEPTION(s, s.split(buff(R"(some,random,"strings\)"))); + REQUIRE_EXCEPTION(s.split(buff(R"(some,random,"strings\)"))); CHECK_FALSE(s.unterminated_quote()); // unterminated quote - CHECK_EXCEPTION(s, s.split(buff("some,random,\"strings"))); + REQUIRE_EXCEPTION(s.split(buff("some,random,\"strings"))); CHECK(s.unterminated_quote()); // invalid resplit char new_line[] = "some"; - CHECK_EXCEPTION(s, c.resplit(new_line, strlen(new_line))); + REQUIRE_EXCEPTION(c.resplit(new_line, strlen(new_line))); CHECK_FALSE(s.unterminated_quote()); } @@ -1248,4 +1237,3 @@ TEST_CASE("splitter test with quote and escape, trim_left and trim_right") { ss::trim_right<'-'>>(p, delims); } } - From a7a97b3ba8b614dd2387a5417a187769d31a63a1 Mon Sep 17 00:00:00 2001 From: ado Date: Mon, 10 Jul 2023 02:39:24 +0200 Subject: [PATCH 04/64] Make throw_on_error and string_error separate options, update parser to have line reading a separate function with respect to splitting --- include/ss/converter.hpp | 81 +++++++++++++------ include/ss/exception.hpp | 7 -- include/ss/parser.hpp | 163 +++++++++++++++++++++++++-------------- include/ss/setup.hpp | 19 +++-- include/ss/splitter.hpp | 71 +++++++++++------ test/test_converter.cpp | 85 ++++++++++---------- test/test_parser.cpp | 1 + test/test_splitter.cpp | 40 +++++----- 8 files changed, 285 insertions(+), 182 deletions(-) diff --git a/include/ss/converter.hpp b/include/ss/converter.hpp index ea911fd..ca12fc4 100644 --- a/include/ss/converter.hpp +++ b/include/ss/converter.hpp @@ -236,8 +236,7 @@ private: if constexpr (string_error) { error_.clear(); error_.append(splitter_.error_msg()); - throw_if_throw_on_error(error_); - } else { + } else if constexpr (!throw_on_error) { error_ = true; } } @@ -247,28 +246,46 @@ private: error_.clear(); splitter_.set_error_unterminated_escape(); error_.append(splitter_.error_msg()); - throw_if_throw_on_error(error_); + } else if constexpr (throw_on_error) { + splitter_.set_error_unterminated_escape(); + } else { + error_ = true; + } + } + + void set_error_unterminated_quote() { + if constexpr (string_error) { + error_.clear(); + splitter_.set_error_unterminated_quote(); + error_.append(splitter_.error_msg()); + } else if constexpr (throw_on_error) { + splitter_.set_error_unterminated_quote(); } else { error_ = true; } } void set_error_multiline_limit_reached() { + constexpr static auto error_msg = "multiline limit reached"; + if constexpr (string_error) { error_.clear(); - error_.append("multiline limit reached"); - throw_if_throw_on_error(error_); + error_.append(error_msg); + } else if constexpr (throw_on_error) { + throw ss::exception{error_msg}; } else { error_ = true; } } void set_error_invalid_conversion(const string_range msg, size_t pos) { + constexpr static auto error_msg = "invalid conversion for parameter "; + if constexpr (string_error) { error_.clear(); - error_.append("invalid conversion for parameter ") - .append(error_sufix(msg, pos)); - throw_if_throw_on_error(error_); + error_.append(error_msg).append(error_sufix(msg, pos)); + } else if constexpr (throw_on_error) { + throw ss::exception{error_msg + error_sufix(msg, pos)}; } else { error_ = true; } @@ -279,20 +296,27 @@ private: if constexpr (string_error) { error_.clear(); error_.append(error).append(" ").append(error_sufix(msg, pos)); - throw_if_throw_on_error(error_); + } else if constexpr (throw_on_error) { + throw ss::exception{error + (" " + error_sufix(msg, pos))}; } else { error_ = true; } } void set_error_number_of_columns(size_t expected_pos, size_t pos) { + constexpr static auto error_msg1 = + "invalid number of columns, expected: "; + constexpr static auto error_msg2 = ", got: "; + if constexpr (string_error) { error_.clear(); - error_.append("invalid number of columns, expected: ") + error_.append(error_msg1) .append(std::to_string(expected_pos)) - .append(", got: ") + .append(error_msg2) .append(std::to_string(pos)); - throw_if_throw_on_error(error_); + } else if constexpr (throw_on_error) { + throw ss::exception{error_msg1 + std::to_string(expected_pos) + + error_msg2 + std::to_string(pos)}; } else { error_ = true; } @@ -300,25 +324,32 @@ private: void set_error_incompatible_mapping(size_t argument_size, size_t mapping_size) { + constexpr static auto error_msg1 = + "number of arguments does not match mapping, expected: "; + constexpr static auto error_msg2 = ", got: "; + if constexpr (string_error) { error_.clear(); - error_ - .append( - "number of arguments does not match mapping, expected: ") + error_.append(error_msg1) .append(std::to_string(mapping_size)) - .append(", got: ") + .append(error_msg2) .append(std::to_string(argument_size)); - throw_if_throw_on_error(error_); + } else if constexpr (throw_on_error) { + throw ss::exception{error_msg1 + std::to_string(mapping_size) + + error_msg2 + std::to_string(argument_size)}; } else { error_ = true; } } void set_error_invalid_mapping() { + constexpr static auto error_msg = "received empty mapping"; + if constexpr (string_error) { error_.clear(); - error_.append("received empty mapping"); - throw_if_throw_on_error(error_); + error_.append(error_msg); + } else if constexpr (throw_on_error) { + throw ss::exception{error_msg}; } else { error_ = true; } @@ -326,13 +357,19 @@ private: void set_error_mapping_out_of_range(size_t maximum_index, size_t number_of_columnts) { + constexpr static auto error_msg1 = "maximum index: "; + constexpr static auto error_msg2 = ", greater than number of columns: "; + if constexpr (string_error) { error_.clear(); - error_.append("maximum index: ") + error_.append(error_msg1) .append(std::to_string(maximum_index)) - .append(", greater then number of columns: ") + .append(error_msg2) .append(std::to_string(number_of_columnts)); - throw_if_throw_on_error(error_); + } else if constexpr (throw_on_error) { + throw ss::exception{error_msg1 + std::to_string(maximum_index) + + error_msg2 + + std::to_string(number_of_columnts)}; } else { error_ = true; } diff --git a/include/ss/exception.hpp b/include/ss/exception.hpp index 4122a1f..97c714a 100644 --- a/include/ss/exception.hpp +++ b/include/ss/exception.hpp @@ -20,11 +20,4 @@ public: } }; -template -void throw_if_throw_on_error(const std::string& msg) { - if constexpr (throw_on_error) { - throw ss::exception(msg); - } -} - } /* ss */ diff --git a/include/ss/parser.hpp b/include/ss/parser.hpp index 28062ef..1485324 100644 --- a/include/ss/parser.hpp +++ b/include/ss/parser.hpp @@ -87,8 +87,19 @@ public: template no_void_validator_tup_t get_next() { + if (!eof_) { + reader_.parse(); + } + reader_.update(); + if (!reader_.converter_.valid()) { + set_error_invalid_conversion(); + read_line(); + return {}; + } + clear_error(); + if (eof_) { set_error_eof_reached(); return {}; @@ -424,27 +435,36 @@ private: } void set_error_failed_check() { + constexpr static auto error_msg = " failed check"; + if constexpr (string_error) { - error_.append(file_name_).append(" failed check"); - throw_if_throw_on_error(error_); + error_.append(file_name_).append(error_msg); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg}; } else { error_ = true; } } void set_error_file_not_open() { + constexpr static auto error_msg = " could not be opened"; + if constexpr (string_error) { error_.append(file_name_).append(" could not be opened"); - throw_if_throw_on_error(error_); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg}; } else { error_ = true; } } void set_error_eof_reached() { + constexpr static auto error_msg = " reached end of file"; + if constexpr (string_error) { error_.append(file_name_).append(" reached end of file"); - throw_if_throw_on_error(error_); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg}; } else { error_ = true; } @@ -460,42 +480,45 @@ private: .append(": \"") .append(reader_.buffer_) .append("\""); - throw_if_throw_on_error(error_); - } else { + } else if constexpr (!throw_on_error) { error_ = true; } } void set_error_header_ignored() { + constexpr static auto error_msg = + ": \"the header row is ignored within the setup it cannot be " + "used\""; + if constexpr (string_error) { - error_.append(file_name_) - .append(": \"") - .append("the header row is ignored within the setup, it cannot " - "be used") - .append("\""); - throw_if_throw_on_error(error_); + error_.append(file_name_).append(error_msg); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg}; } else { error_ = true; } } void set_error_invalid_field(const std::string& field) { + constexpr static auto error_msg = + ": header does not contain given field: "; + if constexpr (string_error) { - error_.append(file_name_) - .append(": header does not contain given field: ") - .append(field); - throw_if_throw_on_error(error_); + error_.append(file_name_).append(error_msg).append(field); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg + field}; } else { error_ = true; } } void set_error_field_used_multiple_times(const std::string& field) { + constexpr static auto error_msg = ": given field used multiple times: "; + if constexpr (string_error) { - error_.append(file_name_) - .append(": given field used multiple times: ") - .append(field); - throw_if_throw_on_error(error_); + error_.append(file_name_).append(error_msg).append(field); + } else if constexpr (throw_on_error) { + throw ss::exception{file_name_ + error_msg + field}; } else { error_ = true; } @@ -514,13 +537,15 @@ private: : delim_{delim}, file_{fopen(file_name_.c_str(), "rb")} { } + // TODO update for size_ and ssize_ reader(reader&& other) : buffer_{other.buffer_}, next_line_buffer_{other.next_line_buffer_}, helper_buffer_{other.helper_buffer_}, converter_{std::move( other.converter_)}, next_line_converter_{std::move(other.next_line_converter_)}, - size_{other.size_}, next_line_size_{other.next_line_size_}, + buffer_size_{other.size_}, + next_line_buffer_size_{other.next_line_buffer_size_}, helper_size_{other.helper_size_}, delim_{std::move(other.delim_)}, file_{other.file_}, crlf_{other.crlf_}, line_number_{ other.line_number_} { @@ -537,8 +562,8 @@ private: helper_buffer_ = other.helper_buffer_; converter_ = std::move(other.converter_); next_line_converter_ = std::move(other.next_line_converter_); - size_ = other.size_; - next_line_size_ = other.next_line_size_; + buffer_size_ = other.size_; + next_line_buffer_size_ = other.next_line_buffer_size_; helper_size_ = other.helper_size_; delim_ = std::move(other.delim_); file_ = other.file_; @@ -568,16 +593,18 @@ private: reader(const reader& other) = delete; reader& operator=(const reader& other) = delete; + // read next line each time in order to set eof_ bool read_next() { - - ssize_t ssize; + next_line_converter_.clear_error(); + ssize_t ssize = 0; size_t size = 0; while (size == 0) { ++line_number_; - if (next_line_size_ > 0) { + if (next_line_buffer_size_ > 0) { next_line_buffer_[0] = '\0'; } - ssize = get_line(&next_line_buffer_, &next_line_size_, file_); + ssize = get_line(&next_line_buffer_, &next_line_buffer_size_, + file_); if (ssize == -1) { return false; @@ -590,17 +617,24 @@ private: } } + size_ = size; + ssize_ = ssize; + return true; + } + + void parse() { size_t limit = 0; if constexpr (escaped_multiline_enabled) { - while (escaped_eol(size)) { + while (escaped_eol(size_)) { if (multiline_limit_reached(limit)) { - return true; + return; } - if (!append_next_line_to_buffer(next_line_buffer_, size)) { - remove_eol(next_line_buffer_, ssize); + + if (!append_next_line_to_buffer(next_line_buffer_, size_)) { + // remove_eol(next_line_buffer_, ssize_); next_line_converter_.set_error_unterminated_escape(); - return true; + return; } } } @@ -610,38 +644,40 @@ private: if constexpr (quoted_multiline_enabled) { while (unterminated_quote()) { if (multiline_limit_reached(limit)) { - return true; + return; } - if (!append_next_line_to_buffer(next_line_buffer_, size)) { - remove_eol(next_line_buffer_, ssize); - return true; + + if (!append_next_line_to_buffer(next_line_buffer_, size_)) { + // remove_eol(next_line_buffer_, ssize_); + next_line_converter_.set_error_unterminated_quote(); + return; } if constexpr (escaped_multiline_enabled) { - while (escaped_eol(size)) { + while (escaped_eol(size_)) { if (multiline_limit_reached(limit)) { - return true; + return; } + if (!append_next_line_to_buffer(next_line_buffer_, - size)) { - remove_eol(next_line_buffer_, ssize); + size_)) { + // TODO not needed + // remove_eol(next_line_buffer_, ssize_); next_line_converter_ .set_error_unterminated_escape(); - return true; + return; } } } - next_line_converter_.resplit(next_line_buffer_, size); + next_line_converter_.resplit(next_line_buffer_, size_); } } - - return true; } void update() { std::swap(buffer_, next_line_buffer_); - std::swap(size_, next_line_size_); + std::swap(buffer_size_, next_line_buffer_size_); std::swap(converter_, next_line_converter_); } @@ -666,10 +702,7 @@ private: } bool unterminated_quote() { - if (next_line_converter_.unterminated_quote()) { - return true; - } - return false; + return next_line_converter_.unterminated_quote(); } void undo_remove_eol(char* buffer, size_t& string_end) { @@ -685,24 +718,29 @@ private: } } - size_t remove_eol(char*& buffer, size_t size) { - size_t new_size = size - 1; - if (size >= 2 && buffer[size - 2] == '\r') { + size_t remove_eol(char*& buffer, size_t ssize) { + size_t size = ssize - 1; + if (ssize >= 2 && buffer[ssize - 2] == '\r') { crlf_ = true; - new_size--; + size--; } else { crlf_ = false; } - buffer[new_size] = '\0'; - return new_size; + buffer[size] = '\0'; + return size; } void realloc_concat(char*& first, size_t& first_size, const char* const second, size_t second_size) { - next_line_size_ = first_size + second_size + 3; + // TODO make buffer_size an argument !!!!!! + next_line_buffer_size_ = first_size + second_size + 3; first = static_cast( - realloc(static_cast(first), next_line_size_)); + realloc(static_cast(first), next_line_buffer_size_)); + // TODO handle realloc + if (!first) { + exit(EXIT_FAILURE); + } std::copy_n(second, second_size + 1, first + first_size); first_size += second_size; } @@ -722,8 +760,9 @@ private: return true; } - std::vector get_next_row() const { + std::vector get_next_row() { std::vector next_row; + next_line_converter_.split(next_line_buffer_, delim_); auto& next_row_raw = next_line_converter_.splitter_.split_data_; for (const auto& [begin, end] : next_row_raw) { next_row.emplace_back(begin, end); @@ -741,8 +780,8 @@ private: converter converter_; converter next_line_converter_; - size_t size_{0}; - size_t next_line_size_{0}; + size_t buffer_size_{0}; + size_t next_line_buffer_size_{0}; size_t helper_size_{0}; std::string delim_; @@ -750,6 +789,10 @@ private: bool crlf_; size_t line_number_{0}; + + // TODO check if needed + size_t size_{0}; + ssize_t ssize_{0}; }; //////////////// diff --git a/include/ss/setup.hpp b/include/ss/setup.hpp index e80da45..ae2fc7f 100644 --- a/include/ss/setup.hpp +++ b/include/ss/setup.hpp @@ -112,7 +112,7 @@ struct get_matcher { "the same matcher is cannot" "be defined multiple times"); using type = std::conditional_t::value, T, - typename get_matcher::type>; + typename get_matcher::type>; }; template