diff --git a/include/ss/parser.hpp b/include/ss/parser.hpp index 3a4244b..cd5e7b1 100644 --- a/include/ss/parser.hpp +++ b/include/ss/parser.hpp @@ -733,10 +733,10 @@ private: return -1; } - c = buffer[curr_char++]; if (curr_char >= csv_data_size) { return -1; } + c = buffer[curr_char++]; // TODO maybe remove this too if (*lineptr == nullptr) { diff --git a/test/test_parser1_3.cpp b/test/test_parser1_3.cpp index 6885eb1..612d17d 100644 --- a/test/test_parser1_3.cpp +++ b/test/test_parser1_3.cpp @@ -1,6 +1,6 @@ #include "test_parser1.hpp" -template +template void test_multiline_restricted() { unique_file_name f{"test_parser"}; { @@ -23,9 +23,9 @@ void test_multiline_restricted() { auto bad_lines = 15; auto num_errors = 0; - ss::parser, ss::quote<'"'>, ss::escape<'\\'>, - Ts...> - p{f.name, ","}; + auto [p, _] = + make_parser, ss::quote<'"'>, + ss::escape<'\\'>, Ts...>(f.name, ","); std::vector i; while (!p.eof()) { @@ -64,12 +64,15 @@ void test_multiline_restricted() { } TEST_CASE("parser test multiline restricted") { - test_multiline_restricted(); - test_multiline_restricted(); - test_multiline_restricted(); + test_multiline_restricted(); + test_multiline_restricted(); + test_multiline_restricted(); + test_multiline_restricted(); + test_multiline_restricted(); + test_multiline_restricted(); } -template +template void test_unterminated_line_impl(const std::vector& lines, size_t bad_line) { unique_file_name f{"test_parser"}; @@ -79,10 +82,12 @@ void test_unterminated_line_impl(const std::vector& lines, } out.close(); - ss::parser p{f.name}; + auto [p, _] = make_parser(f.name); size_t line = 0; while (!p.eof()) { - auto command = [&] { p.template get_next(); }; + auto command = [&p = p] { + p.template get_next(); + }; if (line == bad_line) { expect_error_on_command(p, command); @@ -98,9 +103,15 @@ void test_unterminated_line_impl(const std::vector& lines, template void test_unterminated_line(const std::vector& lines, size_t bad_line) { - test_unterminated_line_impl(lines, bad_line); - test_unterminated_line_impl(lines, bad_line); - test_unterminated_line_impl(lines, bad_line); + test_unterminated_line_impl(lines, bad_line); + test_unterminated_line_impl(lines, + bad_line); + test_unterminated_line_impl(lines, + bad_line); + test_unterminated_line_impl(lines, bad_line); + test_unterminated_line_impl(lines, bad_line); + test_unterminated_line_impl(lines, + bad_line); } TEST_CASE("parser test csv on multiline with errors") { @@ -317,199 +328,3 @@ TEST_CASE("parser test csv on multiline with errors") { test_unterminated_line(lines, 1); } } - -template -struct has_type; - -template -struct has_type> - : std::disjunction...> {}; - -static inline void check_size(size_t size1, size_t size2) { - CHECK_EQ(size1, size2); -} - -template -static void test_fields_impl(const std::string file_name, - const std::vector& data, - const std::vector& fields) { - using CaseType = std::tuple; - - ss::parser p{file_name, ","}; - CHECK_FALSE(p.field_exists("Unknown")); - p.use_fields(fields); - std::vector i; - - for (const auto& a : p.template iterate()) { - i.push_back(a); - } - - check_size(i.size(), data.size()); - for (size_t j = 0; j < i.size(); ++j) { - if constexpr (has_type::value) { - CHECK_EQ(std::get(i[j]), data[j].i); - } - if constexpr (has_type::value) { - CHECK_EQ(std::get(i[j]), data[j].d); - } - if constexpr (has_type::value) { - CHECK_EQ(std::get(i[j]), data[j].s); - } - } -} - -template -static void test_fields(const std::string file_name, const std::vector& data, - const std::vector& fields) { - test_fields_impl, Ts...>(file_name, data, fields); - test_fields_impl, Ts...>(file_name, data, - fields); - test_fields_impl, Ts...>(file_name, data, - fields); -} - -TEST_CASE("parser test various cases with header") { - unique_file_name f{"test_parser"}; - constexpr static auto Int = "Int"; - constexpr static auto Dbl = "Double"; - constexpr static auto Str = "String"; - using str = std::string; - - std::vector header{Int, Dbl, Str}; - - std::vector data = {{1, 2, "x"}, {3, 4, "y"}, {5, 6, "z"}, - {7, 8, "u"}, {9, 10, "v"}, {11, 12, "w"}}; - - make_and_write(f.name, data, header); - const auto& o = f.name; - const auto& d = data; - - { - ss::parser p{f.name, ","}; - std::vector i; - - for (const auto& a : p.iterate()) { - i.emplace_back(ss::to_object(a)); - } - - CHECK_NE(i, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - p.ignore_next(); - for (const auto& a : p.iterate()) { - i.emplace_back(ss::to_object(a)); - } - - CHECK_EQ(i, data); - } - - { - ss::parser p{f.name, ","}; - std::vector i; - - for (const auto& a : p.iterate()) { - i.emplace_back(ss::to_object(a)); - } - - CHECK_EQ(i, data); - } - - { - ss::parser p{f.name, ","}; - p.use_fields(Int, Dbl, Str); - CHECK_FALSE(p.valid()); - } - - { - ss::parser p{f.name, ","}; - CHECK_FALSE(p.field_exists("Unknown")); - - p.use_fields(Int, "Unknown"); - CHECK_FALSE(p.valid()); - } - - { - ss::parser p{f.name, ","}; - p.use_fields(Int, Int); - CHECK_FALSE(p.valid()); - } - - { - ss::parser p{f.name, ","}; - p.use_fields(Int, Dbl); - - { - auto [int_, double_] = p.get_next(); - CHECK_EQ(int_, data[0].i); - CHECK_EQ(double_, data[0].d); - } - - p.use_fields(Dbl, Int); - - { - auto [double_, int_] = p.get_next(); - CHECK_EQ(int_, data[1].i); - CHECK_EQ(double_, data[1].d); - } - - p.use_fields(Str); - - { - auto string_ = p.get_next(); - CHECK_EQ(string_, data[2].s); - } - - p.use_fields(Str, Int, Dbl); - - { - auto [string_, int_, double_] = - p.get_next(); - CHECK_EQ(double_, data[3].d); - CHECK_EQ(int_, data[3].i); - CHECK_EQ(string_, data[3].s); - } - } - - /* python used to generate permutations - import itertools - - header = {'str': 'Str', - 'double': 'Dbl', - 'int': 'Int'} - - keys = ['str', 'int', 'double'] - - for r in range (1, 3): - combinations = list(itertools.permutations(keys, r = r)) - - for combination in combinations: - template_params = [] - arg_params = [] - for type in combination: - template_params.append(type) - arg_params.append(header[type]) - call = 'testFields<' + ', '.join(template_params) + \ - '>(o, d, {' + ', '.join(arg_params) + '});' - print(call) - */ - - test_fields(o, d, {Str}); - test_fields(o, d, {Int}); - test_fields(o, d, {Dbl}); - test_fields(o, d, {Str, Int}); - test_fields(o, d, {Str, Dbl}); - test_fields(o, d, {Int, Str}); - test_fields(o, d, {Int, Dbl}); - test_fields(o, d, {Dbl, Str}); - test_fields(o, d, {Dbl, Int}); - test_fields(o, d, {Str, Int, Dbl}); - test_fields(o, d, {Str, Dbl, Int}); - test_fields(o, d, {Int, Str, Dbl}); - test_fields(o, d, {Int, Dbl, Str}); - test_fields(o, d, {Dbl, Str, Int}); - test_fields(o, d, {Dbl, Int, Str}); -} diff --git a/test/test_parser1_4.cpp b/test/test_parser1_4.cpp index 42a385b..1045850 100644 --- a/test/test_parser1_4.cpp +++ b/test/test_parser1_4.cpp @@ -1,5 +1,201 @@ #include "test_parser1.hpp" +template +struct has_type; + +template +struct has_type> + : std::disjunction...> {}; + +static inline void check_size(size_t size1, size_t size2) { + CHECK_EQ(size1, size2); +} + +template +static void test_fields_impl(const std::string file_name, + const std::vector& data, + const std::vector& fields) { + using CaseType = std::tuple; + + ss::parser p{file_name, ","}; + CHECK_FALSE(p.field_exists("Unknown")); + p.use_fields(fields); + std::vector i; + + for (const auto& a : p.template iterate()) { + i.push_back(a); + } + + check_size(i.size(), data.size()); + for (size_t j = 0; j < i.size(); ++j) { + if constexpr (has_type::value) { + CHECK_EQ(std::get(i[j]), data[j].i); + } + if constexpr (has_type::value) { + CHECK_EQ(std::get(i[j]), data[j].d); + } + if constexpr (has_type::value) { + CHECK_EQ(std::get(i[j]), data[j].s); + } + } +} + +template +static void test_fields(const std::string file_name, const std::vector& data, + const std::vector& fields) { + test_fields_impl, Ts...>(file_name, data, fields); + test_fields_impl, Ts...>(file_name, data, + fields); + test_fields_impl, Ts...>(file_name, data, + fields); +} + +TEST_CASE("parser test various cases with header") { + unique_file_name f{"test_parser"}; + constexpr static auto Int = "Int"; + constexpr static auto Dbl = "Double"; + constexpr static auto Str = "String"; + using str = std::string; + + std::vector header{Int, Dbl, Str}; + + std::vector data = {{1, 2, "x"}, {3, 4, "y"}, {5, 6, "z"}, + {7, 8, "u"}, {9, 10, "v"}, {11, 12, "w"}}; + + make_and_write(f.name, data, header); + const auto& o = f.name; + const auto& d = data; + + { + ss::parser p{f.name, ","}; + std::vector i; + + for (const auto& a : p.iterate()) { + i.emplace_back(ss::to_object(a)); + } + + CHECK_NE(i, data); + } + + { + ss::parser p{f.name, ","}; + std::vector i; + + p.ignore_next(); + for (const auto& a : p.iterate()) { + i.emplace_back(ss::to_object(a)); + } + + CHECK_EQ(i, data); + } + + { + ss::parser p{f.name, ","}; + std::vector i; + + for (const auto& a : p.iterate()) { + i.emplace_back(ss::to_object(a)); + } + + CHECK_EQ(i, data); + } + + { + ss::parser p{f.name, ","}; + p.use_fields(Int, Dbl, Str); + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{f.name, ","}; + CHECK_FALSE(p.field_exists("Unknown")); + + p.use_fields(Int, "Unknown"); + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{f.name, ","}; + p.use_fields(Int, Int); + CHECK_FALSE(p.valid()); + } + + { + ss::parser p{f.name, ","}; + p.use_fields(Int, Dbl); + + { + auto [int_, double_] = p.get_next(); + CHECK_EQ(int_, data[0].i); + CHECK_EQ(double_, data[0].d); + } + + p.use_fields(Dbl, Int); + + { + auto [double_, int_] = p.get_next(); + CHECK_EQ(int_, data[1].i); + CHECK_EQ(double_, data[1].d); + } + + p.use_fields(Str); + + { + auto string_ = p.get_next(); + CHECK_EQ(string_, data[2].s); + } + + p.use_fields(Str, Int, Dbl); + + { + auto [string_, int_, double_] = + p.get_next(); + CHECK_EQ(double_, data[3].d); + CHECK_EQ(int_, data[3].i); + CHECK_EQ(string_, data[3].s); + } + } + + /* python used to generate permutations + import itertools + + header = {'str': 'Str', + 'double': 'Dbl', + 'int': 'Int'} + + keys = ['str', 'int', 'double'] + + for r in range (1, 3): + combinations = list(itertools.permutations(keys, r = r)) + + for combination in combinations: + template_params = [] + arg_params = [] + for type in combination: + template_params.append(type) + arg_params.append(header[type]) + call = 'testFields<' + ', '.join(template_params) + \ + '>(o, d, {' + ', '.join(arg_params) + '});' + print(call) + */ + + test_fields(o, d, {Str}); + test_fields(o, d, {Int}); + test_fields(o, d, {Dbl}); + test_fields(o, d, {Str, Int}); + test_fields(o, d, {Str, Dbl}); + test_fields(o, d, {Int, Str}); + test_fields(o, d, {Int, Dbl}); + test_fields(o, d, {Dbl, Str}); + test_fields(o, d, {Dbl, Int}); + test_fields(o, d, {Str, Int, Dbl}); + test_fields(o, d, {Str, Dbl, Int}); + test_fields(o, d, {Int, Str, Dbl}); + test_fields(o, d, {Int, Dbl, Str}); + test_fields(o, d, {Dbl, Str, Int}); + test_fields(o, d, {Dbl, Int, Str}); +} + template void test_invalid_fields_impl(const std::vector& lines, const std::vector& fields) {