Compare commits

..

No commits in common. "d4fc2ee5611850a2e2781e90990883daf93977fc" and "dbaa8131e71313ceb3703fb5eec8852de382edd9" have entirely different histories.

18 changed files with 166 additions and 271 deletions

2
.gitignore vendored
View File

@ -6,3 +6,5 @@ build/
hbuild/ hbuild/
subprojects/* subprojects/*
!subprojects/*.wrap !subprojects/*.wrap
ssp.cpp
ssp.bin

View File

@ -12,7 +12,6 @@ using string_range = std::pair<const char*, const char*>;
using split_data = std::vector<string_range>; using split_data = std::vector<string_range>;
constexpr inline auto default_delimiter = ","; constexpr inline auto default_delimiter = ",";
constexpr static auto get_line_initial_buffer_size = 128;
template <bool StringError> template <bool StringError>
inline void assert_string_error_defined() { inline void assert_string_error_defined() {
@ -47,7 +46,7 @@ inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
} }
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
*lineptr = static_cast<char*>(malloc(get_line_initial_buffer_size)); *lineptr = static_cast<char*>(malloc(128));
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
return -1; return -1;
} }
@ -58,8 +57,8 @@ inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
while (c != EOF) { while (c != EOF) {
if (pos + 1 >= *n) { if (pos + 1 >= *n) {
size_t new_size = *n + (*n >> 2); size_t new_size = *n + (*n >> 2);
if (new_size < get_line_initial_buffer_size) { if (new_size < 128) {
new_size = get_line_initial_buffer_size; new_size = 128;
} }
char* new_ptr = static_cast<char*>( char* new_ptr = static_cast<char*>(
realloc(static_cast<void*>(*lineptr), new_size)); realloc(static_cast<void*>(*lineptr), new_size));

View File

@ -52,17 +52,12 @@ public:
const std::string& delim = ss::default_delimiter) const std::string& delim = ss::default_delimiter)
: file_name_{"buffer line"}, : file_name_{"buffer line"},
reader_{csv_data_buffer, csv_data_size, delim} { reader_{csv_data_buffer, csv_data_size, delim} {
if (csv_data_buffer) {
read_line(); read_line();
if constexpr (ignore_header) { if constexpr (ignore_header) {
ignore_next(); ignore_next();
} else { } else {
raw_header_ = reader_.get_buffer(); raw_header_ = reader_.get_buffer();
} }
} else {
handle_error_null_buffer();
eof_ = true;
}
} }
parser(parser&& other) = default; parser(parser&& other) = default;
@ -529,19 +524,6 @@ private:
} }
} }
void handle_error_null_buffer() {
constexpr static auto error_msg = " received null data buffer";
if constexpr (string_error) {
error_.clear();
error_.append(file_name_).append(error_msg);
} else if constexpr (throw_on_error) {
throw ss::exception{file_name_ + error_msg};
} else {
error_ = true;
}
}
void handle_error_file_not_open() { void handle_error_file_not_open() {
constexpr static auto error_msg = " could not be opened"; constexpr static auto error_msg = " could not be opened";
@ -746,14 +728,19 @@ private:
size_t pos; size_t pos;
int c; int c;
// TODO remove check
if (lineptr == nullptr || buffer == nullptr || n == nullptr) {
return -1;
}
if (curr_char >= csv_data_size) { if (curr_char >= csv_data_size) {
return -1; return -1;
} }
c = buffer[curr_char++]; c = buffer[curr_char++];
// TODO maybe remove this too
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
*lineptr = *lineptr = static_cast<char*>(malloc(128));
static_cast<char*>(malloc(get_line_initial_buffer_size));
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
return -1; return -1;
} }
@ -764,11 +751,13 @@ private:
while (curr_char <= csv_data_size) { while (curr_char <= csv_data_size) {
if (pos + 1 >= *n) { if (pos + 1 >= *n) {
size_t new_size = *n + (*n >> 2); size_t new_size = *n + (*n >> 2);
if (new_size < get_line_initial_buffer_size) { // TODO maybe remove this too
new_size = get_line_initial_buffer_size; if (new_size < 128) {
new_size = 128;
} }
char* new_ptr = static_cast<char*>( char* new_ptr = static_cast<char*>(
realloc(static_cast<void*>(*lineptr), new_size)); realloc(static_cast<void*>(*lineptr), new_size));
// TODO check for failed malloc in the callee
if (new_ptr == nullptr) { if (new_ptr == nullptr) {
return -1; return -1;
} }
@ -935,11 +924,11 @@ private:
} }
void realloc_concat(char*& first, size_t& first_size, void realloc_concat(char*& first, size_t& first_size,
size_t& buffer_size, const char* const second, const char* const second, size_t second_size) {
size_t second_size) { // TODO make buffer_size an argument
buffer_size = first_size + second_size + 3; next_line_buffer_size_ = first_size + second_size + 3;
auto new_first = static_cast<char*>( auto new_first = static_cast<char*>(
realloc(static_cast<void*>(first), buffer_size)); realloc(static_cast<void*>(first), next_line_buffer_size_));
if (!first) { if (!first) {
throw std::bad_alloc{}; throw std::bad_alloc{};
} }
@ -969,8 +958,7 @@ private:
++line_number_; ++line_number_;
size_t next_size = remove_eol(helper_buffer_, next_ssize); size_t next_size = remove_eol(helper_buffer_, next_ssize);
realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_, realloc_concat(buffer, size, helper_buffer_, next_size);
next_size);
return true; return true;
} }

View File

@ -109,7 +109,7 @@ struct get_matcher<Matcher, T, Ts...> {
struct is_matcher : is_instance_of_matcher<U, Matcher> {}; struct is_matcher : is_instance_of_matcher<U, Matcher> {};
static_assert(count_v<is_matcher, T, Ts...> <= 1, static_assert(count_v<is_matcher, T, Ts...> <= 1,
"the same matcher cannot " "the same matcher cannot"
"be defined multiple times"); "be defined multiple times");
using type = std::conditional_t<is_matcher<T>::value, T, using type = std::conditional_t<is_matcher<T>::value, T,
typename get_matcher<Matcher, Ts...>::type>; typename get_matcher<Matcher, Ts...>::type>;

53
ssp.hpp
View File

@ -625,7 +625,6 @@ using string_range = std::pair<const char*, const char*>;
using split_data = std::vector<string_range>; using split_data = std::vector<string_range>;
constexpr inline auto default_delimiter = ","; constexpr inline auto default_delimiter = ",";
constexpr static auto get_line_initial_buffer_size = 128;
template <bool StringError> template <bool StringError>
inline void assert_string_error_defined() { inline void assert_string_error_defined() {
@ -660,7 +659,7 @@ inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
} }
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
*lineptr = static_cast<char*>(malloc(get_line_initial_buffer_size)); *lineptr = static_cast<char*>(malloc(128));
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
return -1; return -1;
} }
@ -671,8 +670,8 @@ inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
while (c != EOF) { while (c != EOF) {
if (pos + 1 >= *n) { if (pos + 1 >= *n) {
size_t new_size = *n + (*n >> 2); size_t new_size = *n + (*n >> 2);
if (new_size < get_line_initial_buffer_size) { if (new_size < 128) {
new_size = get_line_initial_buffer_size; new_size = 128;
} }
char* new_ptr = static_cast<char*>( char* new_ptr = static_cast<char*>(
realloc(static_cast<void*>(*lineptr), new_size)); realloc(static_cast<void*>(*lineptr), new_size));
@ -804,7 +803,7 @@ struct get_matcher<Matcher, T, Ts...> {
struct is_matcher : is_instance_of_matcher<U, Matcher> {}; struct is_matcher : is_instance_of_matcher<U, Matcher> {};
static_assert(count_v<is_matcher, T, Ts...> <= 1, static_assert(count_v<is_matcher, T, Ts...> <= 1,
"the same matcher cannot " "the same matcher cannot"
"be defined multiple times"); "be defined multiple times");
using type = std::conditional_t<is_matcher<T>::value, T, using type = std::conditional_t<is_matcher<T>::value, T,
typename get_matcher<Matcher, Ts...>::type>; typename get_matcher<Matcher, Ts...>::type>;
@ -2179,17 +2178,12 @@ public:
const std::string& delim = ss::default_delimiter) const std::string& delim = ss::default_delimiter)
: file_name_{"buffer line"}, : file_name_{"buffer line"},
reader_{csv_data_buffer, csv_data_size, delim} { reader_{csv_data_buffer, csv_data_size, delim} {
if (csv_data_buffer) {
read_line(); read_line();
if constexpr (ignore_header) { if constexpr (ignore_header) {
ignore_next(); ignore_next();
} else { } else {
raw_header_ = reader_.get_buffer(); raw_header_ = reader_.get_buffer();
} }
} else {
handle_error_null_buffer();
eof_ = true;
}
} }
parser(parser&& other) = default; parser(parser&& other) = default;
@ -2656,19 +2650,6 @@ private:
} }
} }
void handle_error_null_buffer() {
constexpr static auto error_msg = " received null data buffer";
if constexpr (string_error) {
error_.clear();
error_.append(file_name_).append(error_msg);
} else if constexpr (throw_on_error) {
throw ss::exception{file_name_ + error_msg};
} else {
error_ = true;
}
}
void handle_error_file_not_open() { void handle_error_file_not_open() {
constexpr static auto error_msg = " could not be opened"; constexpr static auto error_msg = " could not be opened";
@ -2873,14 +2854,19 @@ private:
size_t pos; size_t pos;
int c; int c;
// TODO remove check
if (lineptr == nullptr || buffer == nullptr || n == nullptr) {
return -1;
}
if (curr_char >= csv_data_size) { if (curr_char >= csv_data_size) {
return -1; return -1;
} }
c = buffer[curr_char++]; c = buffer[curr_char++];
// TODO maybe remove this too
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
*lineptr = *lineptr = static_cast<char*>(malloc(128));
static_cast<char*>(malloc(get_line_initial_buffer_size));
if (*lineptr == nullptr) { if (*lineptr == nullptr) {
return -1; return -1;
} }
@ -2891,11 +2877,13 @@ private:
while (curr_char <= csv_data_size) { while (curr_char <= csv_data_size) {
if (pos + 1 >= *n) { if (pos + 1 >= *n) {
size_t new_size = *n + (*n >> 2); size_t new_size = *n + (*n >> 2);
if (new_size < get_line_initial_buffer_size) { // TODO maybe remove this too
new_size = get_line_initial_buffer_size; if (new_size < 128) {
new_size = 128;
} }
char* new_ptr = static_cast<char*>( char* new_ptr = static_cast<char*>(
realloc(static_cast<void*>(*lineptr), new_size)); realloc(static_cast<void*>(*lineptr), new_size));
// TODO check for failed malloc in the callee
if (new_ptr == nullptr) { if (new_ptr == nullptr) {
return -1; return -1;
} }
@ -3062,11 +3050,11 @@ private:
} }
void realloc_concat(char*& first, size_t& first_size, void realloc_concat(char*& first, size_t& first_size,
size_t& buffer_size, const char* const second, const char* const second, size_t second_size) {
size_t second_size) { // TODO make buffer_size an argument
buffer_size = first_size + second_size + 3; next_line_buffer_size_ = first_size + second_size + 3;
auto new_first = static_cast<char*>( auto new_first = static_cast<char*>(
realloc(static_cast<void*>(first), buffer_size)); realloc(static_cast<void*>(first), next_line_buffer_size_));
if (!first) { if (!first) {
throw std::bad_alloc{}; throw std::bad_alloc{};
} }
@ -3096,8 +3084,7 @@ private:
++line_number_; ++line_number_;
size_t next_size = remove_eol(helper_buffer_, next_ssize); size_t next_size = remove_eol(helper_buffer_, next_ssize);
realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_, realloc_concat(buffer, size, helper_buffer_, next_size);
next_size);
return true; return true;
} }

View File

@ -35,8 +35,8 @@ enable_testing()
foreach(name IN ITEMS test_splitter test_parser1_1 test_parser1_2 foreach(name IN ITEMS test_splitter test_parser1_1 test_parser1_2
test_parser1_3 test_parser1_4 test_converter test_parser1_3 test_parser1_4 test_converter
test_extractions test_parser2_1 test_parser2_2 test_extractions test_parser2_1 test_parser2_2
test_parser2_3 test_parser2_4 test_parser2_5 test_parser2_3 test_parser2_4
test_parser2_6 test_extractions_without_fast_float) test_extractions_without_fast_float)
add_executable("${name}" "${name}.cpp") add_executable("${name}" "${name}.cpp")
target_link_libraries("${name}" PRIVATE ssp::ssp fast_float target_link_libraries("${name}" PRIVATE ssp::ssp fast_float
doctest::doctest) doctest::doctest)

View File

@ -13,8 +13,6 @@ tests = [
'parser2_2', 'parser2_2',
'parser2_3', 'parser2_3',
'parser2_4', 'parser2_4',
'parser2_5',
'parser2_6',
'extractions_without_fast_float', 'extractions_without_fast_float',
] ]

View File

@ -12,11 +12,6 @@
#include <doctest.h> #include <doctest.h>
#endif #endif
namespace ss {
template <typename... Ts>
class parser;
} /* ss */
namespace { namespace {
struct buffer { struct buffer {
std::string data_; std::string data_;
@ -39,7 +34,7 @@ struct buffer {
[[maybe_unused]] inline buffer buff; [[maybe_unused]] inline buffer buff;
[[maybe_unused]] std::string time_now_rand() { std::string time_now_rand() {
srand(time(nullptr)); srand(time(nullptr));
std::stringstream ss; std::stringstream ss;
auto t = std::time(nullptr); auto t = std::time(nullptr);
@ -120,8 +115,8 @@ struct unique_file_name {
} }
template <typename T> template <typename T>
[[maybe_unused]] std::vector<std::vector<T>> vector_combinations( std::vector<std::vector<T>> vector_combinations(const std::vector<T>& v,
const std::vector<T>& v, size_t n) { size_t n) {
std::vector<std::vector<T>> ret; std::vector<std::vector<T>> ret;
if (n <= 1) { if (n <= 1) {
for (const auto& i : v) { for (const auto& i : v) {
@ -139,54 +134,4 @@ template <typename T>
} }
return ret; return ret;
} }
[[maybe_unused]] std::string make_buffer(const std::string& file_name) {
std::ifstream in{file_name, std::ios::binary};
std::string tmp;
std::string out;
auto copy_if_whitespaces = [&] {
std::string matches = "\n\r\t ";
while (std::any_of(matches.begin(), matches.end(),
[&](auto c) { return in.peek() == c; })) {
if (in.peek() == '\r') {
out += "\r\n";
in.ignore(2);
} else {
out += std::string{static_cast<char>(in.peek())};
in.ignore(1);
}
}
};
out.reserve(sizeof(out) + 1);
copy_if_whitespaces();
while (in >> tmp) {
out += tmp;
copy_if_whitespaces();
}
return out;
}
template <bool buffer_mode, typename... Ts>
[[maybe_unused]] std::tuple<ss::parser<Ts...>, std::string> make_parser(
const std::string& file_name, const std::string& delim = "") {
if (buffer_mode) {
auto buffer = make_buffer(file_name);
if (delim.empty()) {
return {ss::parser<Ts...>{buffer.data(), buffer.size()},
std::move(buffer)};
} else {
return {ss::parser<Ts...>{buffer.data(), buffer.size(), delim},
std::move(buffer)};
}
} else {
if (delim.empty()) {
return {ss::parser<Ts...>{file_name}, std::string{}};
} else {
return {ss::parser<Ts...>{file_name, delim}, std::string{}};
}
}
}
} /* namespace */ } /* namespace */

View File

@ -105,4 +105,47 @@ static void make_and_write(const std::string& file_name,
} }
} }
std::string make_buffer(const std::string& file_name) {
std::ifstream in{file_name, std::ios::binary};
std::string tmp;
std::string out;
out.reserve(sizeof(out) + 1);
while (in >> tmp) {
out += tmp;
std::string matches = "\n\r\t ";
while (std::any_of(matches.begin(), matches.end(),
[&](auto c) { return in.peek() == c; })) {
if (in.peek() == '\r') {
out += "\r\n";
in.ignore(2);
} else {
out += std::string{static_cast<char>(in.peek())};
in.ignore(1);
}
}
}
return out;
}
template <bool buffer_mode, typename... Ts>
std::tuple<ss::parser<Ts...>, std::string> make_parser(
const std::string& file_name, const std::string& delim = "") {
if (buffer_mode) {
auto buffer = make_buffer(file_name);
if (delim.empty()) {
return {ss::parser<Ts...>{buffer.data(), buffer.size()},
std::move(buffer)};
} else {
return {ss::parser<Ts...>{buffer.data(), buffer.size(), delim},
std::move(buffer)};
}
} else {
if (delim.empty()) {
return {ss::parser<Ts...>{file_name}, std::string{}};
} else {
return {ss::parser<Ts...>{file_name, delim}, std::string{}};
}
}
}
} /* namespace */ } /* namespace */

View File

@ -21,25 +21,6 @@ TEST_CASE("test file not found") {
} }
} }
TEST_CASE("test null buffer") {
{
ss::parser p{nullptr, 10, ","};
CHECK_FALSE(p.valid());
}
{
ss::parser<ss::string_error> p{nullptr, 10, ","};
CHECK_FALSE(p.valid());
}
try {
ss::parser<ss::throw_on_error> p{nullptr, 10, ","};
FAIL("Expected exception...");
} catch (const std::exception& e) {
CHECK_FALSE(std::string{e.what()}.empty());
}
}
template <bool buffer_mode, typename... Ts> template <bool buffer_mode, typename... Ts>
void test_various_cases() { void test_various_cases() {
unique_file_name f{"test_parser"}; unique_file_name f{"test_parser"};

View File

@ -7,13 +7,17 @@ template <typename T, typename... Us>
struct has_type<T, std::tuple<Us...>> struct has_type<T, std::tuple<Us...>>
: std::disjunction<std::is_same<T, Us>...> {}; : std::disjunction<std::is_same<T, Us>...> {};
template <bool buffer_mode, typename Setup, typename... Ts> static inline void check_size(size_t size1, size_t size2) {
CHECK_EQ(size1, size2);
}
template <typename Setup, typename... Ts>
static void test_fields_impl(const std::string file_name, static void test_fields_impl(const std::string file_name,
const std::vector<X>& data, const std::vector<X>& data,
const std::vector<std::string>& fields) { const std::vector<std::string>& fields) {
using CaseType = std::tuple<Ts...>; using CaseType = std::tuple<Ts...>;
auto [p, _] = make_parser<buffer_mode, Setup>(file_name, ","); ss::parser<Setup> p{file_name, ","};
CHECK_FALSE(p.field_exists("Unknown")); CHECK_FALSE(p.field_exists("Unknown"));
p.use_fields(fields); p.use_fields(fields);
std::vector<CaseType> i; std::vector<CaseType> i;
@ -22,7 +26,7 @@ static void test_fields_impl(const std::string file_name,
i.push_back(a); i.push_back(a);
} }
CHECK_EQ(i.size(), data.size()); check_size(i.size(), data.size());
for (size_t j = 0; j < i.size(); ++j) { for (size_t j = 0; j < i.size(); ++j) {
if constexpr (has_type<int, CaseType>::value) { if constexpr (has_type<int, CaseType>::value) {
CHECK_EQ(std::get<int>(i[j]), data[j].i); CHECK_EQ(std::get<int>(i[j]), data[j].i);
@ -39,16 +43,11 @@ static void test_fields_impl(const std::string file_name,
template <typename... Ts> template <typename... Ts>
static void test_fields(const std::string file_name, const std::vector<X>& data, static void test_fields(const std::string file_name, const std::vector<X>& data,
const std::vector<std::string>& fields) { const std::vector<std::string>& fields) {
test_fields_impl<false, ss::setup<>, Ts...>(file_name, data, fields); test_fields_impl<ss::setup<>, Ts...>(file_name, data, fields);
test_fields_impl<false, ss::setup<ss::string_error>, Ts...>(file_name, data, test_fields_impl<ss::setup<ss::string_error>, Ts...>(file_name, data,
fields); fields);
test_fields_impl<false, ss::setup<ss::throw_on_error>, Ts...>(file_name, test_fields_impl<ss::setup<ss::throw_on_error>, Ts...>(file_name, data,
data, fields);
test_fields_impl<true, ss::setup<>, Ts...>(file_name, data, fields);
test_fields_impl<true, ss::setup<ss::string_error>, Ts...>(file_name, data,
fields); fields);
test_fields_impl<true, ss::setup<ss::throw_on_error>, Ts...>(file_name,
data, fields);
} }
TEST_CASE("parser test various cases with header") { TEST_CASE("parser test various cases with header") {
@ -197,37 +196,34 @@ TEST_CASE("parser test various cases with header") {
test_fields<double, int, str>(o, d, {Dbl, Int, Str}); test_fields<double, int, str>(o, d, {Dbl, Int, Str});
} }
template <bool buffer_mode, typename... Ts> template <typename... Ts>
void test_invalid_fields_impl(const std::vector<std::string>& lines, void test_invalid_fields_impl(const std::vector<std::string>& lines,
const std::vector<std::string>& fields) { const std::vector<std::string>& fields) {
unique_file_name f{"test_parser"}; unique_file_name f{"test_parser"};
{
std::ofstream out{f.name}; std::ofstream out{f.name};
for (const auto& line : lines) { for (const auto& line : lines) {
out << line << std::endl; out << line << std::endl;
} }
} out.close();
{ {
// No fields specified // No fields specified
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ","); ss::parser<Ts...> p{f.name, ","};
auto command = [&p = p] { p.use_fields(); }; auto command = [&] { p.use_fields(); };
expect_error_on_command(p, command); expect_error_on_command(p, command);
} }
{ {
// Unknown field // Unknown field
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ","); ss::parser<Ts...> p{f.name, ","};
auto command = [&p = p] { p.use_fields("Unknown"); }; auto command = [&] { p.use_fields("Unknown"); };
expect_error_on_command(p, command); expect_error_on_command(p, command);
} }
{ {
// Field used multiple times // Field used multiple times
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ","); ss::parser<Ts...> p{f.name, ","};
auto command = [&p = p, &fields = fields] { auto command = [&] { p.use_fields(fields.at(0), fields.at(0)); };
p.use_fields(fields.at(0), fields.at(0));
};
if (!fields.empty()) { if (!fields.empty()) {
expect_error_on_command(p, command); expect_error_on_command(p, command);
} }
@ -235,8 +231,8 @@ void test_invalid_fields_impl(const std::vector<std::string>& lines,
{ {
// Mapping out of range // Mapping out of range
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ","); ss::parser<Ts...> p{f.name, ","};
auto command = [&p = p, &fields = fields] { auto command = [&] {
p.use_fields(fields.at(0)); p.use_fields(fields.at(0));
p.template get_next<std::string, std::string>(); p.template get_next<std::string, std::string>();
}; };
@ -247,8 +243,8 @@ void test_invalid_fields_impl(const std::vector<std::string>& lines,
{ {
// Invalid header // Invalid header
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ","); ss::parser<Ts...> p{f.name, ","};
auto command = [&p = p, &fields = fields] { p.use_fields(fields); }; auto command = [&] { p.use_fields(fields); };
if (!fields.empty()) { if (!fields.empty()) {
// Pass if there are no duplicates, fail otherwise // Pass if there are no duplicates, fail otherwise
@ -271,12 +267,9 @@ void test_invalid_fields_impl(const std::vector<std::string>& lines,
template <typename... Ts> template <typename... Ts>
void test_invalid_fields(const std::vector<std::string>& lines, void test_invalid_fields(const std::vector<std::string>& lines,
const std::vector<std::string>& fields) { const std::vector<std::string>& fields) {
test_invalid_fields_impl<false>(lines, fields); test_invalid_fields_impl(lines, fields);
test_invalid_fields_impl<false, ss::string_error>(lines, fields); test_invalid_fields_impl<ss::string_error>(lines, fields);
test_invalid_fields_impl<false, ss::throw_on_error>(lines, fields); test_invalid_fields_impl<ss::throw_on_error>(lines, fields);
test_invalid_fields_impl<true>(lines, fields);
test_invalid_fields_impl<true, ss::string_error>(lines, fields);
test_invalid_fields_impl<true, ss::throw_on_error>(lines, fields);
} }
TEST_CASE("parser test invalid header fields usage") { TEST_CASE("parser test invalid header fields usage") {
@ -303,7 +296,7 @@ TEST_CASE("parser test invalid header fields usage") {
test_invalid_fields({"Int,String,Int", "1,hi,3"}, {"Int", "String", "Int"}); test_invalid_fields({"Int,String,Int", "1,hi,3"}, {"Int", "String", "Int"});
} }
template <bool buffer_mode, typename... Ts> template <typename... Ts>
void test_invalid_rows_with_header() { void test_invalid_rows_with_header() {
unique_file_name f{"test_parser"}; unique_file_name f{"test_parser"};
{ {
@ -318,7 +311,7 @@ void test_invalid_rows_with_header() {
} }
{ {
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name); ss::parser<Ts...> p{f.name};
p.use_fields("Int", "String", "Double"); p.use_fields("Int", "String", "Double");
using data = std::tuple<int, std::string, double>; using data = std::tuple<int, std::string, double>;
@ -344,7 +337,7 @@ void test_invalid_rows_with_header() {
} }
{ {
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name); ss::parser<Ts...> p{f.name};
p.use_fields("Double", "Int"); p.use_fields("Double", "Int");
using data = std::tuple<double, int>; using data = std::tuple<double, int>;
@ -368,7 +361,7 @@ void test_invalid_rows_with_header() {
} }
{ {
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name); ss::parser<Ts...> p{f.name};
p.use_fields("String", "Double"); p.use_fields("String", "Double");
using data = std::tuple<std::string, double>; using data = std::tuple<std::string, double>;
@ -396,15 +389,12 @@ void test_invalid_rows_with_header() {
} }
TEST_CASE("parser test invalid rows with header") { TEST_CASE("parser test invalid rows with header") {
test_invalid_rows_with_header<false>(); test_invalid_rows_with_header();
test_invalid_rows_with_header<false, ss::string_error>(); test_invalid_rows_with_header<ss::string_error>();
test_invalid_rows_with_header<false, ss::throw_on_error>(); test_invalid_rows_with_header<ss::throw_on_error>();
test_invalid_rows_with_header<true>();
test_invalid_rows_with_header<true, ss::string_error>();
test_invalid_rows_with_header<true, ss::throw_on_error>();
} }
template <bool buffer_mode, typename... Ts> template <typename... Ts>
void test_ignore_empty_impl(const std::vector<X>& data) { void test_ignore_empty_impl(const std::vector<X>& data) {
unique_file_name f{"test_parser"}; unique_file_name f{"test_parser"};
make_and_write(f.name, data); make_and_write(f.name, data);
@ -417,8 +407,7 @@ void test_ignore_empty_impl(const std::vector<X>& data) {
} }
{ {
auto [p, _] = ss::parser<ss::ignore_empty, Ts...> p{f.name, ","};
make_parser<buffer_mode, ss::ignore_empty, Ts...>(f.name, ",");
std::vector<X> i; std::vector<X> i;
for (const auto& a : p.template iterate<X>()) { for (const auto& a : p.template iterate<X>()) {
@ -429,7 +418,7 @@ void test_ignore_empty_impl(const std::vector<X>& data) {
} }
{ {
auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ","); ss::parser<Ts...> p{f.name, ","};
std::vector<X> i; std::vector<X> i;
size_t n = 0; size_t n = 0;
while (!p.eof()) { while (!p.eof()) {
@ -452,12 +441,9 @@ void test_ignore_empty_impl(const std::vector<X>& data) {
template <typename... Ts> template <typename... Ts>
void test_ignore_empty(const std::vector<X>& data) { void test_ignore_empty(const std::vector<X>& data) {
test_ignore_empty_impl<false>(data); test_ignore_empty_impl(data);
test_ignore_empty_impl<false, ss::string_error>(data); test_ignore_empty_impl<ss::string_error>(data);
test_ignore_empty_impl<false, ss::throw_on_error>(data); test_ignore_empty_impl<ss::throw_on_error>(data);
test_ignore_empty_impl<true>(data);
test_ignore_empty_impl<true, ss::string_error>(data);
test_ignore_empty_impl<true, ss::throw_on_error>(data);
} }
TEST_CASE("parser test various cases with empty lines") { TEST_CASE("parser test various cases with empty lines") {

View File

@ -257,8 +257,7 @@ std::vector<std::string> generate_csv_data(const std::vector<field>& data,
} }
[[maybe_unused]] void write_to_file(const std::vector<std::string>& data, [[maybe_unused]] void write_to_file(const std::vector<std::string>& data,
const std::string& delim, const std::string& delim, const std::string& file_name) {
const std::string& file_name) {
std::ofstream out{file_name, std::ios_base::app}; std::ofstream out{file_name, std::ios_base::app};
std::string line; std::string line;
for (size_t i = 0; i < data.size(); ++i) { for (size_t i = 0; i < data.size(); ++i) {
@ -300,7 +299,7 @@ std::vector<std::string> generate_csv_data(const std::vector<field>& data,
CHECK(V1 == V2); \ CHECK(V1 == V2); \
} }
template <bool buffer_mode, typename... Ts> template <typename... Ts>
void test_data_combinations(const std::vector<column>& input_data, void test_data_combinations(const std::vector<column>& input_data,
const std::string& delim, bool include_header) { const std::string& delim, bool include_header) {
using setup = ss::setup<Ts...>; using setup = ss::setup<Ts...>;
@ -389,7 +388,7 @@ void test_data_combinations(const std::vector<column>& input_data,
} }
for (const auto& layout : unique_layout_combinations) { for (const auto& layout : unique_layout_combinations) {
auto [p, _] = make_parser<buffer_mode, setup>(f.name, delim); ss::parser<setup> p{f.name, delim};
if (include_header && !setup::ignore_header) { if (include_header && !setup::ignore_header) {
std::vector<std::string> fields; std::vector<std::string> fields;
@ -410,7 +409,7 @@ void test_data_combinations(const std::vector<column>& input_data,
REQUIRE(p.valid()); REQUIRE(p.valid());
} }
auto check_error = [&p = p] { auto check_error = [&p] {
CHECK(p.valid()); CHECK(p.valid());
if (!p.valid()) { if (!p.valid()) {
if constexpr (setup::string_error) { if constexpr (setup::string_error) {
@ -571,14 +570,8 @@ void test_option_combinations() {
{columns0, columns1, columns2, columns3, columns4, columns5, {columns0, columns1, columns2, columns3, columns4, columns5,
columns6, columns7}) { columns6, columns7}) {
try { try {
test_data_combinations<false, Ts...>(columns, delimiter, test_data_combinations<Ts...>(columns, delimiter, false);
false); test_data_combinations<Ts...>(columns, delimiter, true);
test_data_combinations<false, Ts...>(columns, delimiter,
true);
test_data_combinations<true, Ts...>(columns, delimiter,
false);
test_data_combinations<true, Ts...>(columns, delimiter,
true);
} catch (std::exception& e) { } catch (std::exception& e) {
std::cout << typeid(ss::parser<Ts...>).name() << std::endl; std::cout << typeid(ss::parser<Ts...>).name() << std::endl;
FAIL_CHECK(std::string{e.what()}); FAIL_CHECK(std::string{e.what()});
@ -623,7 +616,6 @@ void test_option_combinations3() {
} /* namespace */ } /* namespace */
// Tests split into multiple compilation units
#if 0 #if 0
TEST_CASE("parser test various cases version 2 segment 1") { TEST_CASE("parser test various cases version 2 segment 1") {
@ -635,33 +627,25 @@ TEST_CASE("parser test various cases version 2 segment 1") {
using multiline_r = ss::multiline_restricted<10>; using multiline_r = ss::multiline_restricted<10>;
using trimr = ss::trim_right<' '>; using trimr = ss::trim_right<' '>;
using triml = ss::trim_left<' '>; using triml = ss::trim_left<' '>;
using trim = ss::trim<' '>;
// segment 1 // segment 1
test_option_combinations3<>(); test_option_combinations3<>();
test_option_combinations3<escape>(); test_option_combinations3<escape>();
test_option_combinations3<quote>();
// segment 2 // segment 2
test_option_combinations3<quote>();
test_option_combinations3<escape, quote>(); test_option_combinations3<escape, quote>();
// segment 3
test_option_combinations3<escape, multiline>(); test_option_combinations3<escape, multiline>();
test_option_combinations3<quote, multiline>(); test_option_combinations3<quote, multiline>();
// segment 4 // segment 3
test_option_combinations3<escape, quote, multiline>(); test_option_combinations3<escape, quote, multiline>();
test_option_combinations3<escape, quote, multiline_r>(); test_option_combinations3<escape, quote, multiline_r>();
// segment 5 // segment 4
test_option_combinations<escape, quote, multiline, triml>(); test_option_combinations<escape, quote, multiline, triml>();
test_option_combinations<escape, quote, multiline, trimr>(); test_option_combinations<escape, quote, multiline, trimr>();
// segment 6
test_option_combinations3<escape, quote, multiline>();
test_option_combinations3<escape, quote, multiline, trim>();
#else #else
test_option_combinations3<escape, quote, multiline>(); test_option_combinations3<escape, quote, multiline>();
#endif #endif
} }

View File

@ -3,10 +3,12 @@
TEST_CASE("parser test various cases version 2 segment 1") { TEST_CASE("parser test various cases version 2 segment 1") {
#ifdef CMAKE_GITHUB_CI #ifdef CMAKE_GITHUB_CI
using quote = ss::quote<'"'>;
using escape = ss::escape<'\\'>; using escape = ss::escape<'\\'>;
test_option_combinations3<>(); test_option_combinations3<>();
test_option_combinations3<escape>(); test_option_combinations3<escape>();
test_option_combinations3<quote>();
#endif #endif
} }

View File

@ -5,9 +5,11 @@ TEST_CASE("parser test various cases version 2 segment 2") {
#ifdef CMAKE_GITHUB_CI #ifdef CMAKE_GITHUB_CI
using quote = ss::quote<'"'>; using quote = ss::quote<'"'>;
using escape = ss::escape<'\\'>; using escape = ss::escape<'\\'>;
using multiline = ss::multiline;
test_option_combinations3<quote>();
test_option_combinations3<escape, quote>(); test_option_combinations3<escape, quote>();
test_option_combinations3<escape, multiline>();
test_option_combinations3<quote, multiline>();
#endif #endif
} }

View File

@ -6,9 +6,10 @@ TEST_CASE("parser test various cases version 2 segment 3") {
using quote = ss::quote<'"'>; using quote = ss::quote<'"'>;
using escape = ss::escape<'\\'>; using escape = ss::escape<'\\'>;
using multiline = ss::multiline; using multiline = ss::multiline;
using multiline_r = ss::multiline_restricted<10>;
test_option_combinations3<escape, multiline>(); test_option_combinations3<escape, quote, multiline>();
test_option_combinations3<quote, multiline>(); test_option_combinations3<escape, quote, multiline_r>();
#endif #endif
} }

View File

@ -2,14 +2,18 @@
#include "test_parser2.hpp" #include "test_parser2.hpp"
TEST_CASE("parser test various cases version 2 segment 4") { TEST_CASE("parser test various cases version 2 segment 4") {
#ifdef CMAKE_GITHUB_CI
using quote = ss::quote<'"'>; using quote = ss::quote<'"'>;
using escape = ss::escape<'\\'>; using escape = ss::escape<'\\'>;
using multiline = ss::multiline; using multiline = ss::multiline;
using multiline_r = ss::multiline_restricted<10>;
#ifdef CMAKE_GITHUB_CI
using trimr = ss::trim_right<' '>;
using triml = ss::trim_left<' '>;
test_option_combinations<escape, quote, multiline, triml>();
test_option_combinations<escape, quote, multiline, trimr>();
#else
test_option_combinations3<escape, quote, multiline>(); test_option_combinations3<escape, quote, multiline>();
test_option_combinations3<escape, quote, multiline_r>();
#endif #endif
} }

View File

@ -1,16 +0,0 @@
#define SEGMENT_NAME "segment5"
#include "test_parser2.hpp"
TEST_CASE("parser test various cases version 2 segment 5") {
#ifdef CMAKE_GITHUB_CI
using quote = ss::quote<'"'>;
using escape = ss::escape<'\\'>;
using multiline = ss::multiline;
using trimr = ss::trim_right<' '>;
using triml = ss::trim_left<' '>;
test_option_combinations<escape, quote, multiline, triml>();
test_option_combinations<escape, quote, multiline, trimr>();
#endif
}

View File

@ -1,11 +0,0 @@
#define SEGMENT_NAME "segment6"
#include "test_parser2.hpp"
TEST_CASE("parser test various cases version 2 segment 6") {
using quote = ss::quote<'"'>;
using escape = ss::escape<'\\'>;
using multiline = ss::multiline;
test_option_combinations3<escape, quote, multiline>();
}