Enable ignore_empty functionality

include/ss/common.hpp

@@ -1,11 +1,7 @@
 #pragma once
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
 #include <vector>
 
 namespace ss {
-
 struct none {};
 
 using string_range = std::pair<const char*, const char*>;
@@ -24,61 +20,4 @@ inline void assert_throw_on_error_not_defined() {
     static_assert(!ThrowOnError, "cannot handle errors manually if "
                                  "'throw_on_error' is enabled");
 }
-
-#if __unix__
-inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
-    return getline(lineptr, n, stream);
-}
-#else
-using ssize_t = int64_t;
-inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
-    size_t pos;
-    int c;
-
-    if (lineptr == nullptr || stream == nullptr || n == nullptr) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    c = getc(stream);
-    if (c == EOF) {
-        return -1;
-    }
-
-    if (*lineptr == nullptr) {
-        *lineptr = static_cast<char*>(malloc(128));
-        if (*lineptr == nullptr) {
-            return -1;
-        }
-        *n = 128;
-    }
-
-    pos = 0;
-    while (c != EOF) {
-        if (pos + 1 >= *n) {
-            size_t new_size = *n + (*n >> 2);
-            if (new_size < 128) {
-                new_size = 128;
-            }
-            char* new_ptr = static_cast<char*>(
-                realloc(static_cast<void*>(*lineptr), new_size));
-            if (new_ptr == nullptr) {
-                return -1;
-            }
-            *n = new_size;
-            *lineptr = new_ptr;
-        }
-
-        (*lineptr)[pos++] = c;
-        if (c == '\n') {
-            break;
-        }
-        c = getc(stream);
-    }
-
-    (*lineptr)[pos] = '\0';
-    return pos;
-}
-#endif
-
 } /* ss */

test/test_parser.cpp

@@ -38,7 +38,7 @@ void expect_error_on_command(ss::parser<Ts...>& p,
     }
 }
 
-[[maybe_unused]] void update_if_crlf(std::string& s) {
+void update_if_crlf(std::string& s) {
 #ifdef _WIN32
     replace_all(s, "\r\n", "\n");
 #else
@@ -102,31 +102,6 @@ static void make_and_write(const std::string& file_name,
         out << data[i].to_string() << new_lines[i % new_lines.size()];
     }
 }
-
-std::string make_buffer(const std::string& file_name) {
-    std::ifstream in{file_name, std::ios::binary};
-    std::string tmp;
-    std::string out;
-    out.reserve(sizeof(out) + 1);
-    while (in >> tmp) {
-        out += tmp;
-        out.append("\n");
-    }
-    return out;
-}
-
-template <bool buffer_mode, typename... Ts>
-std::tuple<ss::parser<Ts...>, std::string> make_parser(
-    const std::string& file_name, const std::string& delim) {
-    if (buffer_mode) {
-        auto buffer = make_buffer(file_name);
-        return {ss::parser<Ts...>{buffer.data(), buffer.size(), delim},
-                std::move(buffer)};
-    } else {
-        return {ss::parser<Ts...>{file_name, delim}, std::string{}};
-    }
-}
-
 } /* namespace */
 
 TEST_CASE("test file not found") {
@@ -150,23 +125,22 @@ TEST_CASE("test file not found") {
     }
 }
 
-template <bool buffer_mode, typename... Ts>
+template <typename... Ts>
 void test_various_cases() {
     unique_file_name f{"test_parser"};
     std::vector<X> data = {{1, 2, "x"}, {3, 4, "y"},  {5, 6, "z"},
                            {7, 8, "u"}, {9, 10, "v"}, {11, 12, "w"}};
     make_and_write(f.name, data);
-    auto csv_data_buffer = make_buffer(f.name);
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser<Ts...> p{f.name, ","};
         ss::parser p0{std::move(p)};
         p = std::move(p0);
         std::vector<X> i;
 
-        auto [p2, __] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser<ss::string_error> p2{f.name, ","};
         std::vector<X> i2;
 
-        auto move_rotate = [&p = p, &p0 = p0] {
+        auto move_rotate = [&] {
             auto p1 = std::move(p);
             p0 = std::move(p1);
             p = std::move(p0);
@@ -178,7 +152,7 @@ void test_various_cases() {
             i.emplace_back(ss::to_object<X>(a));
         }
 
-        for (const auto& a : p2.template iterate<int, double, std::string>()) {
+        for (const auto& a : p2.iterate<int, double, std::string>()) {
             i2.emplace_back(ss::to_object<X>(a));
         }
 
@@ -187,13 +161,13 @@ void test_various_cases() {
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
-        auto [p2, __] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p2{f.name, ","};
         std::vector<X> i2;
 
-        auto [p3, ___] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p3{f.name, ","};
         std::vector<X> i3;
 
         std::vector<X> expected = {std::begin(data) + 1, std::end(data)};
@@ -201,18 +175,18 @@ void test_various_cases() {
 
         p.ignore_next();
         while (!p.eof()) {
-            auto a = p.template get_next<tup>();
+            auto a = p.get_next<tup>();
             i.emplace_back(ss::to_object<X>(a));
         }
 
         p2.ignore_next();
-        for (const auto& a : p2.template iterate<tup>()) {
+        for (const auto& a : p2.iterate<tup>()) {
             i2.emplace_back(ss::to_object<X>(a));
         }
 
         p3.ignore_next();
-        for (auto it = p3.template iterate<tup>().begin();
+        for (auto it = p3.iterate<tup>().begin(); it != p3.iterate<tup>().end();
-             it != p3.template iterate<tup>().end(); ++it) {
+             ++it) {
             i3.emplace_back(ss::to_object<X>(*it));
         }
 
@@ -222,17 +196,16 @@ void test_various_cases() {
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
-        auto [p2, __] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p2{f.name, ","};
         std::vector<X> i2;
 
         while (!p.eof()) {
-            i.push_back(p.template get_object<X, int, double, std::string>());
+            i.push_back(p.get_object<X, int, double, std::string>());
         }
 
-        for (auto&& a :
+        for (auto&& a : p2.iterate_object<X, int, double, std::string>()) {
-             p2.template iterate_object<X, int, double, std::string>()) {
             i2.push_back(std::move(a));
         }
 
@@ -241,11 +214,10 @@ void test_various_cases() {
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
-        for (auto&& a :
+        for (auto&& a : p.iterate_object<X, int, double, std::string>()) {
-             p.template iterate_object<X, int, double, std::string>()) {
             i.push_back(std::move(a));
         }
 
@@ -253,19 +225,19 @@ void test_various_cases() {
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
-        auto [p2, __] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p2{f.name, ","};
         std::vector<X> i2;
 
         using tup = std::tuple<int, double, std::string>;
         while (!p.eof()) {
-            i.push_back(p.template get_object<X, tup>());
+            i.push_back(p.get_object<X, tup>());
         }
 
-        for (auto it = p2.template iterate_object<X, tup>().begin();
+        for (auto it = p2.iterate_object<X, tup>().begin();
-             it != p2.template iterate_object<X, tup>().end(); it++) {
+             it != p2.iterate_object<X, tup>().end(); it++) {
             i2.push_back({it->i, it->d, it->s});
         }
 
@@ -274,11 +246,11 @@ void test_various_cases() {
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
         using tup = std::tuple<int, double, std::string>;
-        for (auto&& a : p.template iterate_object<X, tup>()) {
+        for (auto&& a : p.iterate_object<X, tup>()) {
             i.push_back(std::move(a));
         }
 
@@ -286,21 +258,21 @@ void test_various_cases() {
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
         while (!p.eof()) {
-            i.push_back(p.template get_next<X>());
+            i.push_back(p.get_next<X>());
         }
 
         CHECK_EQ(i, data);
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
-        for (auto&& a : p.template iterate<X>()) {
+        for (auto&& a : p.iterate<X>()) {
             i.push_back(std::move(a));
         }
 
@@ -309,30 +281,24 @@ void test_various_cases() {
 
     {
         constexpr int excluded = 3;
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
-        auto [p2, __] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p2{f.name, ","};
         std::vector<X> i2;
 
         while (!p.eof()) {
-            try {
+            auto a =
-                auto a = p.template get_object<X, ss::ax<int, excluded>, double,
+                p.get_object<X, ss::ax<int, excluded>, double, std::string>();
-                                               std::string>();
+            if (p.valid()) {
-                if (p.valid()) {
+                i.push_back(a);
-                    i.push_back(a);
+            }
-                }
-            } catch (...) {
-                // ignore
-            };
         }
 
-        if (!ss::setup<Ts...>::throw_on_error) {
+        for (auto&& a : p2.iterate_object<X, ss::ax<int, excluded>, double,
-            for (auto&& a : p2.template iterate_object<X, ss::ax<int, excluded>,
+                                          std::string>()) {
-                                                       double, std::string>()) {
+            if (p2.valid()) {
-                if (p2.valid()) {
+                i2.push_back(std::move(a));
-                    i2.push_back(std::move(a));
-                }
             }
         }
 
@@ -346,45 +312,33 @@ void test_various_cases() {
         std::copy_if(data.begin(), data.end(), expected.begin(),
                      [&](const X& x) { return x.i != excluded; });
         CHECK_EQ(i, expected);
-
+        CHECK_EQ(i2, expected);
-        if (!ss::setup<Ts...>::throw_on_error) {
-            CHECK_EQ(i2, expected);
-        }
     }
 
     {
-        auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p{f.name, ","};
         std::vector<X> i;
 
-        auto [p2, __] = make_parser<buffer_mode, Ts...>(f.name, ",");
+        ss::parser p2{f.name, ","};
         std::vector<X> i2;
 
         while (!p.eof()) {
-            try {
+            auto a = p.get_object<X, ss::nx<int, 3>, double, std::string>();
-                auto a = p.template get_object<X, ss::nx<int, 3>, double,
+            if (p.valid()) {
-                                               std::string>();
+                i.push_back(a);
-                if (p.valid()) {
-                    i.push_back(a);
-                }
-            } catch (...) {
-                // ignore
             }
         }
 
-        if (!ss::setup<Ts...>::throw_on_error) {
+        for (auto&& a :
-            for (auto&& a : p2.template iterate_object<X, ss::nx<int, 3>,
+             p2.iterate_object<X, ss::nx<int, 3>, double, std::string>()) {
-                                                       double, std::string>()) {
+            if (p2.valid()) {
-                if (p2.valid()) {
+                i2.push_back(std::move(a));
-                    i2.push_back(std::move(a));
-                }
             }
         }
 
         std::vector<X> expected = {{3, 4, "y"}};
         CHECK_EQ(i, expected);
-        if (!ss::setup<Ts...>::throw_on_error) {
+        CHECK_EQ(i2, expected);
-            CHECK_EQ(i2, expected);
-        }
     }
 
     {
@@ -393,17 +347,17 @@ void test_various_cases() {
 
         make_and_write(empty_f.name, empty_data);
 
-        auto [p, _] = make_parser<buffer_mode, Ts...>(empty_f.name, ",");
+        ss::parser p{empty_f.name, ","};
         std::vector<X> i;
 
-        auto [p2, __] = make_parser<buffer_mode, Ts...>(empty_f.name, ",");
+        ss::parser p2{empty_f.name, ","};
         std::vector<X> i2;
 
         while (!p.eof()) {
-            i.push_back(p.template get_next<X>());
+            i.push_back(p.get_next<X>());
         }
 
-        for (auto&& a : p2.template iterate<X>()) {
+        for (auto&& a : p2.iterate<X>()) {
             i2.push_back(std::move(a));
         }
 
@@ -413,12 +367,9 @@ void test_various_cases() {
 }
 
 TEST_CASE("parser test various cases") {
-    test_various_cases<false>();
+    test_various_cases();
-    test_various_cases<false, ss::string_error>();
+    test_various_cases<ss::string_error>();
-    test_various_cases<false, ss::throw_on_error>();
+    test_various_cases<ss::throw_on_error>();
-    test_various_cases<true>();
-    test_various_cases<true, ss::string_error>();
-    test_various_cases<true, ss::throw_on_error>();
 }
 
 using test_tuple = std::tuple<double, char, double>;
@@ -434,7 +385,7 @@ struct test_struct {
 static inline void expect_test_struct(const test_struct&) {
 }
 
-template <bool buffer_mode, typename... Ts>
+template <typename... Ts>
 void test_composite_conversion() {
     unique_file_name f{"test_parser"};
     {
@@ -446,7 +397,7 @@ void test_composite_conversion() {
         }
     }
 
-    auto [p, _] = make_parser<buffer_mode, Ts...>(f.name, ",");
+    ss::parser<Ts...> p{f.name, ","};
     auto fail = [] { FAIL(""); };
     auto expect_error = [](auto error) { CHECK(!error.empty()); };
     auto ignore_error = [] {};
@@ -658,8 +609,7 @@ void test_composite_conversion() {
 
 // various scenarios
 TEST_CASE("parser test composite conversion") {
-    test_composite_conversion<false, ss::string_error>();
+    test_composite_conversion<ss::string_error>();
-    test_composite_conversion<true, ss::string_error>();
 }
 
 struct my_string {
@@ -703,7 +653,7 @@ struct xyz {
     }
 };
 
-template <bool buffer_mode, typename... Ts>
+template <typename... Ts>
 void test_moving_of_parsed_composite_values() {
     // to compile is enough
     return;
@@ -719,10 +669,8 @@ void test_moving_of_parsed_composite_values() {
 }
 
 TEST_CASE("parser test the moving of parsed composite values") {
-    test_moving_of_parsed_composite_values<false>();
+    test_moving_of_parsed_composite_values();
-    test_moving_of_parsed_composite_values<false, ss::string_error>();
+    test_moving_of_parsed_composite_values<ss::string_error>();
-    test_moving_of_parsed_composite_values<true>();
-    test_moving_of_parsed_composite_values<true, ss::string_error>();
 }
 
 TEST_CASE("parser test error mode") {
@@ -733,23 +681,12 @@ TEST_CASE("parser test error mode") {
         out << "junk" << std::endl;
     }
 
-    {
+    ss::parser<ss::string_error> p(f.name, ",");
-        auto [p, _] = make_parser<false, ss::string_error>(f.name, ",");
 
-        REQUIRE_FALSE(p.eof());
+    REQUIRE_FALSE(p.eof());
-        p.get_next<int>();
+    p.get_next<int>();
-        CHECK_FALSE(p.valid());
+    CHECK_FALSE(p.valid());
-        CHECK_FALSE(p.error_msg().empty());
+    CHECK_FALSE(p.error_msg().empty());
-    }
-
-    {
-        auto [p, _] = make_parser<true, ss::string_error>(f.name, ",");
-
-        REQUIRE_FALSE(p.eof());
-        p.get_next<int>();
-        CHECK_FALSE(p.valid());
-        CHECK_FALSE(p.error_msg().empty());
-    }
 }
 
 TEST_CASE("parser throw on error mode") {
@@ -960,7 +897,7 @@ void test_multiline_restricted() {
         out << "17,18,\"ju\\\n\\\n\\\n\\\\\n\nnk\"" << std::endl;
         out << "19,20,just strings" << std::endl;
     }
-    auto bad_lines = 15;
+    auto bad_lines = 19;
     auto num_errors = 0;
 
     ss::parser<ss::multiline_restricted<2>, ss::quote<'"'>, ss::escape<'\\'>,
@@ -1704,6 +1641,7 @@ void test_ignore_empty(const std::vector<X>& data) {
     test_ignore_empty_impl<ss::throw_on_error>(data);
 }
 
+// TODO test with different initial buffer sizes
 TEST_CASE("parser test various cases with empty lines") {
     test_ignore_empty({{1, 2, "x"}, {3, 4, "y"}, {9, 10, "v"}, {11, 12, "w"}});
 
@@ -1733,14 +1671,53 @@ TEST_CASE("parser test various cases with empty lines") {
                        {9, 10, X::empty},
                        {11, 12, X::empty}});
 
-    test_ignore_empty(
+    test_ignore_empty({{1, 2, X::empty},
-        {{1, 2, "x"}, {3, 4, X::empty}, {9, 10, X::empty}, {11, 12, X::empty}});
+                       {3, 4, X::empty},
+                       {5, 6, X::empty},
+                       {7, 8, X::empty},
+                       {9, 10, X::empty},
+                       {11, 12, X::empty},
+                       {13, 14, X::empty},
+                       {15, 16, X::empty},
+                       {17, 18, X::empty}});
 
-    test_ignore_empty(
+    test_ignore_empty({{1, 2, X::empty},
-        {{1, 2, X::empty}, {3, 4, X::empty}, {9, 10, X::empty}, {11, 12, "w"}});
+                       {3, 4, X::empty},
+                       {5, 6, X::empty},
+                       {7, 8, X::empty},
+                       {9, 10, X::empty},
+                       {11, 12, X::empty},
+                       {13, 14, X::empty},
+                       {15, 16, X::empty},
+                       {17, 18, "x"}});
 
-    test_ignore_empty({{11, 12, X::empty}});
+        test_ignore_empty({{1, 2, "x"},
+                           {3, 4, X::empty},
+                           {9, 10, X::empty},
+                           {11, 12, X::empty}});
 
-    test_ignore_empty({});
+        test_ignore_empty({{1, 2, "x"},
+                           {3, 4, X::empty},
+                           {3, 4, X::empty},
+                           {5, 6, X::empty},
+                           {7, 8, X::empty},
+                           {9, 10, X::empty},
+                           {11, 12, X::empty}});
+
+        test_ignore_empty({{1, 2, "x"},
+                           {3, 4, X::empty},
+                           {3, 4, X::empty},
+                           {5, 6, X::empty},
+                           {7, 8, X::empty},
+                           {9, 10, X::empty},
+                           {11, 12, "y"}});
+
+        test_ignore_empty({{1, 2, X::empty},
+                           {3, 4, X::empty},
+                           {9, 10, X::empty},
+                           {11, 12, "w"}});
+
+        test_ignore_empty({{11, 12, X::empty}});
+
+        test_ignore_empty({});
 }
-