Merge pull request #26 from red0124/bugfix/terminated_escape_on_quoted_multiline

Fix bug where the parsing of data containing escaped new lines within…

.github/workflows/win-msvc.yml

@@ -0,0 +1,63 @@
+name: win-msvc-ci
+
+on:
+  push:
+    branches:
+      - master
+      - feature/**
+      - improvement/**
+      - bugfix/**
+
+  pull_request:
+    branches:
+      - master
+      - feature/**
+      - improvement/**
+      - bugfix/**
+
+jobs:
+  ci:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+
+    defaults:
+      run:
+        shell: bash
+
+    runs-on: ${{ matrix.config.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - os: windows-2019
+            vs: "Visual Studio 16 2019"
+
+          - os: windows-latest
+            vs: "Visual Studio 17 2022"
+
+        build: [Debug, Release]
+        platform: [Win32, x64]
+
+    name: "${{matrix.config.vs}}:${{matrix.platform}}:${{matrix.build}}"
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: script/ci_install_deps.sh
+
+      - name: Configure
+        run: >-
+          cmake -S test -B build -D CMAKE_BUILD_TYPE=${{matrix.build}}
+          -G "${{matrix.config.vs}}" -A ${{matrix.platform}}
+
+      - name: Build
+        run: cmake --build build -j ${{steps.cores.outputs.count}}
+
+      - name: Run
+        working-directory: build
+        run: >-
+          ctest -C Debug --output-on-failure -j ${{steps.cores.outputs.count}}

README.md

@@ -8,11 +8,12 @@
 ```
 
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-![ubuntu-latest-gcc](https://github.com/red0124/ssp/workflows/ubuntu-latest-gcc-ci/badge.svg)
+[![ubuntu-latest-gcc](https://github.com/red0124/ssp/workflows/ubuntu-latest-gcc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/ubuntu-latest-gcc.yml)
-![ubuntu-latest-clang](https://github.com/red0124/ssp/workflows/ubuntu-latest-clang-ci/badge.svg)
+[![ubuntu-latest-clang](https://github.com/red0124/ssp/workflows/ubuntu-latest-clang-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/ubuntu-latest-clang.yml)
-![ubuntu-latest-icc](https://github.com/red0124/ssp/workflows/ubuntu-latest-icc-ci/badge.svg)
+[![ubuntu-latest-icc](https://github.com/red0124/ssp/workflows/ubuntu-latest-icc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/ubuntu-latest-icc.yml)
-![windows-msys2-gcc](https://github.com/red0124/ssp/workflows/win-msys2-gcc-ci/badge.svg)
+[![windows-msys2-gcc](https://github.com/red0124/ssp/workflows/win-msys2-gcc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msys2-gcc.yml)
-![windows-msys2-clang](https://github.com/red0124/ssp/workflows/win-msys2-clang-ci/badge.svg)   
+[![windows-msys2-clang](https://github.com/red0124/ssp/workflows/win-msys2-clang-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msys2-clang.yml)
+[![win-msvc-ci](https://github.com/red0124/ssp/workflows/win-msvc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msvc.yml)
 
 A header only "csv" parser which is fast and versatile with modern C++ api. Requires compiler with C++17 support. [Can also be used to convert strings to specific types.](#The-converter)
 
@@ -53,20 +54,20 @@ Brian S. Wolfe 40 1.9
 Bill (Heath) Gates 65 3.3
 ```
 # Features
- * [Works on any type](#Custom-conversions)
+ * [Works on any type](#custom-conversions)
  * Easy to use
  * No exceptions
- * [Works with headers](#Headers)
+ * [Works with headers](#headers)
- * [Works with quotes, escapes and spacings](#Setup)
+ * [Works with quotes, escapes and spacings](#setup)
- * [Works with values containing new lines](#Multiline)
+ * [Works with values containing new lines](#multiline)
- * [Columns and rows can be ignored](#Special-types)
+ * [Columns and rows can be ignored](#special-types)
  * Works with any type of delimiter
  * Can return whole objects composed of converted values
- * [Descriptive error handling can be enabled](#Error-handling)
+ * [Descriptive error handling can be enabled](#error-handling)
- * [Restrictions can be added for each column](#Restrictions)
+ * [Restrictions can be added for each column](#restrictions)
- * [Works with `std::optional` and `std::variant`](#Special-types)
+ * [Works with `std::optional` and `std::variant`](#special-types)
  * Works with **`CRLF`** and **`LF`**
- * [Conversions can be chained if invalid](#Substitute-conversions)
+ * [Conversions can be chained if invalid](#substitute-conversions)
  * Fast
 
 # Single header
@@ -378,7 +379,7 @@ if(std::holds_alternative<float>(grade)) {
 ```
 ## Restrictions
 
-Custom **`restrictions`** can be used to narrow down the conversions of unwanted values. **`ss::ir`** (in range) and **`ss::ne`** (none empty) are one of those:
+Custom **`restrictions`** can be used to narrow down the conversions of unwanted values. **`ss::ir`** (in range) and **`ss::ne`** (none empty) are some of those:
 ```cpp
 // ss::ne makes sure that the name is not empty
 // ss::ir makes sure that the grade will be in range [0, 10]
@@ -472,7 +473,6 @@ The delimiter is " ", and the number of columns varies depending on which shape
 ```cpp
 ss::parser p{"shapes.txt", " "};
 if (!p.valid()) {
-    std::cout << p.error_msg() << std::endl;
     exit(EXIT_FAILURE);
 }
 

include/ss/converter.hpp

@@ -194,7 +194,7 @@ private:
     ////////////////
 
     const split_data& resplit(line_ptr_type new_line, ssize_t new_size,
-                              const std::string& delim = default_delimiter) {
+                              const std::string& delim) {
         return splitter_.resplit(new_line, new_size, delim);
     }
 

include/ss/parser.hpp

@@ -152,7 +152,7 @@ public:
     struct iterable {
         struct iterator {
             using value = std::conditional_t<get_object, T,
-                                            no_void_validator_tup_t<T, Ts...>>;
+                                             no_void_validator_tup_t<T, Ts...>>;
 
             iterator() : parser_{nullptr} {
             }
@@ -600,6 +600,8 @@ private:
 
             if constexpr (quoted_multiline_enabled) {
                 while (unterminated_quote()) {
+                    size -= next_line_converter_.size_shifted();
+
                     if (multiline_limit_reached(limit)) {
                         return true;
                     }
@@ -623,7 +625,8 @@ private:
                         }
                     }
 
-                    next_line_converter_.resplit(next_line_buffer_, size);
+                    next_line_converter_.resplit(next_line_buffer_, size,
+                                                 delim_);
                 }
             }
 
@@ -664,9 +667,6 @@ private:
         }
 
         void undo_remove_eol(char* buffer, size_t& string_end) {
-            if (next_line_converter_.unterminated_quote()) {
-                string_end -= next_line_converter_.size_shifted();
-            }
             if (crlf_) {
                 std::copy_n("\r\n\0", 3, buffer + string_end);
                 string_end += 2;

include/ss/setup.hpp

@@ -109,10 +109,10 @@ struct get_matcher<Matcher, T, Ts...> {
     struct is_matcher : is_instance_of_matcher<U, Matcher> {};
 
     static_assert(count_v<is_matcher, T, Ts...> <= 1,
-                  "the same matcher is cannot"
+                  "the same matcher cannot"
                   "be defined multiple times");
     using type = std::conditional_t<is_matcher<T>::value, T,
-                               typename get_matcher<Matcher, Ts...>::type>;
+                                    typename get_matcher<Matcher, Ts...>::type>;
 };
 
 template <template <char...> class Matcher>
@@ -150,7 +150,7 @@ struct get_multiline;
 template <typename T, typename... Ts>
 struct get_multiline<T, Ts...> {
     using type = std::conditional_t<is_instance_of_multiline<T>::value, T,
-                               typename get_multiline<Ts...>::type>;
+                                    typename get_multiline<Ts...>::type>;
 };
 
 template <>

ssp.hpp

@@ -17,6 +17,7 @@
 #include <vector>
 #define SSP_DISABLE_FAST_FLOAT
 
+
 namespace ss {
 
 ////////////////
@@ -394,6 +395,7 @@ T to_object(U&& data) {
 
 } /* trait */
 
+
 namespace ss {
 
 ////////////////
@@ -775,7 +777,7 @@ struct get_matcher<Matcher, T, Ts...> {
     struct is_matcher : is_instance_of_matcher<U, Matcher> {};
 
     static_assert(count_v<is_matcher, T, Ts...> <= 1,
-                  "the same matcher is cannot"
+                  "the same matcher cannot"
                   "be defined multiple times");
     using type = std::conditional_t<is_matcher<T>::value, T,
                                     typename get_matcher<Matcher, Ts...>::type>;
@@ -1380,6 +1382,7 @@ public:
 
 } /* ss */
 
+
 #ifndef SSP_DISABLE_FAST_FLOAT
 #else
 #endif
@@ -1909,7 +1912,7 @@ private:
     ////////////////
 
     const split_data& resplit(line_ptr_type new_line, ssize_t new_size,
-                              const std::string& delim = default_delimiter) {
+                              const std::string& delim) {
         return splitter_.resplit(new_line, new_size, delim);
     }
 
@@ -2198,6 +2201,7 @@ private:
 
 } /* ss */
 
+
 namespace ss {
 
 template <typename... Matchers>
@@ -2339,9 +2343,8 @@ public:
     template <bool get_object, typename T, typename... Ts>
     struct iterable {
         struct iterator {
-            using value =
+            using value = std::conditional_t<get_object, T,
-                std::conditional_t<get_object, T,
+                                             no_void_validator_tup_t<T, Ts...>>;
-                                       no_void_validator_tup_t<T, Ts...>>;
 
             iterator() : parser_{nullptr} {
             }
@@ -2789,6 +2792,8 @@ private:
 
             if constexpr (quoted_multiline_enabled) {
                 while (unterminated_quote()) {
+                    size -= next_line_converter_.size_shifted();
+
                     if (multiline_limit_reached(limit)) {
                         return true;
                     }
@@ -2812,7 +2817,8 @@ private:
                         }
                     }
 
-                    next_line_converter_.resplit(next_line_buffer_, size);
+                    next_line_converter_.resplit(next_line_buffer_, size,
+                                                 delim_);
                 }
             }
 
@@ -2853,9 +2859,6 @@ private:
         }
 
         void undo_remove_eol(char* buffer, size_t& string_end) {
-            if (next_line_converter_.unterminated_quote()) {
-                string_end -= next_line_converter_.size_shifted();
-            }
             if (crlf_) {
                 std::copy_n("\r\n\0", 3, buffer + string_end);
                 string_end += 2;

test/test_helpers.hpp

@@ -1,6 +1,6 @@
 #pragma once
-#include <cstdlib>
+
-#include <cstring>
+#include <string>
 
 #ifdef CMAKE_GITHUB_CI
 #include <doctest/doctest.h>
@@ -9,73 +9,55 @@
 #endif
 
 struct buffer {
-    char* data_{nullptr};
+  std::string data_;
 
-    char* operator()(const char* data) {
+  char *operator()(const std::string &data) {
-        if (data_) {
+    data_ = data;
-            delete[] data_;
+    return data_.data();
-        }
+  }
-        data_ = new char[strlen(data) + 1];
-        strcpy(data_, data);
-        return data_;
-    }
 
-    char* append(const char* data) {
+  char *append(const std::string &data) {
-        if (data_) {
+    data_ += data;
-            char* new_data_ = new char[strlen(data_) + strlen(data) + 1];
+    return data_.data();
-            strcpy(new_data_, data_);
+  }
-            strcat(new_data_, data);
-            delete[] data_;
-            data_ = new_data_;
-            return data_;
-        } else {
-            return operator()(data);
-        }
-    }
 
-    char* append_overwrite_last(const char* data, size_t size) {
+  char *append_overwrite_last(const std::string &data, size_t size) {
-        data_[strlen(data_) - size] = '\0';
+    data_.resize(data_.size() - size);
-        return append(data);
+    return append(data);
-    }
+  }
-
-    ~buffer() {
-        if (data_) {
-            delete[] data_;
-        }
-    }
 };
 
 [[maybe_unused]] inline buffer buff;
 
 #define CHECK_FLOATING_CONVERSION(input, type)                                 \
-    {                                                                          \
+  {                                                                            \
-        auto eps = std::numeric_limits<type>::min();                           \
+    auto eps = std::numeric_limits<type>::min();                               \
-        std::string s = #input;                                                \
+    std::string s = #input;                                                    \
-        auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());            \
+    auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());                \
-        REQUIRE(t.has_value());                                                \
+    REQUIRE(t.has_value());                                                    \
-        CHECK_LT(std::abs(t.value() - type(input)), eps);                      \
+    CHECK_LT(std::abs(t.value() - type(input)), eps);                          \
-    }                                                                          \
+  }                                                                            \
-    {                                                                          \
+  {                                                                            \
-        /* check negative too */                                               \
+    /* check negative too */                                                   \
-        auto eps = std::numeric_limits<type>::min();                           \
+    auto eps = std::numeric_limits<type>::min();                               \
-        auto s = std::string("-") + #input;                                    \
+    auto s = std::string("-") + #input;                                        \
-        auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());            \
+    auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());                \
-        REQUIRE(t.has_value());                                                \
+    REQUIRE(t.has_value());                                                    \
-        CHECK_LT(std::abs(t.value() - type(-input)), eps);                     \
+    CHECK_LT(std::abs(t.value() - type(-input)), eps);                         \
-    }
+  }
 
 #define CHECK_INVALID_CONVERSION(input, type)                                  \
-    {                                                                          \
+  {                                                                            \
-        std::string s = input;                                                 \
+    std::string s = input;                                                     \
-        auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());            \
+    auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());                \
-        CHECK_FALSE(t.has_value());                                            \
+    CHECK_FALSE(t.has_value());                                                \
-    }
+  }
 
 #define REQUIRE_VARIANT(var, el, type)                                         \
-    {                                                                          \
+  {                                                                            \
-        auto ptr = std::get_if<type>(&var);                                    \
+    auto ptr = std::get_if<type>(&var);                                        \
-        REQUIRE(ptr);                                                          \
+    REQUIRE(ptr);                                                              \
-        REQUIRE_EQ(el, *ptr);                                                  \
+    REQUIRE_EQ(el, *ptr);                                                      \
-    }
+  }
 
 #define CHECK_NOT_VARIANT(var, type) CHECK(!std::holds_alternative<type>(var));

test/test_parser.cpp

@@ -283,7 +283,7 @@ TEST_CASE("parser test various cases") {
         }
 
         std::copy_if(data.begin(), data.end(), expected.begin(),
-                     [](const X& x) { return x.i != excluded; });
+                     [&](const X& x) { return x.i != excluded; });
         CHECK_EQ(i, expected);
         CHECK_EQ(i2, expected);
     }
@@ -378,7 +378,7 @@ TEST_CASE("parser test composite conversion") {
             p.try_next<int, int, double>(fail)
                 .or_else<test_struct>(fail)
                 .or_else<int, char, double>(
-                    [](auto&& data) { CHECK_EQ(data, expectedData); })
+                    [&](auto&& data) { CHECK_EQ(data, expectedData); })
                 .on_error(fail)
                 .or_else<test_tuple>(fail)
                 .values();
@@ -396,7 +396,7 @@ TEST_CASE("parser test composite conversion") {
         constexpr static auto expectedData = std::tuple{10, 20, 11.1};
 
         auto [d1, d2, d3, d4] =
-            p.try_next<int, int, double>([](auto& i1, auto i2, double d) {
+            p.try_next<int, int, double>([&](auto& i1, auto i2, double d) {
                  CHECK_EQ(std::tie(i1, i2, d), expectedData);
              })
                 .on_error(fail)
@@ -564,8 +564,6 @@ TEST_CASE("parser test composite conversion") {
     CHECK(p.eof());
 }
 
-size_t move_called = 0;
-
 struct my_string {
     char* data{nullptr};
 
@@ -580,12 +578,10 @@ struct my_string {
     my_string& operator=(const my_string&) = delete;
 
     my_string(my_string&& other) : data{other.data} {
-        move_called++;
         other.data = nullptr;
     }
 
     my_string& operator=(my_string&& other) {
-        move_called++;
         data = other.data;
         return *this;
     }
@@ -609,49 +605,6 @@ struct xyz {
     }
 };
 
-TEST_CASE("parser test the moving of parsed values") {
-    {
-        unique_file_name f;
-        {
-            std::ofstream out{f.name};
-            out << "x" << std::endl;
-        }
-
-        ss::parser p{f.name, ","};
-        auto x = p.get_next<my_string>();
-        CHECK_LE(move_called, 1);
-        move_called = 0;
-    }
-
-    unique_file_name f;
-    {
-        std::ofstream out{f.name};
-        out << "a,b,c" << std::endl;
-    }
-
-    {
-
-        ss::parser p{f.name, ","};
-        auto x = p.get_next<my_string, my_string, my_string>();
-        CHECK_LE(move_called, 3);
-        move_called = 0;
-    }
-
-    {
-        ss::parser p{f.name, ","};
-        auto x = p.get_object<xyz, my_string, my_string, my_string>();
-        CHECK_LE(move_called, 6);
-        move_called = 0;
-    }
-
-    {
-        ss::parser p{f.name, ","};
-        auto x = p.get_next<xyz>();
-        CHECK_LE(move_called, 6);
-        move_called = 0;
-    }
-}
-
 TEST_CASE("parser test the moving of parsed composite values") {
     // to compile is enough
     return;