Merge pull request #26 from red0124/bugfix/terminated_escape_on_quoted_multiline

Fix bug where the parsing of data containing escaped new lines within…

.github/workflows/win-msvc.yml

@@ -0,0 +1,63 @@
+name: win-msvc-ci
+
+on:
+  push:
+    branches:
+      - master
+      - feature/**
+      - improvement/**
+      - bugfix/**
+
+  pull_request:
+    branches:
+      - master
+      - feature/**
+      - improvement/**
+      - bugfix/**
+
+jobs:
+  ci:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+
+    defaults:
+      run:
+        shell: bash
+
+    runs-on: ${{ matrix.config.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - os: windows-2019
+            vs: "Visual Studio 16 2019"
+
+          - os: windows-latest
+            vs: "Visual Studio 17 2022"
+
+        build: [Debug, Release]
+        platform: [Win32, x64]
+
+    name: "${{matrix.config.vs}}:${{matrix.platform}}:${{matrix.build}}"
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: script/ci_install_deps.sh
+
+      - name: Configure
+        run: >-
+          cmake -S test -B build -D CMAKE_BUILD_TYPE=${{matrix.build}}
+          -G "${{matrix.config.vs}}" -A ${{matrix.platform}}
+
+      - name: Build
+        run: cmake --build build -j ${{steps.cores.outputs.count}}
+
+      - name: Run
+        working-directory: build
+        run: >-
+          ctest -C Debug --output-on-failure -j ${{steps.cores.outputs.count}}

README.md

@@ -8,11 +8,12 @@
 ```
 
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-![ubuntu-latest-gcc](https://github.com/red0124/ssp/workflows/ubuntu-latest-gcc-ci/badge.svg)
-![ubuntu-latest-clang](https://github.com/red0124/ssp/workflows/ubuntu-latest-clang-ci/badge.svg)
-![ubuntu-latest-icc](https://github.com/red0124/ssp/workflows/ubuntu-latest-icc-ci/badge.svg)
-![windows-msys2-gcc](https://github.com/red0124/ssp/workflows/win-msys2-gcc-ci/badge.svg)
-![windows-msys2-clang](https://github.com/red0124/ssp/workflows/win-msys2-clang-ci/badge.svg)   
+[![ubuntu-latest-gcc](https://github.com/red0124/ssp/workflows/ubuntu-latest-gcc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/ubuntu-latest-gcc.yml)
+[![ubuntu-latest-clang](https://github.com/red0124/ssp/workflows/ubuntu-latest-clang-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/ubuntu-latest-clang.yml)
+[![ubuntu-latest-icc](https://github.com/red0124/ssp/workflows/ubuntu-latest-icc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/ubuntu-latest-icc.yml)
+[![windows-msys2-gcc](https://github.com/red0124/ssp/workflows/win-msys2-gcc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msys2-gcc.yml)
+[![windows-msys2-clang](https://github.com/red0124/ssp/workflows/win-msys2-clang-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msys2-clang.yml)
+[![win-msvc-ci](https://github.com/red0124/ssp/workflows/win-msvc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msvc.yml)
 
 A header only "csv" parser which is fast and versatile with modern C++ api. Requires compiler with C++17 support. [Can also be used to convert strings to specific types.](#The-converter)
 

include/ss/parser.hpp

@@ -600,6 +600,8 @@ private:
 
             if constexpr (quoted_multiline_enabled) {
                 while (unterminated_quote()) {
+                    size -= next_line_converter_.size_shifted();
+
                     if (multiline_limit_reached(limit)) {
                         return true;
                     }
@@ -665,9 +667,6 @@ private:
         }
 
         void undo_remove_eol(char* buffer, size_t& string_end) {
-            if (next_line_converter_.unterminated_quote()) {
-                string_end -= next_line_converter_.size_shifted();
-            }
             if (crlf_) {
                 std::copy_n("\r\n\0", 3, buffer + string_end);
                 string_end += 2;

ssp.hpp

@@ -2792,6 +2792,8 @@ private:
 
             if constexpr (quoted_multiline_enabled) {
                 while (unterminated_quote()) {
+                    size -= next_line_converter_.size_shifted();
+
                     if (multiline_limit_reached(limit)) {
                         return true;
                     }
@@ -2857,9 +2859,6 @@ private:
         }
 
         void undo_remove_eol(char* buffer, size_t& string_end) {
-            if (next_line_converter_.unterminated_quote()) {
-                string_end -= next_line_converter_.size_shifted();
-            }
             if (crlf_) {
                 std::copy_n("\r\n\0", 3, buffer + string_end);
                 string_end += 2;

test/test_helpers.hpp

@@ -1,6 +1,6 @@
 #pragma once
-#include <cstdlib>
-#include <cstring>
+
+#include <string>
 
 #ifdef CMAKE_GITHUB_CI
 #include <doctest/doctest.h>
@@ -9,73 +9,55 @@
 #endif
 
 struct buffer {
-    char* data_{nullptr};
+  std::string data_;
 
-    char* operator()(const char* data) {
-        if (data_) {
-            delete[] data_;
-        }
-        data_ = new char[strlen(data) + 1];
-        strcpy(data_, data);
-        return data_;
-    }
+  char *operator()(const std::string &data) {
+    data_ = data;
+    return data_.data();
+  }
 
-    char* append(const char* data) {
-        if (data_) {
-            char* new_data_ = new char[strlen(data_) + strlen(data) + 1];
-            strcpy(new_data_, data_);
-            strcat(new_data_, data);
-            delete[] data_;
-            data_ = new_data_;
-            return data_;
-        } else {
-            return operator()(data);
-        }
-    }
+  char *append(const std::string &data) {
+    data_ += data;
+    return data_.data();
+  }
 
-    char* append_overwrite_last(const char* data, size_t size) {
-        data_[strlen(data_) - size] = '\0';
-        return append(data);
-    }
-
-    ~buffer() {
-        if (data_) {
-            delete[] data_;
-        }
-    }
+  char *append_overwrite_last(const std::string &data, size_t size) {
+    data_.resize(data_.size() - size);
+    return append(data);
+  }
 };
 
 [[maybe_unused]] inline buffer buff;
 
 #define CHECK_FLOATING_CONVERSION(input, type)                                 \
-    {                                                                          \
-        auto eps = std::numeric_limits<type>::min();                           \
-        std::string s = #input;                                                \
-        auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());            \
-        REQUIRE(t.has_value());                                                \
-        CHECK_LT(std::abs(t.value() - type(input)), eps);                      \
-    }                                                                          \
-    {                                                                          \
-        /* check negative too */                                               \
-        auto eps = std::numeric_limits<type>::min();                           \
-        auto s = std::string("-") + #input;                                    \
-        auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());            \
-        REQUIRE(t.has_value());                                                \
-        CHECK_LT(std::abs(t.value() - type(-input)), eps);                     \
-    }
+  {                                                                            \
+    auto eps = std::numeric_limits<type>::min();                               \
+    std::string s = #input;                                                    \
+    auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());                \
+    REQUIRE(t.has_value());                                                    \
+    CHECK_LT(std::abs(t.value() - type(input)), eps);                          \
+  }                                                                            \
+  {                                                                            \
+    /* check negative too */                                                   \
+    auto eps = std::numeric_limits<type>::min();                               \
+    auto s = std::string("-") + #input;                                        \
+    auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());                \
+    REQUIRE(t.has_value());                                                    \
+    CHECK_LT(std::abs(t.value() - type(-input)), eps);                         \
+  }
 
 #define CHECK_INVALID_CONVERSION(input, type)                                  \
-    {                                                                          \
-        std::string s = input;                                                 \
-        auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());            \
-        CHECK_FALSE(t.has_value());                                            \
-    }
+  {                                                                            \
+    std::string s = input;                                                     \
+    auto t = ss::to_num<type>(s.c_str(), s.c_str() + s.size());                \
+    CHECK_FALSE(t.has_value());                                                \
+  }
 
 #define REQUIRE_VARIANT(var, el, type)                                         \
-    {                                                                          \
-        auto ptr = std::get_if<type>(&var);                                    \
-        REQUIRE(ptr);                                                          \
-        REQUIRE_EQ(el, *ptr);                                                  \
-    }
+  {                                                                            \
+    auto ptr = std::get_if<type>(&var);                                        \
+    REQUIRE(ptr);                                                              \
+    REQUIRE_EQ(el, *ptr);                                                      \
+  }
 
 #define CHECK_NOT_VARIANT(var, type) CHECK(!std::holds_alternative<type>(var));

test/test_parser.cpp

@@ -283,7 +283,7 @@ TEST_CASE("parser test various cases") {
         }
 
         std::copy_if(data.begin(), data.end(), expected.begin(),
-                     [](const X& x) { return x.i != excluded; });
+                     [&](const X& x) { return x.i != excluded; });
         CHECK_EQ(i, expected);
         CHECK_EQ(i2, expected);
     }
@@ -378,7 +378,7 @@ TEST_CASE("parser test composite conversion") {
             p.try_next<int, int, double>(fail)
                 .or_else<test_struct>(fail)
                 .or_else<int, char, double>(
-                    [](auto&& data) { CHECK_EQ(data, expectedData); })
+                    [&](auto&& data) { CHECK_EQ(data, expectedData); })
                 .on_error(fail)
                 .or_else<test_tuple>(fail)
                 .values();
@@ -396,7 +396,7 @@ TEST_CASE("parser test composite conversion") {
         constexpr static auto expectedData = std::tuple{10, 20, 11.1};
 
         auto [d1, d2, d3, d4] =
-            p.try_next<int, int, double>([](auto& i1, auto i2, double d) {
+            p.try_next<int, int, double>([&](auto& i1, auto i2, double d) {
                  CHECK_EQ(std::tie(i1, i2, d), expectedData);
              })
                 .on_error(fail)
@@ -564,8 +564,6 @@ TEST_CASE("parser test composite conversion") {
     CHECK(p.eof());
 }
 
-size_t move_called = 0;
-
 struct my_string {
     char* data{nullptr};
 
@@ -580,12 +578,10 @@ struct my_string {
     my_string& operator=(const my_string&) = delete;
 
     my_string(my_string&& other) : data{other.data} {
-        move_called++;
         other.data = nullptr;
     }
 
     my_string& operator=(my_string&& other) {
-        move_called++;
         data = other.data;
         return *this;
     }
@@ -609,49 +605,6 @@ struct xyz {
     }
 };
 
-TEST_CASE("parser test the moving of parsed values") {
-    {
-        unique_file_name f;
-        {
-            std::ofstream out{f.name};
-            out << "x" << std::endl;
-        }
-
-        ss::parser p{f.name, ","};
-        auto x = p.get_next<my_string>();
-        CHECK_LE(move_called, 1);
-        move_called = 0;
-    }
-
-    unique_file_name f;
-    {
-        std::ofstream out{f.name};
-        out << "a,b,c" << std::endl;
-    }
-
-    {
-
-        ss::parser p{f.name, ","};
-        auto x = p.get_next<my_string, my_string, my_string>();
-        CHECK_LE(move_called, 3);
-        move_called = 0;
-    }
-
-    {
-        ss::parser p{f.name, ","};
-        auto x = p.get_object<xyz, my_string, my_string, my_string>();
-        CHECK_LE(move_called, 6);
-        move_called = 0;
-    }
-
-    {
-        ss::parser p{f.name, ","};
-        auto x = p.get_next<xyz>();
-        CHECK_LE(move_called, 6);
-        move_called = 0;
-    }
-}
-
 TEST_CASE("parser test the moving of parsed composite values") {
     // to compile is enough
     return;