Merge pull request #42 from red0124/dev

Merge with development

include/ss/common.hpp

@@ -38,45 +38,40 @@ inline void* strict_realloc(void* ptr, size_t size) {
 }
 
 #if __unix__
-inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
+inline ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
-    return getline(lineptr, n, stream);
+    return getline(&lineptr, &n, file);
 }
 #else
 
 using ssize_t = intptr_t;
 
-ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
+ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
-    if (lineptr == nullptr || n == nullptr || fp == nullptr) {
-        errno = EINVAL;
-        return -1;
-    }
-
     char buff[get_line_initial_buffer_size];
 
-    if (*lineptr == nullptr || *n < sizeof(buff)) {
+    if (lineptr == nullptr || n < sizeof(buff)) {
         size_t new_n = sizeof(buff);
-        *lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
+        lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
-        *n = new_n;
+        n = new_n;
     }
 
-    (*lineptr)[0] = '\0';
+    lineptr[0] = '\0';
 
     size_t line_used = 0;
-    while (std::fgets(buff, sizeof(buff), fp) != nullptr) {
+    while (std::fgets(buff, sizeof(buff), file) != nullptr) {
-        line_used = std::strlen(*lineptr);
+        line_used = std::strlen(lineptr);
         size_t buff_used = std::strlen(buff);
 
-        if (*n <= buff_used + line_used) {
+        if (n <= buff_used + line_used) {
-            size_t new_n = *n * 2;
+            size_t new_n = n * 2;
-            *lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
+            lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
-            *n = new_n;
+            n = new_n;
         }
 
-        std::memcpy(*lineptr + line_used, buff, buff_used);
+        std::memcpy(lineptr + line_used, buff, buff_used);
         line_used += buff_used;
-        (*lineptr)[line_used] = '\0';
+        lineptr[line_used] = '\0';
 
-        if ((*lineptr)[line_used - 1] == '\n') {
+        if (lineptr[line_used - 1] == '\n') {
             return line_used;
         }
     }
@@ -86,4 +81,68 @@ ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
 
 #endif
 
+ssize_t get_line_buffer(char*& lineptr, size_t& n,
+                        const char* const csv_data_buffer, size_t csv_data_size,
+                        size_t& curr_char) {
+    if (curr_char >= csv_data_size) {
+        return -1;
+    }
+
+    if (lineptr == nullptr || n < get_line_initial_buffer_size) {
+        auto new_lineptr = static_cast<char*>(
+            strict_realloc(lineptr, get_line_initial_buffer_size));
+        lineptr = new_lineptr;
+        n = get_line_initial_buffer_size;
+    }
+
+    size_t line_used = 0;
+    while (curr_char < csv_data_size) {
+        if (line_used + 1 >= n) {
+            size_t new_n = n * 2;
+
+            char* new_lineptr =
+                static_cast<char*>(strict_realloc(lineptr, new_n));
+            n = new_n;
+            lineptr = new_lineptr;
+        }
+
+        auto c = csv_data_buffer[curr_char++];
+        lineptr[line_used++] = c;
+        if (c == '\n') {
+            lineptr[line_used] = '\0';
+            return line_used;
+        }
+    }
+
+    if (line_used != 0) {
+        lineptr[line_used] = '\0';
+        return line_used;
+    }
+
+    return -1;
+}
+
+std::tuple<ssize_t, bool> get_line(char*& buffer, size_t& buffer_size,
+                                   FILE* file,
+                                   const char* const csv_data_buffer,
+                                   size_t csv_data_size, size_t& curr_char) {
+    ssize_t ssize;
+    if (file) {
+        ssize = get_line_file(buffer, buffer_size, file);
+        curr_char = std::ftell(file);
+    } else {
+        ssize = get_line_buffer(buffer, buffer_size, csv_data_buffer,
+                                csv_data_size, curr_char);
+    }
+
+    if (ssize == -1) {
+        if (errno == ENOMEM) {
+            throw std::bad_alloc{};
+        }
+        return {ssize, true};
+    }
+
+    return {ssize, false};
+}
+
 } /* ss */

include/ss/converter.hpp

@@ -150,7 +150,7 @@ public:
         if constexpr (sizeof...(Ts) == 0 && is_instance_of_v<std::tuple, T>) {
             return convert_impl(elems, static_cast<T*>(nullptr));
         } else if constexpr (tied_class_v<T, Ts...>) {
-            using arg_ref_tuple = std::result_of_t<decltype (&T::tied)(T)>;
+            using arg_ref_tuple = std::invoke_result_t<decltype(&T::tied), T>;
             using arg_tuple = apply_trait_t<std::decay, arg_ref_tuple>;
 
             return to_object<T>(
@@ -269,6 +269,7 @@ private:
 
     void handle_error_multiline_limit_reached() {
         constexpr static auto error_msg = "multiline limit reached";
+        splitter_.unterminated_quote_ = false;
 
         if constexpr (string_error) {
             error_.clear();

include/ss/parser.hpp

@@ -749,46 +749,9 @@ private:
         reader(const reader& other) = delete;
         reader& operator=(const reader& other) = delete;
 
-        ssize_t get_line_buffer(char** lineptr, size_t* n,
-                                const char* const csv_data_buffer,
-                                size_t csv_data_size, size_t& curr_char) {
-            if (curr_char >= csv_data_size) {
-                return -1;
-            }
-
-            if (*lineptr == nullptr || *n < get_line_initial_buffer_size) {
-                auto new_lineptr = static_cast<char*>(
-                    strict_realloc(*lineptr, get_line_initial_buffer_size));
-                *lineptr = new_lineptr;
-                *n = get_line_initial_buffer_size;
-            }
-
-            size_t line_used = 0;
-            while (curr_char <= csv_data_size) {
-                if (line_used + 1 >= *n) {
-                    size_t new_n = *n * 2;
-
-                    char* new_lineptr =
-                        static_cast<char*>(strict_realloc(*lineptr, new_n));
-                    *n = new_n;
-                    *lineptr = new_lineptr;
-                }
-
-                auto c = csv_data_buffer[curr_char++];
-                (*lineptr)[line_used++] = c;
-                if (c == '\n') {
-                    (*lineptr)[line_used] = '\0';
-                    return line_used;
-                }
-            }
-
-            return (line_used != 0) ? line_used : -1;
-        }
-
         // read next line each time in order to set eof_
         bool read_next() {
             next_line_converter_.clear_error();
-            ssize_t ssize = 0;
             size_t size = 0;
             while (size == 0) {
                 ++line_number_;
@@ -797,21 +760,11 @@ private:
                 }
 
                 chars_read_ = curr_char_;
-                if (file_) {
+                auto [ssize, eof] =
-                    ssize = get_line_file(&next_line_buffer_,
+                    get_line(next_line_buffer_, next_line_buffer_size_, file_,
-                                          &next_line_buffer_size_, file_);
+                             csv_data_buffer_, csv_data_size_, curr_char_);
-                    curr_char_ = std::ftell(file_);
-                } else {
-                    ssize = get_line_buffer(&next_line_buffer_,
-                                            &next_line_buffer_size_,
-                                            csv_data_buffer_, csv_data_size_,
-                                            curr_char_);
-                }
 
-                if (ssize == -1) {
+                if (eof) {
-                    if (errno == ENOMEM) {
-                        throw std::bad_alloc{};
-                    }
                     return false;
                 }
 
@@ -836,7 +789,8 @@ private:
                     }
 
                     if (!append_next_line_to_buffer(next_line_buffer_,
-                                                    next_line_size_)) {
+                                                    next_line_size_,
+                                                    next_line_buffer_size_)) {
                         next_line_converter_.handle_error_unterminated_escape();
                         return;
                     }
@@ -854,7 +808,8 @@ private:
                     }
 
                     if (!append_next_line_to_buffer(next_line_buffer_,
-                                                    next_line_size_)) {
+                                                    next_line_size_,
+                                                    next_line_buffer_size_)) {
                         next_line_converter_.handle_error_unterminated_quote();
                         return;
                     }
@@ -865,8 +820,9 @@ private:
                                 return;
                             }
 
-                            if (!append_next_line_to_buffer(next_line_buffer_,
+                            if (!append_next_line_to_buffer(
-                                                            next_line_size_)) {
+                                    next_line_buffer_, next_line_size_,
+                                    next_line_buffer_size_)) {
                                 next_line_converter_
                                     .handle_error_unterminated_escape();
                                 return;
@@ -910,18 +866,20 @@ private:
             return next_line_converter_.unterminated_quote();
         }
 
-        void undo_remove_eol(char* buffer, size_t& string_end) {
+        void undo_remove_eol(char* buffer, size_t& line_size,
-            if (crlf_) {
+                             size_t buffer_size) {
-                std::copy_n("\r\n\0", 3, buffer + string_end);
+            if (crlf_ && buffer_size >= line_size + 2) {
-                string_end += 2;
+                std::copy_n("\r\n", 2, buffer + line_size);
-            } else {
+                line_size += 2;
-                std::copy_n("\n\0", 2, buffer + string_end);
+            } else if (buffer_size > line_size) {
-                string_end += 1;
+                std::copy_n("\n", 1, buffer + line_size);
+                line_size += 1;
             }
         }
 
         size_t remove_eol(char*& buffer, size_t ssize) {
             if (buffer[ssize - 1] != '\n') {
+                crlf_ = false;
                 return ssize;
             }
 
@@ -949,28 +907,23 @@ private:
             first_size += second_size;
         }
 
-        bool append_next_line_to_buffer(char*& buffer, size_t& size) {
+        bool append_next_line_to_buffer(char*& buffer, size_t& line_size,
-            undo_remove_eol(buffer, size);
+                                        size_t buffer_size) {
+            undo_remove_eol(buffer, line_size, buffer_size);
 
-            ssize_t next_ssize;
+            chars_read_ = curr_char_;
-            if (file_) {
+            auto [next_ssize, eof] =
-                next_ssize =
+                get_line(helper_buffer_, helper_buffer_size, file_,
-                    get_line_file(&helper_buffer_, &helper_buffer_size, file_);
+                         csv_data_buffer_, csv_data_size_, curr_char_);
-            } else {
-                next_ssize =
-                    get_line_buffer(&helper_buffer_, &helper_buffer_size,
-                                    csv_data_buffer_, csv_data_size_,
-                                    curr_char_);
-            }
 
-            if (next_ssize == -1) {
+            if (eof) {
                 return false;
             }
 
             ++line_number_;
             size_t next_size = remove_eol(helper_buffer_, next_ssize);
-            realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_,
+            realloc_concat(buffer, line_size, next_line_buffer_size_,
-                           next_size);
+                           helper_buffer_, next_size);
             return true;
         }
 

ssp.hpp

@@ -650,45 +650,40 @@ inline void* strict_realloc(void* ptr, size_t size) {
 }
 
 #if __unix__
-inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
+inline ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
-    return getline(lineptr, n, stream);
+    return getline(&lineptr, &n, file);
 }
 #else
 
 using ssize_t = intptr_t;
 
-ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
+ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
-    if (lineptr == nullptr || n == nullptr || fp == nullptr) {
-        errno = EINVAL;
-        return -1;
-    }
-
     char buff[get_line_initial_buffer_size];
 
-    if (*lineptr == nullptr || *n < sizeof(buff)) {
+    if (lineptr == nullptr || n < sizeof(buff)) {
         size_t new_n = sizeof(buff);
-        *lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
+        lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
-        *n = new_n;
+        n = new_n;
     }
 
-    (*lineptr)[0] = '\0';
+    lineptr[0] = '\0';
 
     size_t line_used = 0;
-    while (std::fgets(buff, sizeof(buff), fp) != nullptr) {
+    while (std::fgets(buff, sizeof(buff), file) != nullptr) {
-        line_used = std::strlen(*lineptr);
+        line_used = std::strlen(lineptr);
         size_t buff_used = std::strlen(buff);
 
-        if (*n <= buff_used + line_used) {
+        if (n <= buff_used + line_used) {
-            size_t new_n = *n * 2;
+            size_t new_n = n * 2;
-            *lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
+            lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
-            *n = new_n;
+            n = new_n;
         }
 
-        std::memcpy(*lineptr + line_used, buff, buff_used);
+        std::memcpy(lineptr + line_used, buff, buff_used);
         line_used += buff_used;
-        (*lineptr)[line_used] = '\0';
+        lineptr[line_used] = '\0';
 
-        if ((*lineptr)[line_used - 1] == '\n') {
+        if (lineptr[line_used - 1] == '\n') {
             return line_used;
         }
     }
@@ -698,6 +693,70 @@ ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
 
 #endif
 
+ssize_t get_line_buffer(char*& lineptr, size_t& n,
+                        const char* const csv_data_buffer, size_t csv_data_size,
+                        size_t& curr_char) {
+    if (curr_char >= csv_data_size) {
+        return -1;
+    }
+
+    if (lineptr == nullptr || n < get_line_initial_buffer_size) {
+        auto new_lineptr = static_cast<char*>(
+            strict_realloc(lineptr, get_line_initial_buffer_size));
+        lineptr = new_lineptr;
+        n = get_line_initial_buffer_size;
+    }
+
+    size_t line_used = 0;
+    while (curr_char < csv_data_size) {
+        if (line_used + 1 >= n) {
+            size_t new_n = n * 2;
+
+            char* new_lineptr =
+                static_cast<char*>(strict_realloc(lineptr, new_n));
+            n = new_n;
+            lineptr = new_lineptr;
+        }
+
+        auto c = csv_data_buffer[curr_char++];
+        lineptr[line_used++] = c;
+        if (c == '\n') {
+            lineptr[line_used] = '\0';
+            return line_used;
+        }
+    }
+
+    if (line_used != 0) {
+        lineptr[line_used] = '\0';
+        return line_used;
+    }
+
+    return -1;
+}
+
+std::tuple<ssize_t, bool> get_line(char*& buffer, size_t& buffer_size,
+                                   FILE* file,
+                                   const char* const csv_data_buffer,
+                                   size_t csv_data_size, size_t& curr_char) {
+    ssize_t ssize;
+    if (file) {
+        ssize = get_line_file(buffer, buffer_size, file);
+        curr_char = std::ftell(file);
+    } else {
+        ssize = get_line_buffer(buffer, buffer_size, csv_data_buffer,
+                                csv_data_size, curr_char);
+    }
+
+    if (ssize == -1) {
+        if (errno == ENOMEM) {
+            throw std::bad_alloc{};
+        }
+        return {ssize, true};
+    }
+
+    return {ssize, false};
+}
+
 } /* ss */
 
 namespace ss {
@@ -1843,7 +1902,7 @@ public:
         if constexpr (sizeof...(Ts) == 0 && is_instance_of_v<std::tuple, T>) {
             return convert_impl(elems, static_cast<T*>(nullptr));
         } else if constexpr (tied_class_v<T, Ts...>) {
-            using arg_ref_tuple = std::result_of_t<decltype (&T::tied)(T)>;
+            using arg_ref_tuple = std::invoke_result_t<decltype(&T::tied), T>;
             using arg_tuple = apply_trait_t<std::decay, arg_ref_tuple>;
 
             return to_object<T>(
@@ -1962,6 +2021,7 @@ private:
 
     void handle_error_multiline_limit_reached() {
         constexpr static auto error_msg = "multiline limit reached";
+        splitter_.unterminated_quote_ = false;
 
         if constexpr (string_error) {
             error_.clear();
@@ -2925,46 +2985,9 @@ private:
         reader(const reader& other) = delete;
         reader& operator=(const reader& other) = delete;
 
-        ssize_t get_line_buffer(char** lineptr, size_t* n,
-                                const char* const csv_data_buffer,
-                                size_t csv_data_size, size_t& curr_char) {
-            if (curr_char >= csv_data_size) {
-                return -1;
-            }
-
-            if (*lineptr == nullptr || *n < get_line_initial_buffer_size) {
-                auto new_lineptr = static_cast<char*>(
-                    strict_realloc(*lineptr, get_line_initial_buffer_size));
-                *lineptr = new_lineptr;
-                *n = get_line_initial_buffer_size;
-            }
-
-            size_t line_used = 0;
-            while (curr_char <= csv_data_size) {
-                if (line_used + 1 >= *n) {
-                    size_t new_n = *n * 2;
-
-                    char* new_lineptr =
-                        static_cast<char*>(strict_realloc(*lineptr, new_n));
-                    *n = new_n;
-                    *lineptr = new_lineptr;
-                }
-
-                auto c = csv_data_buffer[curr_char++];
-                (*lineptr)[line_used++] = c;
-                if (c == '\n') {
-                    (*lineptr)[line_used] = '\0';
-                    return line_used;
-                }
-            }
-
-            return (line_used != 0) ? line_used : -1;
-        }
-
         // read next line each time in order to set eof_
         bool read_next() {
             next_line_converter_.clear_error();
-            ssize_t ssize = 0;
             size_t size = 0;
             while (size == 0) {
                 ++line_number_;
@@ -2973,21 +2996,11 @@ private:
                 }
 
                 chars_read_ = curr_char_;
-                if (file_) {
+                auto [ssize, eof] =
-                    ssize = get_line_file(&next_line_buffer_,
+                    get_line(next_line_buffer_, next_line_buffer_size_, file_,
-                                          &next_line_buffer_size_, file_);
+                             csv_data_buffer_, csv_data_size_, curr_char_);
-                    curr_char_ = std::ftell(file_);
-                } else {
-                    ssize = get_line_buffer(&next_line_buffer_,
-                                            &next_line_buffer_size_,
-                                            csv_data_buffer_, csv_data_size_,
-                                            curr_char_);
-                }
 
-                if (ssize == -1) {
+                if (eof) {
-                    if (errno == ENOMEM) {
-                        throw std::bad_alloc{};
-                    }
                     return false;
                 }
 
@@ -3012,7 +3025,8 @@ private:
                     }
 
                     if (!append_next_line_to_buffer(next_line_buffer_,
-                                                    next_line_size_)) {
+                                                    next_line_size_,
+                                                    next_line_buffer_size_)) {
                         next_line_converter_.handle_error_unterminated_escape();
                         return;
                     }
@@ -3030,7 +3044,8 @@ private:
                     }
 
                     if (!append_next_line_to_buffer(next_line_buffer_,
-                                                    next_line_size_)) {
+                                                    next_line_size_,
+                                                    next_line_buffer_size_)) {
                         next_line_converter_.handle_error_unterminated_quote();
                         return;
                     }
@@ -3041,8 +3056,9 @@ private:
                                 return;
                             }
 
-                            if (!append_next_line_to_buffer(next_line_buffer_,
+                            if (!append_next_line_to_buffer(
-                                                            next_line_size_)) {
+                                    next_line_buffer_, next_line_size_,
+                                    next_line_buffer_size_)) {
                                 next_line_converter_
                                     .handle_error_unterminated_escape();
                                 return;
@@ -3086,18 +3102,20 @@ private:
             return next_line_converter_.unterminated_quote();
         }
 
-        void undo_remove_eol(char* buffer, size_t& string_end) {
+        void undo_remove_eol(char* buffer, size_t& line_size,
-            if (crlf_) {
+                             size_t buffer_size) {
-                std::copy_n("\r\n\0", 3, buffer + string_end);
+            if (crlf_ && buffer_size >= line_size + 2) {
-                string_end += 2;
+                std::copy_n("\r\n", 2, buffer + line_size);
-            } else {
+                line_size += 2;
-                std::copy_n("\n\0", 2, buffer + string_end);
+            } else if (buffer_size > line_size) {
-                string_end += 1;
+                std::copy_n("\n", 1, buffer + line_size);
+                line_size += 1;
             }
         }
 
         size_t remove_eol(char*& buffer, size_t ssize) {
             if (buffer[ssize - 1] != '\n') {
+                crlf_ = false;
                 return ssize;
             }
 
@@ -3125,28 +3143,23 @@ private:
             first_size += second_size;
         }
 
-        bool append_next_line_to_buffer(char*& buffer, size_t& size) {
+        bool append_next_line_to_buffer(char*& buffer, size_t& line_size,
-            undo_remove_eol(buffer, size);
+                                        size_t buffer_size) {
+            undo_remove_eol(buffer, line_size, buffer_size);
 
-            ssize_t next_ssize;
+            chars_read_ = curr_char_;
-            if (file_) {
+            auto [next_ssize, eof] =
-                next_ssize =
+                get_line(helper_buffer_, helper_buffer_size, file_,
-                    get_line_file(&helper_buffer_, &helper_buffer_size, file_);
+                         csv_data_buffer_, csv_data_size_, curr_char_);
-            } else {
-                next_ssize =
-                    get_line_buffer(&helper_buffer_, &helper_buffer_size,
-                                    csv_data_buffer_, csv_data_size_,
-                                    curr_char_);
-            }
 
-            if (next_ssize == -1) {
+            if (eof) {
                 return false;
             }
 
             ++line_number_;
             size_t next_size = remove_eol(helper_buffer_, next_ssize);
-            realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_,
+            realloc_concat(buffer, line_size, next_line_buffer_size_,
-                           next_size);
+                           helper_buffer_, next_size);
             return true;
         }
 

test/test_parser1_3.cpp

@@ -16,13 +16,14 @@ TEST_CASE_TEMPLATE("test multiline restricted", T, ParserOptionCombinations) {
         out << "5,6,just\\\n\\\nstrings" << std::endl;
 #endif
         out << "7,8,ju\\\n\\\n\\\nnk" << std::endl;
+        out << "99,100,\"\n\n\n\n" << std::endl;
         out << "9,10,\"just\\\n\nstrings\"" << std::endl;
         out << "11,12,\"ju\\\n|\n\n\n\n\nk\"" << std::endl;
         out << "13,14,\"ju\\\n\\\n15,16\"\\\n\\\\\n\nnk\"" << std::endl;
         out << "17,18,\"ju\\\n\\\n\\\n\\\\\n\nnk\"" << std::endl;
         out << "19,20,just strings" << std::endl;
     }
-    auto bad_lines = 15;
+    auto bad_lines = 20;
     auto num_errors = 0;
 
     auto [p, _] =