5 Commits

7 changed files with 282 additions and 199 deletions

55
.github/workflows/macos-apple-clang.yml vendored Normal file
View File

@@ -0,0 +1,55 @@
name: macos-apple-clang-ci
on:
workflow_dispatch:
push:
branches:
- master
- feature/**
- improvement/**
- bugfix/**
pull_request:
branches:
- master
- feature/**
- improvement/**
- bugfix/**
jobs:
clang_tests:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
strategy:
matrix:
xcode: ['13.4.1', '14.1']
type: [Release, Debug]
runs-on: macos-12
env:
DEVELOPER_DIR: /Applications/Xcode_${{matrix.xcode}}.app/Contents/Developer
name: "Xcode ${{matrix.xcode}}: ${{matrix.type}}"
steps:
- uses: actions/checkout@v3
- uses: friendlyanon/fetch-core-count@v1
id: cores
- name: Install dependencies
run: script/ci_install_deps.sh
- name: Configure
run: cmake -S test -B build -DCMAKE_BUILD_TYPE=${{matrix.type}}
- name: Build
run: cmake --build build -j ${{steps.cores.outputs.count}}
- name: Run
working-directory: build
run: ctest --output-on-failure

View File

@@ -16,6 +16,7 @@
[![windows-msys2-gcc](https://github.com/red0124/ssp/workflows/win-msys2-gcc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msys2-gcc.yml)
[![windows-msys2-clang](https://github.com/red0124/ssp/workflows/win-msys2-clang-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msys2-clang.yml)
[![windows-msvc](https://github.com/red0124/ssp/workflows/win-msvc-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/win-msvc.yml)
[![macos-apple-clang](https://github.com/red0124/ssp/workflows/macos-apple-clang-ci/badge.svg)](https://github.com/red0124/ssp/actions/workflows/macos-apple-clang.yml)
A header only CSV parser which is fast and versatile with modern C++ API. Requires compiler with C++17 support. [Can also be used to efficiently convert strings to specific types.](#the-converter)

View File

@@ -38,45 +38,40 @@ inline void* strict_realloc(void* ptr, size_t size) {
}
#if __unix__
inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
return getline(lineptr, n, stream);
inline ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
return getline(&lineptr, &n, file);
}
#else
using ssize_t = intptr_t;
ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
if (lineptr == nullptr || n == nullptr || fp == nullptr) {
errno = EINVAL;
return -1;
}
ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
char buff[get_line_initial_buffer_size];
if (*lineptr == nullptr || *n < sizeof(buff)) {
if (lineptr == nullptr || n < sizeof(buff)) {
size_t new_n = sizeof(buff);
*lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
*n = new_n;
lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
n = new_n;
}
(*lineptr)[0] = '\0';
lineptr[0] = '\0';
size_t line_used = 0;
while (std::fgets(buff, sizeof(buff), fp) != nullptr) {
line_used = std::strlen(*lineptr);
while (std::fgets(buff, sizeof(buff), file) != nullptr) {
line_used = std::strlen(lineptr);
size_t buff_used = std::strlen(buff);
if (*n <= buff_used + line_used) {
size_t new_n = *n * 2;
*lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
*n = new_n;
if (n <= buff_used + line_used) {
size_t new_n = n * 2;
lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
n = new_n;
}
std::memcpy(*lineptr + line_used, buff, buff_used);
std::memcpy(lineptr + line_used, buff, buff_used);
line_used += buff_used;
(*lineptr)[line_used] = '\0';
lineptr[line_used] = '\0';
if ((*lineptr)[line_used - 1] == '\n') {
if (lineptr[line_used - 1] == '\n') {
return line_used;
}
}
@@ -86,4 +81,68 @@ ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
#endif
ssize_t get_line_buffer(char*& lineptr, size_t& n,
const char* const csv_data_buffer, size_t csv_data_size,
size_t& curr_char) {
if (curr_char >= csv_data_size) {
return -1;
}
if (lineptr == nullptr || n < get_line_initial_buffer_size) {
auto new_lineptr = static_cast<char*>(
strict_realloc(lineptr, get_line_initial_buffer_size));
lineptr = new_lineptr;
n = get_line_initial_buffer_size;
}
size_t line_used = 0;
while (curr_char < csv_data_size) {
if (line_used + 1 >= n) {
size_t new_n = n * 2;
char* new_lineptr =
static_cast<char*>(strict_realloc(lineptr, new_n));
n = new_n;
lineptr = new_lineptr;
}
auto c = csv_data_buffer[curr_char++];
lineptr[line_used++] = c;
if (c == '\n') {
lineptr[line_used] = '\0';
return line_used;
}
}
if (line_used != 0) {
lineptr[line_used] = '\0';
return line_used;
}
return -1;
}
std::tuple<ssize_t, bool> get_line(char*& buffer, size_t& buffer_size,
FILE* file,
const char* const csv_data_buffer,
size_t csv_data_size, size_t& curr_char) {
ssize_t ssize;
if (file) {
ssize = get_line_file(buffer, buffer_size, file);
curr_char = std::ftell(file);
} else {
ssize = get_line_buffer(buffer, buffer_size, csv_data_buffer,
csv_data_size, curr_char);
}
if (ssize == -1) {
if (errno == ENOMEM) {
throw std::bad_alloc{};
}
return {ssize, true};
}
return {ssize, false};
}
} /* ss */

View File

@@ -150,7 +150,7 @@ public:
if constexpr (sizeof...(Ts) == 0 && is_instance_of_v<std::tuple, T>) {
return convert_impl(elems, static_cast<T*>(nullptr));
} else if constexpr (tied_class_v<T, Ts...>) {
using arg_ref_tuple = std::result_of_t<decltype (&T::tied)(T)>;
using arg_ref_tuple = std::invoke_result_t<decltype(&T::tied), T>;
using arg_tuple = apply_trait_t<std::decay, arg_ref_tuple>;
return to_object<T>(
@@ -269,6 +269,7 @@ private:
void handle_error_multiline_limit_reached() {
constexpr static auto error_msg = "multiline limit reached";
splitter_.unterminated_quote_ = false;
if constexpr (string_error) {
error_.clear();

View File

@@ -749,46 +749,9 @@ private:
reader(const reader& other) = delete;
reader& operator=(const reader& other) = delete;
ssize_t get_line_buffer(char** lineptr, size_t* n,
const char* const csv_data_buffer,
size_t csv_data_size, size_t& curr_char) {
if (curr_char >= csv_data_size) {
return -1;
}
if (*lineptr == nullptr || *n < get_line_initial_buffer_size) {
auto new_lineptr = static_cast<char*>(
strict_realloc(*lineptr, get_line_initial_buffer_size));
*lineptr = new_lineptr;
*n = get_line_initial_buffer_size;
}
size_t line_used = 0;
while (curr_char <= csv_data_size) {
if (line_used + 1 >= *n) {
size_t new_n = *n * 2;
char* new_lineptr =
static_cast<char*>(strict_realloc(*lineptr, new_n));
*n = new_n;
*lineptr = new_lineptr;
}
auto c = csv_data_buffer[curr_char++];
(*lineptr)[line_used++] = c;
if (c == '\n') {
(*lineptr)[line_used] = '\0';
return line_used;
}
}
return (line_used != 0) ? line_used : -1;
}
// read next line each time in order to set eof_
bool read_next() {
next_line_converter_.clear_error();
ssize_t ssize = 0;
size_t size = 0;
while (size == 0) {
++line_number_;
@@ -797,21 +760,11 @@ private:
}
chars_read_ = curr_char_;
if (file_) {
ssize = get_line_file(&next_line_buffer_,
&next_line_buffer_size_, file_);
curr_char_ = std::ftell(file_);
} else {
ssize = get_line_buffer(&next_line_buffer_,
&next_line_buffer_size_,
csv_data_buffer_, csv_data_size_,
curr_char_);
}
auto [ssize, eof] =
get_line(next_line_buffer_, next_line_buffer_size_, file_,
csv_data_buffer_, csv_data_size_, curr_char_);
if (ssize == -1) {
if (errno == ENOMEM) {
throw std::bad_alloc{};
}
if (eof) {
return false;
}
@@ -836,7 +789,8 @@ private:
}
if (!append_next_line_to_buffer(next_line_buffer_,
next_line_size_)) {
next_line_size_,
next_line_buffer_size_)) {
next_line_converter_.handle_error_unterminated_escape();
return;
}
@@ -854,7 +808,8 @@ private:
}
if (!append_next_line_to_buffer(next_line_buffer_,
next_line_size_)) {
next_line_size_,
next_line_buffer_size_)) {
next_line_converter_.handle_error_unterminated_quote();
return;
}
@@ -865,8 +820,9 @@ private:
return;
}
if (!append_next_line_to_buffer(next_line_buffer_,
next_line_size_)) {
if (!append_next_line_to_buffer(
next_line_buffer_, next_line_size_,
next_line_buffer_size_)) {
next_line_converter_
.handle_error_unterminated_escape();
return;
@@ -910,18 +866,20 @@ private:
return next_line_converter_.unterminated_quote();
}
void undo_remove_eol(char* buffer, size_t& string_end) {
if (crlf_) {
std::copy_n("\r\n\0", 3, buffer + string_end);
string_end += 2;
} else {
std::copy_n("\n\0", 2, buffer + string_end);
string_end += 1;
void undo_remove_eol(char* buffer, size_t& line_size,
size_t buffer_size) {
if (crlf_ && buffer_size >= line_size + 2) {
std::copy_n("\r\n", 2, buffer + line_size);
line_size += 2;
} else if (buffer_size > line_size) {
std::copy_n("\n", 1, buffer + line_size);
line_size += 1;
}
}
size_t remove_eol(char*& buffer, size_t ssize) {
if (buffer[ssize - 1] != '\n') {
crlf_ = false;
return ssize;
}
@@ -949,28 +907,23 @@ private:
first_size += second_size;
}
bool append_next_line_to_buffer(char*& buffer, size_t& size) {
undo_remove_eol(buffer, size);
bool append_next_line_to_buffer(char*& buffer, size_t& line_size,
size_t buffer_size) {
undo_remove_eol(buffer, line_size, buffer_size);
ssize_t next_ssize;
if (file_) {
next_ssize =
get_line_file(&helper_buffer_, &helper_buffer_size, file_);
} else {
next_ssize =
get_line_buffer(&helper_buffer_, &helper_buffer_size,
csv_data_buffer_, csv_data_size_,
curr_char_);
}
chars_read_ = curr_char_;
auto [next_ssize, eof] =
get_line(helper_buffer_, helper_buffer_size, file_,
csv_data_buffer_, csv_data_size_, curr_char_);
if (next_ssize == -1) {
if (eof) {
return false;
}
++line_number_;
size_t next_size = remove_eol(helper_buffer_, next_ssize);
realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_,
next_size);
realloc_concat(buffer, line_size, next_line_buffer_size_,
helper_buffer_, next_size);
return true;
}

211
ssp.hpp
View File

@@ -650,45 +650,40 @@ inline void* strict_realloc(void* ptr, size_t size) {
}
#if __unix__
inline ssize_t get_line_file(char** lineptr, size_t* n, FILE* stream) {
return getline(lineptr, n, stream);
inline ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
return getline(&lineptr, &n, file);
}
#else
using ssize_t = intptr_t;
ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
if (lineptr == nullptr || n == nullptr || fp == nullptr) {
errno = EINVAL;
return -1;
}
ssize_t get_line_file(char*& lineptr, size_t& n, FILE* file) {
char buff[get_line_initial_buffer_size];
if (*lineptr == nullptr || *n < sizeof(buff)) {
if (lineptr == nullptr || n < sizeof(buff)) {
size_t new_n = sizeof(buff);
*lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
*n = new_n;
lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
n = new_n;
}
(*lineptr)[0] = '\0';
lineptr[0] = '\0';
size_t line_used = 0;
while (std::fgets(buff, sizeof(buff), fp) != nullptr) {
line_used = std::strlen(*lineptr);
while (std::fgets(buff, sizeof(buff), file) != nullptr) {
line_used = std::strlen(lineptr);
size_t buff_used = std::strlen(buff);
if (*n <= buff_used + line_used) {
size_t new_n = *n * 2;
*lineptr = static_cast<char*>(strict_realloc(*lineptr, new_n));
*n = new_n;
if (n <= buff_used + line_used) {
size_t new_n = n * 2;
lineptr = static_cast<char*>(strict_realloc(lineptr, new_n));
n = new_n;
}
std::memcpy(*lineptr + line_used, buff, buff_used);
std::memcpy(lineptr + line_used, buff, buff_used);
line_used += buff_used;
(*lineptr)[line_used] = '\0';
lineptr[line_used] = '\0';
if ((*lineptr)[line_used - 1] == '\n') {
if (lineptr[line_used - 1] == '\n') {
return line_used;
}
}
@@ -698,6 +693,70 @@ ssize_t get_line_file(char** lineptr, size_t* n, FILE* fp) {
#endif
ssize_t get_line_buffer(char*& lineptr, size_t& n,
const char* const csv_data_buffer, size_t csv_data_size,
size_t& curr_char) {
if (curr_char >= csv_data_size) {
return -1;
}
if (lineptr == nullptr || n < get_line_initial_buffer_size) {
auto new_lineptr = static_cast<char*>(
strict_realloc(lineptr, get_line_initial_buffer_size));
lineptr = new_lineptr;
n = get_line_initial_buffer_size;
}
size_t line_used = 0;
while (curr_char < csv_data_size) {
if (line_used + 1 >= n) {
size_t new_n = n * 2;
char* new_lineptr =
static_cast<char*>(strict_realloc(lineptr, new_n));
n = new_n;
lineptr = new_lineptr;
}
auto c = csv_data_buffer[curr_char++];
lineptr[line_used++] = c;
if (c == '\n') {
lineptr[line_used] = '\0';
return line_used;
}
}
if (line_used != 0) {
lineptr[line_used] = '\0';
return line_used;
}
return -1;
}
std::tuple<ssize_t, bool> get_line(char*& buffer, size_t& buffer_size,
FILE* file,
const char* const csv_data_buffer,
size_t csv_data_size, size_t& curr_char) {
ssize_t ssize;
if (file) {
ssize = get_line_file(buffer, buffer_size, file);
curr_char = std::ftell(file);
} else {
ssize = get_line_buffer(buffer, buffer_size, csv_data_buffer,
csv_data_size, curr_char);
}
if (ssize == -1) {
if (errno == ENOMEM) {
throw std::bad_alloc{};
}
return {ssize, true};
}
return {ssize, false};
}
} /* ss */
namespace ss {
@@ -1843,7 +1902,7 @@ public:
if constexpr (sizeof...(Ts) == 0 && is_instance_of_v<std::tuple, T>) {
return convert_impl(elems, static_cast<T*>(nullptr));
} else if constexpr (tied_class_v<T, Ts...>) {
using arg_ref_tuple = std::result_of_t<decltype (&T::tied)(T)>;
using arg_ref_tuple = std::invoke_result_t<decltype(&T::tied), T>;
using arg_tuple = apply_trait_t<std::decay, arg_ref_tuple>;
return to_object<T>(
@@ -1962,6 +2021,7 @@ private:
void handle_error_multiline_limit_reached() {
constexpr static auto error_msg = "multiline limit reached";
splitter_.unterminated_quote_ = false;
if constexpr (string_error) {
error_.clear();
@@ -2925,46 +2985,9 @@ private:
reader(const reader& other) = delete;
reader& operator=(const reader& other) = delete;
ssize_t get_line_buffer(char** lineptr, size_t* n,
const char* const csv_data_buffer,
size_t csv_data_size, size_t& curr_char) {
if (curr_char >= csv_data_size) {
return -1;
}
if (*lineptr == nullptr || *n < get_line_initial_buffer_size) {
auto new_lineptr = static_cast<char*>(
strict_realloc(*lineptr, get_line_initial_buffer_size));
*lineptr = new_lineptr;
*n = get_line_initial_buffer_size;
}
size_t line_used = 0;
while (curr_char <= csv_data_size) {
if (line_used + 1 >= *n) {
size_t new_n = *n * 2;
char* new_lineptr =
static_cast<char*>(strict_realloc(*lineptr, new_n));
*n = new_n;
*lineptr = new_lineptr;
}
auto c = csv_data_buffer[curr_char++];
(*lineptr)[line_used++] = c;
if (c == '\n') {
(*lineptr)[line_used] = '\0';
return line_used;
}
}
return (line_used != 0) ? line_used : -1;
}
// read next line each time in order to set eof_
bool read_next() {
next_line_converter_.clear_error();
ssize_t ssize = 0;
size_t size = 0;
while (size == 0) {
++line_number_;
@@ -2973,21 +2996,11 @@ private:
}
chars_read_ = curr_char_;
if (file_) {
ssize = get_line_file(&next_line_buffer_,
&next_line_buffer_size_, file_);
curr_char_ = std::ftell(file_);
} else {
ssize = get_line_buffer(&next_line_buffer_,
&next_line_buffer_size_,
csv_data_buffer_, csv_data_size_,
curr_char_);
}
auto [ssize, eof] =
get_line(next_line_buffer_, next_line_buffer_size_, file_,
csv_data_buffer_, csv_data_size_, curr_char_);
if (ssize == -1) {
if (errno == ENOMEM) {
throw std::bad_alloc{};
}
if (eof) {
return false;
}
@@ -3012,7 +3025,8 @@ private:
}
if (!append_next_line_to_buffer(next_line_buffer_,
next_line_size_)) {
next_line_size_,
next_line_buffer_size_)) {
next_line_converter_.handle_error_unterminated_escape();
return;
}
@@ -3030,7 +3044,8 @@ private:
}
if (!append_next_line_to_buffer(next_line_buffer_,
next_line_size_)) {
next_line_size_,
next_line_buffer_size_)) {
next_line_converter_.handle_error_unterminated_quote();
return;
}
@@ -3041,8 +3056,9 @@ private:
return;
}
if (!append_next_line_to_buffer(next_line_buffer_,
next_line_size_)) {
if (!append_next_line_to_buffer(
next_line_buffer_, next_line_size_,
next_line_buffer_size_)) {
next_line_converter_
.handle_error_unterminated_escape();
return;
@@ -3086,18 +3102,20 @@ private:
return next_line_converter_.unterminated_quote();
}
void undo_remove_eol(char* buffer, size_t& string_end) {
if (crlf_) {
std::copy_n("\r\n\0", 3, buffer + string_end);
string_end += 2;
} else {
std::copy_n("\n\0", 2, buffer + string_end);
string_end += 1;
void undo_remove_eol(char* buffer, size_t& line_size,
size_t buffer_size) {
if (crlf_ && buffer_size >= line_size + 2) {
std::copy_n("\r\n", 2, buffer + line_size);
line_size += 2;
} else if (buffer_size > line_size) {
std::copy_n("\n", 1, buffer + line_size);
line_size += 1;
}
}
size_t remove_eol(char*& buffer, size_t ssize) {
if (buffer[ssize - 1] != '\n') {
crlf_ = false;
return ssize;
}
@@ -3125,28 +3143,23 @@ private:
first_size += second_size;
}
bool append_next_line_to_buffer(char*& buffer, size_t& size) {
undo_remove_eol(buffer, size);
bool append_next_line_to_buffer(char*& buffer, size_t& line_size,
size_t buffer_size) {
undo_remove_eol(buffer, line_size, buffer_size);
ssize_t next_ssize;
if (file_) {
next_ssize =
get_line_file(&helper_buffer_, &helper_buffer_size, file_);
} else {
next_ssize =
get_line_buffer(&helper_buffer_, &helper_buffer_size,
csv_data_buffer_, csv_data_size_,
curr_char_);
}
chars_read_ = curr_char_;
auto [next_ssize, eof] =
get_line(helper_buffer_, helper_buffer_size, file_,
csv_data_buffer_, csv_data_size_, curr_char_);
if (next_ssize == -1) {
if (eof) {
return false;
}
++line_number_;
size_t next_size = remove_eol(helper_buffer_, next_ssize);
realloc_concat(buffer, size, next_line_buffer_size_, helper_buffer_,
next_size);
realloc_concat(buffer, line_size, next_line_buffer_size_,
helper_buffer_, next_size);
return true;
}

View File

@@ -16,13 +16,14 @@ TEST_CASE_TEMPLATE("test multiline restricted", T, ParserOptionCombinations) {
out << "5,6,just\\\n\\\nstrings" << std::endl;
#endif
out << "7,8,ju\\\n\\\n\\\nnk" << std::endl;
out << "99,100,\"\n\n\n\n" << std::endl;
out << "9,10,\"just\\\n\nstrings\"" << std::endl;
out << "11,12,\"ju\\\n|\n\n\n\n\nk\"" << std::endl;
out << "13,14,\"ju\\\n\\\n15,16\"\\\n\\\\\n\nnk\"" << std::endl;
out << "17,18,\"ju\\\n\\\n\\\n\\\\\n\nnk\"" << std::endl;
out << "19,20,just strings" << std::endl;
}
auto bad_lines = 15;
auto bad_lines = 20;
auto num_errors = 0;
auto [p, _] =