utils_csv.hpp

Go to the documentation of this file.

#ifndef TAKANE_UTILS_CSV_HPP
#define TAKANE_UTILS_CSV_HPP
 
#include "ritsuko/ritsuko.hpp"
#include "comservatory/comservatory.hpp"
 
#include <unordered_set>
#include <string>
#include <stdexcept>
#include <vector>
#include <memory>
 
namespace takane {
 
struct CsvFieldCreator {
    ~CsvFieldCreator() = default;
    virtual comservatory::NumberField* integer() {
        return number();
    }
 
    virtual comservatory::StringField* string() = 0;
 
    virtual comservatory::NumberField* number() = 0;
 
    virtual comservatory::BooleanField* boolean() = 0;
};
 
struct DummyCsvFieldCreator : public CsvFieldCreator {
    comservatory::StringField* string() {
        return new comservatory::DummyStringField;
    }
 
    comservatory::NumberField* number() {
        return new comservatory::DummyNumberField;
    }
 
    comservatory::BooleanField* boolean() {
        return new comservatory::DummyBooleanField;
    }
};
 
struct CsvContents {
    std::vector<std::unique_ptr<comservatory::Field> > fields;
 
    void reconstitute(std::vector<std::unique_ptr<comservatory::Field> >& source) {
        for (size_t c = 0, total = source.size(); c < total; ++c) { 
            if (!fields[c]) {
                fields[c].swap(source[c]);
            }
        }
    }
};
 
struct CsvNameField : public comservatory::StringField {
    CsvNameField(bool ar, comservatory::StringField* p) : as_rownames(ar), child(p) {}
 
public:
    bool as_rownames;
    comservatory::StringField* child;
 
public:
    void add_missing() {
        throw std::runtime_error("missing values should not be present in the " + (as_rownames ? std::string("row names") : std::string("names")) + " column");
    }
 
    void push_back(std::string x) {
        child->push_back(std::move(x));
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const {
        return true;
    }
};
 
template<typename T>
constexpr T upper_integer_limit() {
    return 2147483647;
}
 
template<typename T>
constexpr T lower_integer_limit() {
    return -2147483648;
}
 
struct CsvIntegerField : public comservatory::NumberField {
    CsvIntegerField(int cid, comservatory::NumberField* p) : column_id(cid), child(p) {}
 
public:
    int column_id;
    comservatory::NumberField* child;
 
public:
    void add_missing() {
        child->add_missing();
    }
 
    void push_back(double x) {
        if (x < lower_integer_limit<double>() || x > upper_integer_limit<double>()) { // constrain within limits.
            throw std::runtime_error("value in column " + std::to_string(column_id + 1) + " does not fit inside a 32-bit signed integer");
        }
        if (x != std::floor(x)) {
            throw std::runtime_error("value in column " + std::to_string(column_id + 1) + " is not an integer");
        }
        child->push_back(x);
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const {
        return true;
    }
};
 
struct CsvNonNegativeIntegerField : public comservatory::NumberField {
    CsvNonNegativeIntegerField(int cid, comservatory::NumberField* p) : column_id(cid), child(p) {}
 
public:
    int column_id;
    comservatory::NumberField* child;
 
public:
    void add_missing() {
        child->add_missing();
    }
 
    void push_back(double x) {
        if (x < 0) {
            throw std::runtime_error("value in column " + std::to_string(column_id + 1) + " should not be negative");
        }
        if (x > upper_integer_limit<double>()) {
            throw std::runtime_error("value in column " + std::to_string(column_id + 1) + " does not fit inside a 32-bit signed integer");
        }
        if (x != std::floor(x)) {
            throw std::runtime_error("value in column " + std::to_string(column_id + 1) + " is not an integer");
        }
        child->push_back(x);
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const {
        return true;
    }
};
 
struct CsvDateField : public comservatory::StringField {
    CsvDateField(int cid, comservatory::StringField* p) : column_id(cid), child(p) {}
 
public:
    int column_id;
    comservatory::StringField* child;
 
public:
    void push_back(std::string x) {
        if (!ritsuko::is_date(x.c_str(), x.size())) {
            throw std::runtime_error("expected a date in column " + std::to_string(column_id + 1) + ", got '" + x + "' instead");
        }
        child->push_back(std::move(x));
    }
 
    void add_missing() {
        child->add_missing();
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const { 
        return true;
    }
};
 
struct CsvDateTimeField : public comservatory::StringField {
    CsvDateTimeField(int cid, comservatory::StringField* p) : column_id(cid), child(p) {}
 
public:
    int column_id;
    comservatory::StringField* child;
 
public:
    void push_back(std::string x) {
        if (!ritsuko::is_rfc3339(x.c_str(), x.size())) {
            throw std::runtime_error("expected an Internet date/time in column " + std::to_string(column_id + 1) + ", got '" + x + "' instead");
        }
        child->push_back(std::move(x));
    }
 
    void add_missing() {
        child->add_missing();
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const { 
        return true;
    }
};
 
struct CsvFactorV1Field : public comservatory::StringField {
    CsvFactorV1Field(int cid, const std::unordered_set<std::string>* l, comservatory::StringField* p) : column_id(cid), levels(l), child(p) {}
 
public:
    int column_id;
    const std::unordered_set<std::string>* levels;
    comservatory::StringField* child;
 
public:
    void push_back(std::string x) {
        if (levels->find(x) == levels->end()) {
            throw std::runtime_error("value '" + x + "' in column " + std::to_string(column_id + 1) + " does not refer to a valid level");
        }
        child->push_back(std::move(x));
    }
 
    void add_missing() {
        child->add_missing();
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const { 
        return true;
    }
};
 
struct CsvFactorV2Field : public comservatory::NumberField {
    CsvFactorV2Field(int cid, size_t l, comservatory::NumberField* p) : column_id(cid), nlevels(l), child(p) {
        if (nlevels > upper_integer_limit<size_t>()) {
            throw std::runtime_error("number of levels must fit into a 32-bit signed integer");
        }
    }
 
public:
    int column_id;
    double nlevels; // casting for an easier comparison.
    comservatory::NumberField* child;
 
public:
    void push_back(double x) {
        if (x < 0 || x >= nlevels) {
            throw std::runtime_error("code in column " + std::to_string(column_id + 1) + " should be non-negative and less than the number of levels");
        }
        if (x != std::floor(x)) {
            throw std::runtime_error("value in column " + std::to_string(column_id + 1) + " is not an integer");
        }
        child->push_back(x);
    }
 
    void add_missing() {
        child->add_missing();
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const { 
        return true;
    }
};
 
struct CsvCompressedLengthField : public CsvNonNegativeIntegerField {
    CsvCompressedLengthField(int cid, comservatory::NumberField* p) : CsvNonNegativeIntegerField(cid, p) {}
 
    void add_missing() {
        throw std::runtime_error("lengths should not be missing");
    }
 
    void push_back(double x) {
        CsvNonNegativeIntegerField::push_back(x);
        total += static_cast<size_t>(x);
    }
 
    size_t total = 0;
};
 
struct CsvUniqueStringField : public comservatory::StringField {
    CsvUniqueStringField(int cid, comservatory::StringField* p) : column_id(cid), child(p) {}
 
private:
    int column_id;
    comservatory::StringField* child;
    std::unordered_set<std::string> collected;
 
public:
    void add_missing() {
        throw std::runtime_error("missing values should not be present in column " + std::to_string(column_id));
    }
 
    void push_back(std::string x) {
        if (collected.find(x) != collected.end()) {
            throw std::runtime_error("duplicated value '" + x + "' in column " + std::to_string(column_id));
        }
        collected.insert(x);
        child->push_back(std::move(x));
    }
 
    size_t size() const {
        return child->size();
    }
 
    bool filled() const { 
        return true;
    }
};
}
 
#endif