parse_json.hpp

Go to the documentation of this file.

#ifndef UZUKI2_PARSE_JSON_HPP
#define UZUKI2_PARSE_JSON_HPP
 
#include <memory>
#include <vector>
#include <cctype>
#include <string>
#include <stdexcept>
#include <cmath>
#include <unordered_map>
#include <unordered_set>
#include <type_traits>
#include <cstddef>
 
#include "byteme/byteme.hpp"
#include "millijson/millijson.hpp"
#include "ritsuko/ritsuko.hpp"
#include "sanisizer/sanisizer.hpp"
 
#include "interfaces.hpp"
#include "Dummy.hpp"
#include "ExternalTracker.hpp"
#include "ParsedList.hpp"
#include "utils.hpp"
 
namespace uzuki2 {
 
namespace json {
 
inline const std::vector<std::shared_ptr<millijson::Base> >& extract_array(
    const std::unordered_map<std::string, std::shared_ptr<millijson::Base> >& properties, 
    const std::string& name, 
    const std::string& path) 
{
    auto vIt = properties.find(name);
    if (vIt == properties.end()) {
        throw std::runtime_error("expected '" + name + "' property for object at '" + path + "'");
    }
 
    const auto& values_ptr = vIt->second;
    if (values_ptr->type() != millijson::ARRAY) {
        throw std::runtime_error("expected an array in '" + path + "/" + name + "'"); 
    }
 
    return static_cast<const millijson::Array*>(values_ptr.get())->value();
}
 
inline const millijson::Array* has_names(const std::unordered_map<std::string, std::shared_ptr<millijson::Base> >& properties, const std::string& path) {
    auto nIt = properties.find("names");
    if (nIt == properties.end()) {
        return NULL;
    }
 
    const auto name_ptr = nIt->second;
    if (name_ptr->type() != millijson::ARRAY) {
        throw std::runtime_error("expected an array in '" + path + "/names'"); 
    }
    return static_cast<const millijson::Array*>(name_ptr.get());
}
 
template<class Destination_>
void fill_names(const millijson::Array* names_ptr, Destination_* dest, const std::string& path) {
    const auto& names = names_ptr->value();
    if (names.size() != dest->size()) {
        throw std::runtime_error("length of 'names' and 'values' should be the same in '" + path + "'"); 
    }
 
    const auto nnames = names.size();
    for (I<decltype(nnames)> i = 0; i < nnames; ++i) {
        if (names[i]->type() != millijson::STRING) {
            throw std::runtime_error("expected a string at '" + path + "/names/" + std::to_string(i) + "'");
        }
        dest->set_name(i, static_cast<const millijson::String*>(names[i].get())->value());
    }
}
 
template<class Function_>
auto process_array_or_scalar_values(
    const std::unordered_map<std::string, std::shared_ptr<millijson::Base> >& properties, 
    const std::string& path,
    Function_ fun)
{
    auto vIt = properties.find("values");
    if (vIt == properties.end()) {
        throw std::runtime_error("expected 'values' property for object at '" + path + "'");
    }
 
    auto names_ptr = has_names(properties, path);
    bool has_names = names_ptr != NULL;
 
    typename std::invoke_result<Function_,std::vector<std::shared_ptr<millijson::Base> >,bool,bool>::type out_ptr;
 
    const auto& values_ptr = vIt->second;
    if (values_ptr->type() == millijson::ARRAY) {
        out_ptr = fun(static_cast<const millijson::Array*>(values_ptr.get())->value(), has_names, false);
    } else {
        std::vector<std::shared_ptr<millijson::Base> > temp { values_ptr };
        out_ptr = fun(temp, has_names, true);
    }
 
    if (has_names) {
        fill_names(names_ptr, out_ptr, path);
    }
    return out_ptr;
}
 
template<class Destination_, class Function_>
void extract_integers(const std::vector<std::shared_ptr<millijson::Base> >& values, Destination_* dest, Function_ check, const std::string& path, const Version& version) {
    const auto n = values.size();
    for (I<decltype(n)> i = 0; i < n; ++i) {
        if (values[i]->type() == millijson::NOTHING) {
            dest->set_missing(i);
            continue;
        }
 
        if (values[i]->type() != millijson::NUMBER) {
            throw std::runtime_error("expected a number at '" + path + "/values/" + std::to_string(i) + "'");
        }
 
        auto val = static_cast<const millijson::Number*>(values[i].get())->value();
        if (val != std::floor(val)) {
            throw std::runtime_error("expected an integer at '" + path + "/values/" + std::to_string(i) + "'");
        }
 
        // 32-bit integers are always representable by doubles, as the latter have 53 bits of precision.
        constexpr double upper = std::numeric_limits<std::int32_t>::max();
        constexpr double lower = std::numeric_limits<std::int32_t>::min();
        if (val < lower || val > upper) {
            throw std::runtime_error("value at '" + path + "/values/" + std::to_string(i) + "' cannot be represented by a 32-bit signed integer");
        }
 
        const std::int32_t ival = val;
        if (version.equals(1, 0) && ival == -2147483648) {
            dest->set_missing(i);
            continue;
        }
 
        check(ival);
        dest->set(i, ival);
    }
}
 
template<class Destination_, class Function_>
void extract_strings(const std::vector<std::shared_ptr<millijson::Base> >& values, Destination_* dest, Function_ check, const std::string& path) {
    const auto n = values.size();
    for (I<decltype(n)> i = 0; i < n; ++i) {
        if (values[i]->type() == millijson::NOTHING) {
            dest->set_missing(i);
            continue;
        }
 
        if (values[i]->type() != millijson::STRING) {
            throw std::runtime_error("expected a string at '" + path + "/values/" + std::to_string(i) + "'");
        }
 
        const auto& str = static_cast<const millijson::String*>(values[i].get())->value();
        check(str);
        dest->set(i, str);
    }
}
 
template<class Provisioner_, class Externals_>
std::shared_ptr<Base> parse_object(const millijson::Base* contents, Externals_& ext, const std::string& path, const Version& version) {
    if (contents->type() != millijson::OBJECT) {
        throw std::runtime_error("each R object should be represented by a JSON object at '" + path + "'");
    }
    const auto& map = static_cast<const millijson::Object*>(contents)->value();
 
    auto tIt = map.find("type");
    if (tIt == map.end()) {
        throw std::runtime_error("missing 'type' property for JSON object at '" + path + "'");
    }
    const auto& type_ptr = tIt->second;
    if (type_ptr->type() != millijson::STRING) {
        throw std::runtime_error("expected a string at '" + path + "/type'");
    }
    const auto& type = static_cast<const millijson::String*>(type_ptr.get())->value();
 
    std::shared_ptr<Base> output;
    if (type == "nothing") {
        output.reset(Provisioner_::new_Nothing());
 
    } else if (type == "external") {
        auto iIt = map.find("index");
        if (iIt == map.end()) {
            throw std::runtime_error("expected 'index' property for 'external' type at '" + path + "'");
        }
        const auto& index_ptr = iIt->second;
        if (index_ptr->type() != millijson::NUMBER) {
            throw std::runtime_error("expected a number at '" + path + "/index'");
        }
 
        const auto flt_index = static_cast<const millijson::Number*>(index_ptr.get())->value();
        if (flt_index != std::floor(flt_index)) {
            throw std::runtime_error("expected an integer at '" + path + "/index'");
        } else if (flt_index < 0) {
            throw std::runtime_error("expected a non-negative integer at '" + path + "/index'");
        }
 
        std::int32_t index;
        try {
            index = sanisizer::from_float<std::int32_t>(flt_index);
        } catch (...) {
            throw std::runtime_error("value at '" + path + "/index' should fit in a 32-bit signed integer");
        }
        if (sanisizer::is_greater_than_or_equal(index, ext.size())) {
            throw std::runtime_error("external index out of range at '" + path + "/index'");
        }
        output.reset(Provisioner_::new_External(ext.get(index)));
 
    } else if (type == "integer") {
        process_array_or_scalar_values(map, path, [&](const auto& vals, bool named, bool scalar) -> auto {
            auto ptr = Provisioner_::new_Integer(sanisizer::cast<std::size_t>(vals.size()), named, scalar);
            output.reset(ptr);
            extract_integers(vals, ptr, [](std::int32_t) -> void {}, path, version);
            return ptr;
        });
 
    } else if (type == "factor" || (version.equals(1, 0) && type == "ordered")) {
        bool ordered = false;
        if (type == "ordered") {
            ordered = true;
        } else {
            auto oIt = map.find("ordered");
            if (oIt != map.end()) {
                if (oIt->second->type() != millijson::BOOLEAN) {
                    throw std::runtime_error("expected a boolean at '" + path + "/ordered'");
                }
                ordered = static_cast<const millijson::Boolean*>((oIt->second).get())->value();
            }
        }
 
        const std::string levels_name = "levels"; // avoid dangling reference from casting of string literal.
        const auto& lvals = extract_array(map, levels_name, path);
        const auto nlevels = lvals.size();
        auto fptr = process_array_or_scalar_values(map, path, [&](const auto& vals, bool named, bool scalar) -> auto {
            auto ptr = Provisioner_::new_Factor(sanisizer::cast<std::size_t>(vals.size()), named, scalar, sanisizer::cast<std::size_t>(nlevels), ordered);
            output.reset(ptr);
            extract_integers(vals, ptr, [&](std::int32_t x) -> void {
                if (x < 0) {
                    throw std::runtime_error("factor indices should be non-negative in '" + path + "/values'");
                } else if (sanisizer::is_greater_than_or_equal(x, nlevels)) {
                    throw std::runtime_error("factor indices of out of range of levels in '" + path + "/values'");
                }
            }, path, version);
            return ptr;
        });
 
        std::unordered_set<std::string> existing;
        for (I<decltype(nlevels)> l = 0; l < nlevels; ++l) {
            if (lvals[l]->type() != millijson::STRING) {
                throw std::runtime_error("expected strings at '" + path + "/levels/" + std::to_string(l) + "'");
            }
 
            const auto& level = static_cast<const millijson::String*>(lvals[l].get())->value();
            if (existing.find(level) != existing.end()) {
                throw std::runtime_error("detected duplicate string at '" + path + "/levels/" + std::to_string(l) + "'");
            }
            fptr->set_level(l, level);
            existing.insert(level);
        }
 
    } else if (type == "boolean") {
        process_array_or_scalar_values(map, path, [&](const auto& vals, bool named, bool scalar) -> auto {
            const auto n = vals.size();
            auto ptr = Provisioner_::new_Boolean(sanisizer::cast<std::size_t>(n), named, scalar);
            output.reset(ptr);
 
            for (I<decltype(n)> i = 0; i < n; ++i) {
                if (vals[i]->type() == millijson::NOTHING) {
                    ptr->set_missing(i);
                    continue;
                }
 
                if (vals[i]->type() != millijson::BOOLEAN) {
                    throw std::runtime_error("expected a boolean at '" + path + "/values/" + std::to_string(i) + "'");
                }
                ptr->set(i, static_cast<const millijson::Boolean*>(vals[i].get())->value());
            }
 
            return ptr;
        });
 
    } else if (type == "number") {
        process_array_or_scalar_values(map, path, [&](const auto& vals, bool named, bool scalar) -> auto {
            const auto n = vals.size();
            auto ptr = Provisioner_::new_Number(sanisizer::cast<std::size_t>(n), named, scalar);
            output.reset(ptr);
 
            for (I<decltype(n)> i = 0; i < n; ++i) {
                if (vals[i]->type() == millijson::NOTHING) {
                    ptr->set_missing(i);
                    continue;
                }
 
                if (vals[i]->type() == millijson::NUMBER) {
                    ptr->set(i, static_cast<const millijson::Number*>(vals[i].get())->value());
                } else if (vals[i]->type() == millijson::STRING) {
                    auto str = static_cast<const millijson::String*>(vals[i].get())->value();
                    if (str == "NaN") {
                        ptr->set(i, std::numeric_limits<double>::quiet_NaN());
                    } else if (str == "Inf") {
                        ptr->set(i, std::numeric_limits<double>::infinity());
                    } else if (str == "-Inf") {
                        ptr->set(i, -std::numeric_limits<double>::infinity());
                    } else {
                        throw std::runtime_error("unsupported string '" + str + "' at '" + path + "/values/" + std::to_string(i) + "'");
                    }
                } else {
                    throw std::runtime_error("expected a number at '" + path + "/values/" + std::to_string(i) + "'");
                }
            }
 
            return ptr;
        });
 
    } else if (type == "string" || (version.equals(1, 0) && (type == "date" || type == "date-time"))) {
        StringVector::Format format = StringVector::NONE;
        if (version.equals(1, 0)) {
            if (type == "date") {
                format = StringVector::DATE;
            } else if (type == "date-time") {
                format = StringVector::DATETIME;
            }
        } else {
            auto fIt = map.find("format");
            if (fIt != map.end()) {
                if (fIt->second->type() != millijson::STRING) {
                    throw std::runtime_error("expected a string at '" + path + "/format'");
                }
                auto fptr = static_cast<const millijson::String*>(fIt->second.get());
                if (fptr->value() == "date") {
                    format = StringVector::DATE;
                } else if (fptr->value() == "date-time") {
                    format = StringVector::DATETIME;
                } else {
                    throw std::runtime_error("unsupported format '" + fptr->value() + "' at '" + path + "/format'");
                }
            }
        }
 
        process_array_or_scalar_values(map, path, [&](const auto& vals, bool named, bool scalar) -> auto {
            auto ptr = Provisioner_::new_String(sanisizer::cast<std::size_t>(vals.size()), named, scalar, format);
            output.reset(ptr);
 
            if (format == StringVector::NONE) {
                extract_strings(vals, ptr, [](const std::string&) -> void {}, path);
            } else if (format == StringVector::DATE) {
                extract_strings(vals, ptr, [&](const std::string& x) -> void {
                    if (!ritsuko::is_date(x.c_str(), x.size())) {
                         throw std::runtime_error("dates should follow YYYY-MM-DD formatting in '" + path + "/values'");
                    }
                }, path);
            } else if (format == StringVector::DATETIME) {
                extract_strings(vals, ptr, [&](const std::string& x) -> void {
                    if (!ritsuko::is_rfc3339(x.c_str(), x.size())) {
                         throw std::runtime_error("date-times should follow the Internet Date/Time format in '" + path + "/values'");
                    }
                }, path);
            }
 
            return ptr;
        });
 
    } else if (type == "list") {
        auto names_ptr = has_names(map, path);
        const bool has_names = names_ptr != NULL;
 
        const std::string values_name = "values"; // avoid dangling reference from casting of string literal.
        const auto& vals = extract_array(map, values_name, path);
 
        const auto n = vals.size();
        auto ptr = Provisioner_::new_List(sanisizer::cast<std::size_t>(n), has_names);
        output.reset(ptr);
 
        for (I<decltype(n)> i = 0; i < n; ++i) {
            ptr->set(i, parse_object<Provisioner_>(vals[i].get(), ext, path + "/values/" + std::to_string(i), version));
        }
 
        if (has_names) {
            fill_names(names_ptr, ptr, path);
        }
 
    } else {
        throw std::runtime_error("unknown object type '" + type + "' at '" + path + "/type'");
    }
 
    return output;
}
struct Options {
    bool parallel = false;
 
    bool strict_list = true;
 
    std::size_t buffer_size = 65536;
};
 
template<class Provisioner_, class Reader_, class Externals_>
ParsedList parse(Reader_& reader, Externals_ ext, const Options& options) {
    millijson::ParseOptions popt;
    popt.buffer_size = options.buffer_size;
    popt.parallel = options.parallel;
    auto contents = millijson::parse(reader, popt);
 
    Version version;
    if (contents->type() == millijson::OBJECT) {
        const auto& map = static_cast<const millijson::Object*>(contents.get())->value();
        auto vIt = map.find("version");
        if (vIt != map.end()) {
            if (vIt->second->type() != millijson::STRING) {
                throw std::runtime_error("expected a string in 'version'");
            }
            const auto& vstr = static_cast<const millijson::String*>(vIt->second.get())->value();
            auto vraw = ritsuko::parse_version_string(vstr.c_str(), vstr.size(), /* skip_patch = */ true);
            version.major = vraw.major;
            version.minor = vraw.minor;
        }
    }
 
    ExternalTracker etrack(std::move(ext));
    auto output = parse_object<Provisioner_>(contents.get(), etrack, "", version);
 
    if (options.strict_list && output->type() != LIST) {
        throw std::runtime_error("top-level object should represent an R list");
    }
    etrack.validate();
 
    return ParsedList(std::move(output), std::move(version));
}
 
template<class Provisioner_, class Externals_>
ParsedList parse_file(const std::string& file, Externals_ ext, const Options& options) {
    std::unique_ptr<byteme::Reader> ptr;
    if (byteme::is_gzip(file.c_str())) {
        ptr.reset(new byteme::GzipFileReader(file.c_str(), {}));
    } else {
        ptr.reset(new byteme::RawFileReader(file.c_str(), {}));
    }
    return parse<Provisioner_>(*ptr, std::move(ext), options);
}
 
template<class Provisioner_, class Externals_>
ParsedList parse_buffer(const unsigned char* buffer, std::size_t len, Externals_ ext, const Options& options) {
    std::unique_ptr<byteme::Reader> ptr;
    if (byteme::is_zlib_or_gzip(buffer, len)) {
        ptr.reset(new byteme::ZlibBufferReader(buffer, len, {}));
    } else {
        ptr.reset(new byteme::RawBufferReader(buffer, len));
    }
    return parse<Provisioner_>(*ptr, std::move(ext), options);
}
 
inline void validate(byteme::Reader& reader, int num_external, const Options& options) {
    parse<DummyProvisioner>(reader, DummyExternals(num_external), options);
}
 
inline void validate_file(const std::string& file, int num_external, const Options& options) {
    parse_file<DummyProvisioner>(file, DummyExternals(num_external), options);
}
 
inline void validate_buffer(const unsigned char* buffer, std::size_t len, int num_external, const Options& options) {
    parse_buffer<DummyProvisioner>(buffer, len, DummyExternals(num_external), options);
}
 
}
 
}
 
#endif