Stream1dStringDataset.hpp

Go to the documentation of this file.

#ifndef RITSUKO_HDF5_STREAM_1D_STRING_DATASET_HPP
#define RITSUKO_HDF5_STREAM_1D_STRING_DATASET_HPP
 
#include <vector>
#include <string>
#include <stdexcept>
#include <cassert>
#include <type_traits>
#include <algorithm>
 
#include "H5Cpp.h"
#include "sanisizer/sanisizer.hpp"
 
#include "ReclaimVlsMemory.hpp"
#include "../utils.hpp"
#include "../strnlen.hpp"
#include "../itoa.hpp"
 
namespace ritsuko {
 
namespace hdf5 {
 
struct Stream1dStringDatasetOptions {
    hsize_t contiguous_chunk_size = sanisizer::cap<hsize_t>(10000);
};
 
template<class DataSetPointer_ = const H5::DataSet*>
class Stream1dStringDataset {
public:
    Stream1dStringDataset(DataSetPointer_ data_ptr, hsize_t length, const Stream1dStringDatasetOptions& options) :
        my_data_ptr(std::move(data_ptr)), 
        my_full_length(length), 
        my_block_size([&]{
            assert(my_data_ptr->getSpace().getSimpleExtentNdims() == 1);
            hsize_t output;
            const auto& plist = my_data_ptr->getCreatePlist();
            if (plist.getLayout() == H5D_CHUNKED) {
                plist.getChunk(1, &output);
            } else {
                output = std::min(length, options.contiguous_chunk_size);
            }
            return output;
        }()),
        my_mspace(1, &my_block_size),
        my_fspace(1, &my_full_length),
        my_dtype(my_data_ptr->getDataType()),
        my_is_variable(my_dtype.isVariableStr())
    {
        assert(my_dtype.getClass() == H5T_STRING);
        if (my_is_variable) {
            sanisizer::resize(my_var_buffer, my_block_size);
        } else {
            my_fixed_length = my_dtype.getSize();
            my_fix_buffer.resize(sanisizer::product<I<decltype(my_fix_buffer.size())> >(my_fixed_length, my_block_size));
        }
    }
 
public:
    hsize_t chunk_size() const {
        return my_block_size;
    }
 
    hsize_t load(std::string* buffer) {
        my_last_loaded += my_available;
        my_available = sanisizer::min(my_full_length - my_last_loaded, my_block_size);
        if (my_available == 0) {
            return 0;
        }
 
        constexpr hsize_t zero = 0;
        my_mspace.selectHyperslab(H5S_SELECT_SET, &my_available, &zero);
        my_fspace.selectHyperslab(H5S_SELECT_SET, &my_available, &my_last_loaded);
 
        if (my_is_variable) {
            my_data_ptr->read(my_var_buffer.data(), my_dtype, my_mspace, my_fspace);
            const auto& plist = H5::DSetMemXferPropList::DEFAULT;
            [[maybe_unused]] ReclaimVlsMemory deletor(&my_dtype, &my_mspace, &plist, my_var_buffer.data());
            for (hsize_t i = 0; i < my_available; ++i) {
                if (my_var_buffer[i] == NULL) {
                    throw std::runtime_error("detected a NULL pointer for a variable length string at position " + itoa(start() + i));
                }
                auto& curstr = buffer[i];
                curstr.clear();
                curstr.insert(0, my_var_buffer[i]);
            }
 
        } else {
            my_data_ptr->read(my_fix_buffer.data(), my_dtype, my_mspace, my_fspace);
            for (I<decltype(my_available)> i = 0; i < my_available; ++i) {
                auto& curstr = buffer[i];
                curstr.clear();
                auto bptr = my_fix_buffer.data() + sanisizer::product_unsafe<std::size_t>(i, my_fixed_length);
                curstr.insert(curstr.end(), bptr, bptr + strnlen(bptr, my_fixed_length));
            }
        }
 
        return my_available;
    }
 
    hsize_t start() const {
        return my_last_loaded;
    }
 
private:
    DataSetPointer_ my_data_ptr;
    hsize_t my_full_length, my_block_size;
    H5::DataSpace my_mspace;
    H5::DataSpace my_fspace;
 
    H5::DataType my_dtype;
    bool my_is_variable;
    std::vector<char*> my_var_buffer;
    std::size_t my_fixed_length = 0;
    std::vector<char> my_fix_buffer;
 
    hsize_t my_last_loaded = 0;
    hsize_t my_available = 0;
 
    static_assert(std::is_base_of<H5::DataSet, I<decltype(*my_data_ptr)> >::value);
};
 
}
 
}
 
#endif