Stream1dArray.hpp

Go to the documentation of this file.

#ifndef RITSUKO_CVLS_STREAM_1D_ARRAY_HPP
#define RITSUKO_CVLS_STREAM_1D_ARRAY_HPP
 
#include <vector>
#include <string>
#include <stdexcept>
#include <cstdint>
#include <cassert>
#include <algorithm>
 
#include "H5Cpp.h"
#include "sanisizer/sanisizer.hpp"
 
#include "../strnlen.hpp"
#include "../itoa.hpp"
#include "Pointer.hpp"
 
namespace ritsuko {
 
namespace cvls {
 
struct Stream1dArrayOptions {
    hsize_t contiguous_chunk_size = sanisizer::cap<hsize_t>(10000);
};
 
template<typename Offset_, typename Length_, typename DataSetPointer_ = const H5::DataSet*>
class Stream1dArray {
public:
    Stream1dArray(DataSetPointer_ pointers_ptr, hsize_t pointers_length, DataSetPointer_ heap_ptr, hsize_t heap_length, const Stream1dArrayOptions& options) : 
        my_pointers_ptr(std::move(pointers_ptr)), 
        my_heap_ptr(std::move(heap_ptr)),
        my_pointer_full_length(pointers_length), 
        my_heap_full_length(heap_length),
        my_pointer_block_size([&]{
            assert(my_pointers_ptr->getSpace().getSimpleExtentNdims() == 1);
            hsize_t output;
            const auto& plist = my_pointers_ptr->getCreatePlist();
            if (plist.getLayout() == H5D_CHUNKED) {
                plist.getChunk(1, &output);
            } else {
                output = std::min(my_pointer_full_length, options.contiguous_chunk_size);
            }
            return output;
        }()),
        my_pointer_mspace(1, &my_pointer_block_size),
        my_pointer_dspace(1, &my_pointer_full_length),
        my_heap_dspace(1, &my_heap_full_length),
        my_pointer_dtype(define_pointer_datatype<Offset_, Length_>()),
        my_pointer_buffer(sanisizer::cast<I<decltype(my_pointer_buffer.size())> >(my_pointer_block_size))
    {
        // Check that maximum allocation is possible, so we don't have to check casts for individual string lengths.
        sanisizer::cast<I<decltype(my_heap_buffer.size())> >(my_heap_full_length);
    }
 
public:
    hsize_t chunk_size() const {
        return my_pointer_block_size;
    }
 
    hsize_t load(std::string* buffer) {
        my_last_loaded += my_available;
        my_available = std::min(my_pointer_full_length - my_last_loaded, my_pointer_block_size);
        if (my_available == 0) {
            return 0;
        }
 
        constexpr hsize_t zero = 0;
        my_pointer_mspace.selectHyperslab(H5S_SELECT_SET, &my_available, &zero);
        my_pointer_dspace.selectHyperslab(H5S_SELECT_SET, &my_available, &my_last_loaded);
        my_heap_dspace.selectNone();
        my_pointers_ptr->read(my_pointer_buffer.data(), my_pointer_dtype, my_pointer_mspace, my_pointer_dspace);
 
        for (I<decltype(my_available)> i = 0; i < my_available; ++i) {
            const auto& val = my_pointer_buffer[i];
            if (is_Pointer_out_of_range(val, my_heap_full_length)) {
                throw std::runtime_error("pointer is out of range of the heap at position " + itoa(i + my_last_loaded));
            }
 
            auto& curstr = buffer[i];
            curstr.clear();
 
            if (val.length) {
                // Casts are safe if the pointers are within range.
                const hsize_t start = val.offset;
                const hsize_t count = val.length;
 
                // Don't attempt to batch these reads as we aren't guaranteed
                // that they are non-overlapping or ordered. Hopefully HDF5 is
                // keeping enough things in cache for repeated reads.
                my_heap_mspace.setExtentSimple(1, &count);
                my_heap_mspace.selectAll();
                my_heap_dspace.selectHyperslab(H5S_SELECT_SET, &count, &start);
                my_heap_buffer.resize(count);
                my_heap_ptr->read(my_heap_buffer.data(), H5::PredType::NATIVE_UINT8, my_heap_mspace, my_heap_dspace);
                const char* text_ptr = reinterpret_cast<const char*>(my_heap_buffer.data());
                curstr.insert(curstr.end(), text_ptr, text_ptr + strnlen(text_ptr, count));
            }
        }
 
        return my_available;
    }
 
    hsize_t start() const {
        return my_last_loaded;
    }
 
private:
    DataSetPointer_ my_pointers_ptr;
    DataSetPointer_ my_heap_ptr;
    hsize_t my_pointer_full_length, my_heap_full_length;
    hsize_t my_pointer_block_size;
    H5::DataSpace my_pointer_mspace, my_pointer_dspace;
    H5::DataSpace my_heap_mspace, my_heap_dspace;
 
    H5::DataType my_pointer_dtype;
    std::vector<Pointer<Offset_, Length_> > my_pointer_buffer;
    std::vector<std::uint8_t> my_heap_buffer;
 
    hsize_t my_last_loaded = 0;
    hsize_t my_available = 0;
 
    static_assert(std::is_base_of<H5::DataSet, I<decltype(*my_pointers_ptr)> >::value);
    static_assert(std::is_base_of<H5::DataSet, I<decltype(*my_heap_ptr)> >::value);
};
 
}
 
}
 
#endif