diff --git a/Makefile.am b/Makefile.am index 59e11202e..12ab3565e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -43,10 +43,8 @@ src_libbitcoin_database_la_SOURCES = \ src/locks/file_lock.cpp \ src/locks/flush_lock.cpp \ src/locks/interprocess_lock.cpp \ - src/memory/map.cpp \ + src/memory/mman.cpp \ src/memory/utilities.cpp \ - src/memory/mman-win32/mman.cpp \ - src/memory/mman-win32/mman.hpp \ src/types/history.cpp \ src/types/multisig_view.cpp \ src/types/unspent.cpp @@ -208,7 +206,11 @@ include_bitcoin_database_file_HEADERS = \ include_bitcoin_database_impl_memorydir = ${includedir}/bitcoin/database/impl/memory include_bitcoin_database_impl_memory_HEADERS = \ - include/bitcoin/database/impl/memory/accessor.ipp + include/bitcoin/database/impl/memory/accessor.ipp \ + include/bitcoin/database/impl/memory/map.ipp \ + include/bitcoin/database/impl/memory/map_dispatch.ipp \ + include/bitcoin/database/impl/memory/map_private.ipp \ + include/bitcoin/database/impl/memory/map_storage.ipp include_bitcoin_database_impl_primitivesdir = ${includedir}/bitcoin/database/impl/primitives include_bitcoin_database_impl_primitives_HEADERS = \ @@ -314,6 +316,7 @@ include_bitcoin_database_memory_HEADERS = \ include/bitcoin/database/memory/map.hpp \ include/bitcoin/database/memory/maps.hpp \ include/bitcoin/database/memory/memory.hpp \ + include/bitcoin/database/memory/mman.hpp \ include/bitcoin/database/memory/reader.hpp \ include/bitcoin/database/memory/streamers.hpp \ include/bitcoin/database/memory/utilities.hpp diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj index 6e45b8db2..aab855d3a 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj @@ -130,8 +130,7 @@ - - + $(IntDir)src_memory_utilities.obj @@ -159,6 +158,7 @@ + @@ -219,10 +219,13 @@ - + + + + diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters index 1124f31de..657e77058 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters @@ -91,11 +91,8 @@ {62D7FBEE-4D52-424A-0000-0000000000B1} - - {62D7FBEE-4D52-424A-0000-0000000000C1} - - {62D7FBEE-4D52-424A-0000-0000000000D1} + {62D7FBEE-4D52-424A-0000-0000000000C1} @@ -120,12 +117,9 @@ src\locks - + src\memory - - src\memory\mman-win32 - src\memory @@ -197,6 +191,9 @@ include\bitcoin\database\memory + + include\bitcoin\database\memory + include\bitcoin\database\memory @@ -377,14 +374,23 @@ include\bitcoin\database - - src\memory\mman-win32 - include\bitcoin\database\impl\memory + + include\bitcoin\database\impl\memory + + + include\bitcoin\database\impl\memory + + + include\bitcoin\database\impl\memory + + + include\bitcoin\database\impl\memory + include\bitcoin\database\impl\primitives diff --git a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj index 6219e530d..f3c730059 100644 --- a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj +++ b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj @@ -130,8 +130,7 @@ - - + $(IntDir)src_memory_utilities.obj @@ -159,6 +158,7 @@ + @@ -219,10 +219,13 @@ - + + + + diff --git a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters index 1124f31de..657e77058 100644 --- a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters +++ b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters @@ -91,11 +91,8 @@ {62D7FBEE-4D52-424A-0000-0000000000B1} - - {62D7FBEE-4D52-424A-0000-0000000000C1} - - {62D7FBEE-4D52-424A-0000-0000000000D1} + {62D7FBEE-4D52-424A-0000-0000000000C1} @@ -120,12 +117,9 @@ src\locks - + src\memory - - src\memory\mman-win32 - src\memory @@ -197,6 +191,9 @@ include\bitcoin\database\memory + + include\bitcoin\database\memory + include\bitcoin\database\memory @@ -377,14 +374,23 @@ include\bitcoin\database - - src\memory\mman-win32 - include\bitcoin\database\impl\memory + + include\bitcoin\database\impl\memory + + + include\bitcoin\database\impl\memory + + + include\bitcoin\database\impl\memory + + + include\bitcoin\database\impl\memory + include\bitcoin\database\impl\primitives diff --git a/include/bitcoin/database.hpp b/include/bitcoin/database.hpp index e1c9ecdb1..6f1e7729f 100644 --- a/include/bitcoin/database.hpp +++ b/include/bitcoin/database.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include diff --git a/include/bitcoin/database/impl/memory/map.ipp b/include/bitcoin/database/impl/memory/map.ipp new file mode 100644 index 000000000..c499ecb58 --- /dev/null +++ b/include/bitcoin/database/impl/memory/map.ipp @@ -0,0 +1,130 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_MEMORY_MAP_IPP +#define LIBBITCOIN_DATABASE_MEMORY_MAP_IPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +// Constructors. +// ---------------------------------------------------------------------------- + +TEMPLATE +CLASS::map1(const path& filename, size_t minimum, size_t expansion, + bool random) NOEXCEPT + requires (columns == one) + : filenames_{ filename }, + minimum_(minimum), + expansion_(expansion), + random_(random), + opened_{ file::invalid } +{ +} + +TEMPLATE +CLASS::map1(const paths& filenames, size_t minimum, size_t expansion, + bool random) NOEXCEPT + requires (columns > one) + : filenames_(filenames), + minimum_(minimum), + expansion_(expansion), + random_(random), + opened_{} +{ + opened_.fill(file::invalid); +} + +TEMPLATE +CLASS::~map1() NOEXCEPT +{ + BC_ASSERT_MSG(!loaded_, "..."); + BC_ASSERT_MSG(is_zero(logical_), "..."); + + BC_ASSERT_MSG(std::ranges::all_of(memory_map_, + [](auto map) NOEXCEPT { return is_null(map); }), "..."); + + BC_ASSERT_MSG(std::ranges::all_of(capacity_, + [](auto closed) NOEXCEPT { return is_zero(closed); }), "..."); + + BC_ASSERT_MSG(std::ranges::all_of(opened_, + [](auto opened) NOEXCEPT { return opened == file::invalid; }), "..."); +} + +TEMPLATE +bool CLASS::is_open() const NOEXCEPT +{ + std::shared_lock field_lock(field_mutex_); + return std::ranges::any_of(opened_, [](auto opened) NOEXCEPT + { + return opened != file::invalid; + }); +} + +TEMPLATE +bool CLASS::is_loaded() const NOEXCEPT +{ + std::shared_lock field_lock(field_mutex_); + return loaded_; +} + +// protected +// ---------------------------------------------------------------------------- + +TEMPLATE +size_t CLASS::to_capacity(size_t required) const NOEXCEPT +{ + BC_PUSH_WARNING(NO_STATIC_CAST) + const auto resize = required * ((expansion_ + 100.0) / 100.0); + const auto target = std::max(minimum_, static_cast(resize)); + BC_POP_WARNING() + + BC_ASSERT(target >= required); + return target; +} + +// Read-write protected by atomic, write-write protected by remap_mutex. +TEMPLATE +void CLASS::set_first_code(const error::error_t& ec) NOEXCEPT +{ + if (!fault_) + { + // fault is not exposed so requires no atomic (fast read). + fault_ = true; + + // error is atomic for public read exposure. + error_.store(ec); + } +} + +TEMPLATE +void CLASS::set_disk_space(size_t required) NOEXCEPT +{ + space_.store(required); +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/memory/map_dispatch.ipp b/include/bitcoin/database/impl/memory/map_dispatch.ipp new file mode 100644 index 000000000..52e253c59 --- /dev/null +++ b/include/bitcoin/database/impl/memory/map_dispatch.ipp @@ -0,0 +1,130 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_MEMORY_MAP_DISPATCH_IPP +#define LIBBITCOIN_DATABASE_MEMORY_MAP_DISPATCH_IPP + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +// Dispatch. +// ---------------------------------------------------------------------------- + +TEMPLATE +template > +memory_ptr CLASS::set_column(size_t offset, size_t size, + uint8_t backfill) NOEXCEPT +{ + { + std::unique_lock field_lock(field_mutex_); + + if (fault_ || !loaded_ || system::is_add_overflow(offset, size)) + return {}; + + const auto end = std::max(logical_, offset + size); + if (end > capacity_rows(capacity_)) + { + const auto capacity = to_capacity(end); + + // TODO: Could loop over a try lock here and log deadlock warning. + std::unique_lock remap_lock(remap_mutex_); + + // Disk full condition leaves store in valid state despite null. + if (!remap_all_(capacity, sequence{})) + return {}; + + // Fill new capacity as offset may not be at end due to expansion. + // Scalar-map path: column 0 in bytes (width 1, identity). + const auto first = to_bytes(logical_); + std::fill_n( + std::next(std::get(memory_map_), first), + std::get(capacity_) - first, backfill); + } + + logical_ = end; + } + + return get_column(offset); +} + +TEMPLATE +template > +memory_ptr CLASS::get_column(size_t offset) const NOEXCEPT +{ + // Obtaining size before access prevents mutual mutex wait (deadlock). + // The store could remap between here and next line, but capacity only + // increases. Close zeroizes capacity but file must be unloaded to do so. + // Truncate can reduce logical, but capacity is not affected. It is always + // safe to write past current logical within current capacity. + const auto allocated = to_bytes(size()); + + // Takes a shared lock on remap_mutex_ until destruct, blocking remap. + const auto ptr = std::make_shared(remap_mutex_); + + // loaded_ update is precluded by remap_mutex_, making this read atomic. + if (!loaded_ || is_null(ptr)) + return nullptr; + + // With offset > size the assignment is negative (stream is exhausted). + ptr->assign( + std::next(std::get(memory_map_), offset), + std::next(std::get(memory_map_), allocated)); + + return ptr; +} + +TEMPLATE +template > +memory_ptr CLASS::capacity_column(size_t offset) const NOEXCEPT +{ + // Same as get() but limited by capacity() vs. size(). + const auto allocated = std::get(capacity_); + const auto ptr = std::make_shared(remap_mutex_); + if (!loaded_ || is_null(ptr)) + return nullptr; + + ptr->assign( + std::next(std::get(memory_map_), offset), + std::next(std::get(memory_map_), allocated)); + return ptr; +} + +TEMPLATE +template > +memory::iterator CLASS::raw_column(size_t offset) const NOEXCEPT +{ + // Pointer is otherwise unguarded, not remap safe (use for table heads). + if (offset > to_bytes(size())) + return nullptr; + + return std::next(std::get(memory_map_), offset); +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/memory/map_private.ipp b/include/bitcoin/database/impl/memory/map_private.ipp new file mode 100644 index 000000000..697a8590c --- /dev/null +++ b/include/bitcoin/database/impl/memory/map_private.ipp @@ -0,0 +1,303 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_MEMORY_MAP_PRIVATE_IPP +#define LIBBITCOIN_DATABASE_MEMORY_MAP_PRIVATE_IPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +// mman dispatch, not thread safe. +// ---------------------------------------------------------------------------- +// private + +TEMPLATE +template +bool CLASS::map_all_(std::index_sequence) NOEXCEPT +{ + return (map_() && ...); +} + +TEMPLATE +template +bool CLASS::unmap_all_(std::index_sequence) NOEXCEPT +{ + return (unmap_() && ...); +} + +TEMPLATE +template +bool CLASS::flush_all_(std::index_sequence) NOEXCEPT +{ + return (flush_() && ...); +} + +TEMPLATE +template +bool CLASS::remap_all_(size_t logical, std::index_sequence) NOEXCEPT +{ + return (remap_(to_bytes(logical)) && ...); +} + +// mman wrappers, not thread safe. +// ---------------------------------------------------------------------------- +// private + +// Never results in unmapped. +TEMPLATE +template +bool CLASS::flush_() NOEXCEPT +{ +#if defined(HAVE_MSC) + // unmap (and therefore msync) must be called before ftruncate. + // "To flush all the dirty pages plus the metadata for the file and ensure + // that they are physically written to disk..." + const auto size = to_bytes(logical_); + const auto success = + (::msync(memory_map_[Column], size, MS_SYNC) != fail) + && (::fsync(opened_[Column]) != fail); +#elif defined(F_FULLFSYNC) + // macOS msync fails with zero logical size (but we are no longer calling). + // non-standard macOS behavior: news.ycombinator.com/item?id=30372218 + const auto success = ::fcntl(opened_[Column], F_FULLFSYNC, 0) != fail; +#else + // msync should not be required on modern linux, see linus et al. + // stackoverflow.com/questions/5902629/mmap-msync-and-linux-process-termination + // Linux: fsync "transfers ("flushes") all modified in-core data of + // (i.e., modified buffer cache pages for) the file referred to by the + // file descriptor fd to the disk device so all changed information + // can be retrieved even if the system crashes or is rebooted. This + // includes writing through or flushing a disk cache if present. The + // call blocks until the device reports that transfer has completed." + const auto success = ::fsync(opened_[Column]) != fail; +#endif + + if (!success) + set_first_code(error::fsync_failure); + + return success; +} + +// Always results in unmapped. +// Trims to logical size, can be zero. +TEMPLATE +template +bool CLASS::unmap_() NOEXCEPT +{ + const auto logical = to_bytes(logical_); + +#if defined(HAVE_MSC) + const auto success = + (::msync(memory_map_[Column], logical, MS_SYNC) != fail) + && (::munmap(memory_map_[Column], capacity_[Column]) != fail) + && (::ftruncate(opened_[Column], logical) != fail) + && (::fsync(opened_[Column]) != fail); +#else + const auto success = + (::ftruncate(opened_[Column], logical) != fail) + #if defined(F_FULLFSYNC) + && (::fcntl(opened_[Column], F_FULLFSYNC, 0) != fail) + #else + && (::fsync(opened_[Column]) != fail) + #endif + && (::munmap(memory_map_[Column], capacity_[Column]) != fail); +#endif + if (!success) + set_first_code(error::munmap_failure); + + loaded_ = false; + capacity_[Column] = zero; + memory_map_[Column] = {}; + return success; +} + +// Mapping failure results in unmapped. +// Mapping has no effect on logical size, always maps max(logical, min) size. +TEMPLATE +template +bool CLASS::map_() NOEXCEPT +{ + auto size = to_bytes(logical_); + + // Cannot map empty file, and want mininum capacity, so expand as required. + // disk_full: space is set but no code is set with false return. + const auto minimum = to_bytes(minimum_); + if ((size < minimum) && !resize_((size = minimum))) + return false; + + memory_map_[Column] = system::pointer_cast( + ::mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, + opened_[Column], 0)); + + return finalize_(size); +} + +// Remap failure results in unmapped. +// Remapping has no effect on logical size, sets map_/capacity_. +TEMPLATE +template +bool CLASS::remap_(size_t size) NOEXCEPT +{ + BC_ASSERT(size >= to_bytes(logical_)); + + // Cannot remap empty file, so expand to minimum capacity if zero. + if (is_zero(size)) + size = to_bytes(minimum_); + +#if !defined(HAVE_MSC) && !defined(MREMAP_MAYMOVE) + // macOS: unmap before ftruncate sets new size. + if (!unmap_()) + return false; + + // disk_full: unmap(ok), resize(fail for space), map(ok), return false. + // disk_full: if second unmap fails then code is set, and false return. + if (!resize_(size)) + { + /* bool */ map_(); + return false; + } +#else + // disk_full: space is set but no code is set with false return. + if (!resize_(size)) + return false; +#endif + +#if defined(HAVE_MSC) + + // mman-win32 mremap hack (umap/map) requires flags and file descriptor. + memory_map_[Column] = system::pointer_cast( + ::mremap_(memory_map_[Column], capacity_[Column], size, + PROT_READ | PROT_WRITE, MAP_SHARED, opened_[Column])); + +#elif defined(MREMAP_MAYMOVE) + + memory_map_[Column] = system::pointer_cast( + ::mremap(memory_map_[Column], capacity_[Column], size, MREMAP_MAYMOVE)); + +#else + + // macOS: does not define mremap or MREMAP_MAYMOVE. + // TODO: see "MREMAP_MAYMOVE" in sqlite for map extension technique. + memory_map_[Column] = system::pointer_cast( + ::mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, + opened_[Column], 0)); + +#endif + + return finalize_(size); +} + +// disk_full: space is set but no code is set with false return. +TEMPLATE +template +bool CLASS::resize_(size_t size) NOEXCEPT +{ + // Disk full detection, any other failure is an abort. +#if !defined (WITHOUT_FALLOCATE) + if (::fallocate(opened_[Column], 0, capacity_[Column], + size - capacity_[Column]) == fail) +#else + if (::ftruncate(opened_[Column], size) == fail) +#endif + { + // Disk full is the only restartable store failure (leave mapped). + if (errno == ENOSPC) + { + set_disk_space(size - capacity_[Column]); + return false; + } + + set_first_code(error::ftruncate_failure); + unmap_(); + return false; + } + + return true; +} + +// Finalize failure results in unmapped. +TEMPLATE +template +bool CLASS::finalize_(size_t size) NOEXCEPT +{ + if (memory_map_[Column] == MAP_FAILED) + { + loaded_ = false; + capacity_[Column] = zero; + memory_map_[Column] = {}; + + // mmap or mremap failure (not mapped). + set_first_code(error::mmap_failure); + return false; + } + +#if !defined (WITHOUT_MADVISE) +#if !defined(HAVE_MSC) + // Get page size (usually 4KB). + const int page_size = ::sysconf(_SC_PAGESIZE); + const auto page = system::possible_narrow_sign_cast(page_size); + + // If not one bit then page size is not a power of two as required. + if (page_size == fail || !is_one(system::ones_count(page))) + { + set_first_code(error::sysconf_failure); + unmap_(); + return false; + } + + // Align size up to page boundary. + using namespace system; + const auto max = sub1(page); + const auto align = bit_and(ceilinged_add(size, max), bit_not(max)); + + // Use 1GB chunks to avoid large-length issues. + constexpr auto chunk = power2(30u); + const auto advice = (random_ ? MADV_RANDOM : MADV_SEQUENTIAL) | + MADV_WILLNEED; + + for (auto offset = zero; offset < align; offset += chunk) + { + BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) + const auto start = memory_map_[Column] + offset; + BC_POP_WARNING() + + if (::madvise(start, std::min(chunk, align - offset), advice) == fail) + { + set_first_code(error::madvise_failure); + unmap_(); + return false; + } + } +#endif // !HAVE_MSC +#endif // !WITHOUT_MADVISE + + loaded_ = true; + capacity_[Column] = size; + return true; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/memory/map_storage.ipp b/include/bitcoin/database/impl/memory/map_storage.ipp new file mode 100644 index 000000000..6d083ef1f --- /dev/null +++ b/include/bitcoin/database/impl/memory/map_storage.ipp @@ -0,0 +1,398 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_MEMORY_MAP_STORAGE_IPP +#define LIBBITCOIN_DATABASE_MEMORY_MAP_STORAGE_IPP + +#include +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +// Interface. +// ---------------------------------------------------------------------------- + +TEMPLATE +code CLASS::get_fault() const NOEXCEPT +{ + return error_.load(); +} + +TEMPLATE +size_t CLASS::get_space() const NOEXCEPT +{ + return space_.load(); +} + +TEMPLATE +const std::filesystem::path& CLASS::file() const NOEXCEPT +{ + return filenames_.front(); +} + +TEMPLATE +code CLASS::create() const NOEXCEPT +{ + for (const auto& descriptor: opened_) + if (descriptor != file::invalid) + return error::create_open; + + for (const auto& name: filenames_) + if (const auto ec = file::create_file_ex(name)) + return ec; + + return error::success; +} + +TEMPLATE +code CLASS::open() NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + for (const auto& descriptor : opened_) + if (descriptor != file::invalid) + return error::open_open; + + // Windows doesn't use madvise, instead infers map access from file open. + for (size_t index{}; index < columns; ++index) + if (const auto ec = file::open_ex(opened_.at(index), + filenames_.at(index), random_)) + return ec; + + // logical_ is the shared row count, derived from column 0's byte size. + size_t bytes{}; + if (const auto ec = file::size_ex(bytes, opened_.front())) + return ec; + + logical_ = logical_rows(bytes); + return error::success; +} + +TEMPLATE +code CLASS::close() NOEXCEPT +{ + std::unique_lock map_lock(remap_mutex_); + std::unique_lock field_lock(field_mutex_); + + if (loaded_) + return error::close_loaded; + + for (const auto& descriptor: opened_) + if (descriptor == file::invalid) + return error::success; + + logical_ = zero; + for (auto& descriptor : opened_) + { + if (descriptor != file::invalid) + { + const auto current = descriptor; + descriptor = file::invalid; + if (const auto ec = file::close_ex(current)) + return ec; + } + } + + return error::success; +} + +// map, flush, unmap. +// ---------------------------------------------------------------------------- +// Each accessor holds a shared lock on remap_mutex_ (read/write access to map). +// Exclusive lock on remap_mutex_ ensures there are open accessor objects, +// which allows for safely remapping the memory map. + +TEMPLATE +code CLASS::load() NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (remap_mutex_.try_lock()) + { + if (loaded_) + { + remap_mutex_.unlock(); + return error::load_loaded; + } + + // Updates fields. + if (!map_all_(sequence{})) + { + remap_mutex_.unlock(); + return error::load_failure; + } + + remap_mutex_.unlock(); + return error::success; + } + + return error::load_locked; +} + +// Suspend writes before calling. +TEMPLATE +code CLASS::reload() NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (remap_mutex_.try_lock()) + { + if (!loaded_) + { + remap_mutex_.unlock(); + return error::reload_unloaded; + } + + // Allow resume from disk full. + set_disk_space(zero); + + remap_mutex_.unlock(); + return error::success; + } + + return error::reload_locked; +} + +// Suspend writes before calling. +TEMPLATE +code CLASS::flush() NOEXCEPT +{ + // Prevent unload, resize, remap. + std::shared_lock map_lock(remap_mutex_); + std::shared_lock field_lock(field_mutex_); + + if (!loaded_) + return error::flush_unloaded; + + // Reads fields and the memory map. + return flush_all_(sequence{}) ? error::success : error::flush_failure; +} + +// Suspend writes before calling. +TEMPLATE +code CLASS::unload() NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (remap_mutex_.try_lock()) + { + if (!loaded_) + { + remap_mutex_.unlock(); + return error::success; + } + + // Updates fields. + if (!unmap_all_(sequence{})) + { + remap_mutex_.unlock(); + return error::unload_failure; + } + + remap_mutex_.unlock(); + return error::success; + } + + return error::unload_locked; +} + +// Suspend writes before calling. +TEMPLATE +code CLASS::shrink() NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (remap_mutex_.try_lock()) + { + if (!loaded_) + { + remap_mutex_.unlock(); + return error::shrink_unloaded; + } + + // Updates fields. + if (!unmap_all_(sequence{})) + { + remap_mutex_.unlock(); + return error::shrink_unload_failure; + } + + // Updates fields. + if (!map_all_(sequence{})) + { + remap_mutex_.unlock(); + return error::shrink_load_failure; + } + + remap_mutex_.unlock(); + return error::success; + } + + return error::shrink_locked; +} + +// Used to copy headers in snapshot (scalar only). +TEMPLATE +code CLASS::dump(const std::filesystem::path& path) const NOEXCEPT +{ + BC_ASSERT(is_one(columns)); + const auto ptr = get(); + if (!ptr) + return error::unloaded_file; + + return file::create_file_ex(path, ptr->begin(), ptr->size()); +} + +// ---------------------------------------------------------------------------- + +TEMPLATE +size_t CLASS::size() const NOEXCEPT +{ + std::shared_lock field_lock(field_mutex_); + return logical_; +} + +TEMPLATE +size_t CLASS::capacity() const NOEXCEPT +{ + std::shared_lock field_lock(field_mutex_); + return capacity_rows(capacity_); +} + +TEMPLATE +bool CLASS::truncate(size_t size) NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (size > logical_) + return false; + + logical_ = size; + return true; +} + +TEMPLATE +bool CLASS::expand(size_t size) NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (fault_ || !loaded_) + return false; + + if (size <= logical_) + return true; + + // Column 0 capacity (bytes) transposed to rows for the row-space compare. + if (size > capacity_rows(capacity_)) + { + const auto capacity = to_capacity(size); + std::unique_lock remap_lock(remap_mutex_); + if (!remap_all_(capacity, sequence{})) + return false; + } + + logical_ = size; + return true; +} + +TEMPLATE +bool CLASS::reserve(size_t chunk) NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (fault_ || !loaded_ || system::is_add_overflow(logical_, chunk)) + return false; + + const auto end = logical_ + chunk; + if (end > capacity_rows(capacity_)) + { + const auto capacity = to_capacity(end); + std::unique_lock remap_lock(remap_mutex_); + if (!remap_all_(capacity, sequence{})) + return false; + } + + // Same as allocate except logical does not change. + return true; +} + +// Waits until all access pointers are destructed. Will deadlock if any access +// pointer is waiting on allocation. Lock safety requires that access pointers +// are short-lived and do not block on allocation. +TEMPLATE +size_t CLASS::allocate(size_t chunk) NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + + if (fault_ || !loaded_ || system::is_add_overflow(logical_, chunk)) + return storage::eof; + + const auto current = capacity_rows(capacity_); + auto end = logical_ + chunk; + if (end > current) + { + const auto capacity = to_capacity(end); + + // TODO: Could loop over a try lock here and log deadlock warning. + std::unique_lock remap_lock(remap_mutex_); + + // Disk full condition leaves store in valid state despite eof return. + if (!remap_all_(capacity, sequence{})) + return storage::eof; + } + + std::swap(logical_, end); + return end; +} + +// dispatch +// ---------------------------------------------------------------------------- + +TEMPLATE +memory_ptr CLASS::set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT +{ + return set_column(offset, size, backfill); +} + +TEMPLATE +memory_ptr CLASS::get(size_t offset) const NOEXCEPT +{ + return get_column(offset); +} + +TEMPLATE +memory_ptr CLASS::get_capacity(size_t offset) const NOEXCEPT +{ + return capacity_column(offset); +} + +TEMPLATE +memory::iterator CLASS::get_raw(size_t offset) const NOEXCEPT +{ + return raw_column(offset); +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/memory/interfaces/storage.hpp b/include/bitcoin/database/memory/interfaces/storage.hpp index acd26d67c..a3044e00b 100644 --- a/include/bitcoin/database/memory/interfaces/storage.hpp +++ b/include/bitcoin/database/memory/interfaces/storage.hpp @@ -32,6 +32,15 @@ class storage public: static constexpr auto eof = system::bit_all; + /// Get the fault condition. + virtual code get_fault() const NOEXCEPT = 0; + + /// Get the space required to clear the disk full condition. + virtual size_t get_space() const NOEXCEPT = 0; + + /// The filesystem path of the backing storage. + virtual const std::filesystem::path& file() const NOEXCEPT = 0; + /// Create empty file, must not exist. virtual code create() const NOEXCEPT = 0; @@ -59,9 +68,6 @@ class storage /// Dump current logical map to a new file in path, must not exist. virtual code dump(const std::filesystem::path& path) const NOEXCEPT = 0; - /// The filesystem path of the backing storage. - virtual const std::filesystem::path& file() const NOEXCEPT = 0; - /// The current logical size of the memory map (zero if closed). virtual size_t size() const NOEXCEPT = 0; @@ -95,12 +101,6 @@ class storage /// Get unprotected r/w access to start/offset of memory map (or null). /// Pointer is constrained to starting write within full capacity. virtual memory::iterator get_raw(size_t offset=zero) const NOEXCEPT = 0; - - /// Get the fault condition. - virtual code get_fault() const NOEXCEPT = 0; - - /// Get the space required to clear the disk full condition. - virtual size_t get_space() const NOEXCEPT = 0; }; } // namespace database diff --git a/include/bitcoin/database/memory/map.hpp b/include/bitcoin/database/memory/map.hpp index 9d40c8f7f..830977750 100644 --- a/include/bitcoin/database/memory/map.hpp +++ b/include/bitcoin/database/memory/map.hpp @@ -19,11 +19,11 @@ #ifndef LIBBITCOIN_DATABASE_MEMORY_MAP_HPP #define LIBBITCOIN_DATABASE_MEMORY_MAP_HPP -#include #include #include #include #include +#include #include #include #include @@ -33,144 +33,234 @@ namespace libbitcoin { namespace database { -/// Thread safe access to a memory-mapped file. -class BCD_API map +/// Thread safe access to a memory-mapped file, or to a set of column files +/// sharing one allocation/remap guard set (SoA aggregate). +template +class map1 : public storage { public: - DELETE_COPY_MOVE(map); + DELETE_COPY_MOVE(map1); - map(const std::filesystem::path& filename, size_t minimum=1, - size_t expansion=0, bool random=true) NOEXCEPT; + /// Number of backing columns (1 == scalar map). + static constexpr size_t columns = sizeof...(Widths); + static_assert(!is_zero(columns), "requires at least one column"); + + using path = std::filesystem::path; + using paths = std::array; + using sizes = std::array; + + /// Per-column record widths; transpose row<->byte by these (constexpr). + static constexpr sizes widths{ Widths... }; + + /// Bytes per logical row across the aggregate (sum of column widths). + static constexpr size_t stride = (Widths + ...); + + /// Dispatch. + /// ----------------------------------------------------------------------- + + template = true> + memory_ptr set_column(size_t offset, size_t size, + uint8_t backfill) NOEXCEPT; + + template = true> + memory_ptr get_column(size_t offset=zero) const NOEXCEPT; + + template = true> + memory_ptr capacity_column(size_t offset=zero) const NOEXCEPT; + + template = true> + memory::iterator raw_column(size_t offset=zero) const NOEXCEPT; + + /// Constructors. + /// ----------------------------------------------------------------------- + + /// Scalar construction (columns == 1): unchanged signature and codegen. + map1(const std::filesystem::path& filename, size_t minimum=1, + size_t expansion=0, bool random=true) NOEXCEPT + requires (columns == one); + + /// Aggregate construction (columns > 1): one file per column, shared guards. + map1(const paths& filenames, size_t minimum=1, size_t expansion=0, + bool random=true) NOEXCEPT + requires (columns > one); /// Destruct for debug assertion only. - virtual ~map() NOEXCEPT; + virtual ~map1() NOEXCEPT; - /// True if the file is open. + /// True if the file(s) are open. bool is_open() const NOEXCEPT; - /// True if the memory map is loaded. + /// True if the memory map(s) are loaded. bool is_loaded() const NOEXCEPT; /// storage interface /// ----------------------------------------------------------------------- - /// Create empty file, must be closed. + /// Get the fault condition. + code get_fault() const NOEXCEPT override; + + /// Get the space required to clear the disk full condition. + size_t get_space() const NOEXCEPT override; + + /// The filesystem path of the (first) backing file. + const std::filesystem::path& file() const NOEXCEPT override; + + /// Create empty file(s), must be closed. code create() const NOEXCEPT override; - /// Open file, must be closed. + /// Open file(s), must be closed. code open() NOEXCEPT override; - /// Close file, must be unloaded, idempotent. + /// Close file(s), must be unloaded, idempotent. code close() NOEXCEPT override; - /// Map file to memory, must be loaded. + /// Map file(s) to memory, must be loaded. code load() NOEXCEPT override; /// Clear disk full condition, fails if fault, must be loaded, idempotent. code reload() NOEXCEPT override; - /// Flush memory map to disk, suspend writes for call, must be loaded. + /// Flush memory map(s) to disk, suspend writes for call, must be loaded. code flush() NOEXCEPT override; /// Flush, unmap and truncate to logical, restartable, idempotent. code unload() NOEXCEPT override; - /// Unload and load, causing underyling map to shrink to logical size. + /// Unload and load, causing underyling map(s) to shrink to logical size. code shrink() NOEXCEPT override; /// Dump current logical map to a new file in path, must not exist. code dump(const std::filesystem::path& path) const NOEXCEPT override; - /// The filesystem path of the file. - const std::filesystem::path& file() const NOEXCEPT override; - /// The current logical size of the memory map (zero if closed). size_t size() const NOEXCEPT override; /// The current capacity of the memory map (zero if unloaded). size_t capacity() const NOEXCEPT override; - /// Reduce logical size to specified bytes (false if exceeds logical). + /// Reduce logical size to specified (false if exceeds logical). bool truncate(size_t size) NOEXCEPT override; - /// Increase logical size to specified bytes as required (false if fails). + /// Increase logical size to specified as required (false if fails). bool expand(size_t size) NOEXCEPT override; - /// Increase capacity by specified bytes (false only if fails). - bool reserve(size_t chunk) NOEXCEPT override; + /// Increase capacity by specified (false only if fails). + bool reserve(size_t size) NOEXCEPT override; - /// Increase logical by specified bytes, return offset to first (or eof). + /// Increase logical by specified, return offset to first (or eof). size_t allocate(size_t chunk) NOEXCEPT override; - /// Get remap-protected r/w access to offset (or null) allocated to size. + /// Remap-protected r/w access to offset (or null) allocated to size. memory_ptr set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT override; - /// Get remap-protected r/w access to start/offset of memory map (or null). - /// Pointer is constrained to starting write within logical allocation. + /// Remap-protected r/w access to start/offset (or null), within logical. memory_ptr get(size_t offset=zero) const NOEXCEPT override; - /// Get remap-protected r/w access to start/offset of memory map (or null). - /// Pointer is constrained to starting write within full capacity. + /// Remap-protected r/w access to start/offset (or null), within capacity. memory_ptr get_capacity(size_t offset=zero) const NOEXCEPT override; - /// Get unprotected r/w access to start/offset of memory map (or null). - /// Pointer is constrained to starting write within logical allocation. + /// Unprotected r/w access to start/offset (or null), within logical. memory::iterator get_raw(size_t offset=zero) const NOEXCEPT override; - /// Get the fault condition. - code get_fault() const NOEXCEPT override; - - /// Get the space required to clear the disk full condition. - /// Use load() to clear the indicated space and allow restart. - size_t get_space() const NOEXCEPT override; - protected: + /// Row<->byte transpose by the constexpr column width (folds for width 1). + template + static constexpr size_t to_bytes(size_t offset) NOEXCEPT + { + return offset * std::get(widths); + } + + // Capacity in rows: column 0 bytes transposed back to rows. + static constexpr size_t capacity_rows(const sizes& capacity) NOEXCEPT + { + return std::get(capacity) / std::get(widths); + } + + // Logical rows: column 0 bytes transposed back to rows. + static constexpr size_t logical_rows(size_t bytes) NOEXCEPT + { + return bytes / std::get(widths); + } + size_t to_capacity(size_t required) const NOEXCEPT; void set_first_code(const error::error_t& ec) NOEXCEPT; void set_disk_space(size_t required) NOEXCEPT; private: - using path = std::filesystem::path; + static constexpr auto fail = -1; using access = accessor; - - // Mapping utilities. + using sequence = std::make_index_sequence; + + // mman dispatch, not thread safe. + template + bool map_all_(std::index_sequence) NOEXCEPT; + template + bool unmap_all_(std::index_sequence) NOEXCEPT; + template + bool flush_all_(std::index_sequence) NOEXCEPT; + template + bool remap_all_(size_t logical, std::index_sequence) NOEXCEPT; + + // mman wrappers, not thread safe. + template bool flush_() NOEXCEPT; + template bool unmap_() NOEXCEPT; + template bool map_() NOEXCEPT; + template bool remap_(size_t size) NOEXCEPT; + template bool resize_(size_t size) NOEXCEPT; + template bool finalize_(size_t size) NOEXCEPT; - // Constants. - const std::filesystem::path filename_; + // These are thread safe. + const paths filenames_; const size_t minimum_; const size_t expansion_; const bool random_; + std::atomic space_{ zero }; + std::atomic error_{ error::success }; - // Protected by remap_mutex. - // requires remap_mutex_ exclusive lock for write. - // requires remap_mutex_ minimum shared lock for flush/read. - uint8_t* memory_map_{}; - mutable std::shared_mutex remap_mutex_{}; - - // Protected by field_mutex. - // fields require field_mutex_ exclusive lock for write. - // fields require minimum field_mutex_ shared lock for flush/read. - int opened_{ file::invalid }; + // These are protected by field_mutex_. + // Shared allocation state and per-column file/capacity state. + // Fields require field_mutex_ exclusive for write, shared for flush/read. + // logical_ is the shared row count across all columns. + std::array opened_; + sizes capacity_{}; + size_t logical_{}; bool fault_{}; bool loaded_{}; - size_t capacity_{}; - size_t logical_{}; mutable std::shared_mutex field_mutex_{}; - // These are thread safe. - std::atomic space_{ zero }; - std::atomic error_{ error::success }; + // These are protected by remap_mutex_. + std::array memory_map_{}; + mutable std::shared_mutex remap_mutex_{}; }; +/// Scalar map: single column, width 1. Source and codegen identical to the +/// prior non-aggregate map; existing usage binds the unchanged constructor. +using map = map1<1>; + } // namespace database } // namespace libbitcoin +#define TEMPLATE template +#define CLASS map1 + +BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT) + +#include +#include +#include +#include + +BC_POP_WARNING() + +#undef CLASS +#undef TEMPLATE + #endif diff --git a/include/bitcoin/database/memory/memory.hpp b/include/bitcoin/database/memory/memory.hpp index 08e5663bf..43d4f617b 100644 --- a/include/bitcoin/database/memory/memory.hpp +++ b/include/bitcoin/database/memory/memory.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #endif diff --git a/src/memory/mman-win32/mman.hpp b/include/bitcoin/database/memory/mman.hpp similarity index 72% rename from src/memory/mman-win32/mman.hpp rename to include/bitcoin/database/memory/mman.hpp index bd00401b3..cde0497df 100644 --- a/src/memory/mman-win32/mman.hpp +++ b/include/bitcoin/database/memory/mman.hpp @@ -1,11 +1,19 @@ -// mman-win32 based on code.google.com/p/mman-win32 (MIT License). +// mman_win32 based on code.google.com/p/mman-win32 (MIT License). #ifndef LIBBITCOIN_DATABASE_MMAN_HPP #define LIBBITCOIN_DATABASE_MMAN_HPP -#ifdef _WIN32 +#include + +#if !defined(HAVE_MSC) + #include + #include + #include + #include +#endif + +#if defined(HAVE_MSC) -#include typedef size_t oft__; #define PROT_NONE 0 @@ -30,7 +38,8 @@ typedef size_t oft__; #define MS_SYNC 2 #define MS_INVALIDATE 4 -void* mmap(void* addr, size_t len, int prot, int flags, int fd, oft__ off) noexcept; +void* mmap(void* addr, size_t len, int prot, int flags, int fd, + oft__ off) noexcept; void* mremap_(void* addr, size_t old_size, size_t new_size, int prot, int flags, int fd) noexcept; int munmap(void* addr, size_t len) noexcept; @@ -43,5 +52,10 @@ int fsync(int fd) noexcept; int fallocate(int fd, int mode, oft__ offset, oft__ size) noexcept; int ftruncate(int fd, oft__ size) noexcept; -#endif // _WIN32 +#elif defined(HAVE_APPLE) + +int fallocate(int fd, int, off_t offset, off_t len) NOEXCEPT; + +#endif // HAVE_MSC + #endif diff --git a/src/memory/map.cpp b/src/memory/map.cpp deleted file mode 100644 index 2ccf5fc2c..000000000 --- a/src/memory/map.cpp +++ /dev/null @@ -1,742 +0,0 @@ -/** - * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) - * - * This file is part of libbitcoin. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include - -#if defined(HAVE_MSC) - #include "mman-win32/mman.hpp" -#else - #include - #include - #include - #include -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(HAVE_APPLE) -int fallocate(int fd, int, off_t offset, off_t len) NOEXCEPT; -#endif - -namespace libbitcoin { -namespace database { - -BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT) - -using namespace system; - -map::map(const path& filename, size_t minimum, size_t expansion, - bool random) NOEXCEPT - : filename_(filename), - minimum_(minimum), - expansion_(expansion), - random_(random) -{ -} - -map::~map() NOEXCEPT -{ - BC_ASSERT_MSG(!loaded_, "file mapped at destruct"); - BC_ASSERT_MSG(is_null(memory_map_), "map defined at destruct"); - BC_ASSERT_MSG(is_zero(logical_), "logical nonzero at destruct"); - BC_ASSERT_MSG(is_zero(capacity_), "capacity nonzero at destruct"); - BC_ASSERT_MSG(opened_ == file::invalid, "file open at destruct"); -} - -const std::filesystem::path& map::file() const NOEXCEPT -{ - return filename_; -} - -code map::create() const NOEXCEPT -{ - if (opened_ != file::invalid) - return error::create_open; - - return file::create_file_ex(file()); -} - -code map::open() NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (opened_ != file::invalid) - return error::open_open; - - // Windows doesn't use madvise, instead infers map access from file open. - if (const auto ec = file::open_ex(opened_, filename_, random_)) - return ec; - - return file::size_ex(logical_, opened_); -} - -code map::close() NOEXCEPT -{ - std::unique_lock map_lock(remap_mutex_); - std::unique_lock field_lock(field_mutex_); - - if (loaded_) - return error::close_loaded; - - if (opened_ == file::invalid) - return error::success; - - const auto descriptor = opened_; - opened_ = file::invalid; - logical_ = zero; - - return file::close_ex(descriptor); -} - -bool map::is_open() const NOEXCEPT -{ - std::shared_lock field_lock(field_mutex_); - return opened_ != file::invalid; -} - -// map, flush, unmap. -// ---------------------------------------------------------------------------- -// Each accessor holds a shared lock on remap_mutex_ (read/write access to map). -// Exclusive lock on remap_mutex_ ensures there are open accessor objects, -// which allows for safely remapping the memory map. - -code map::load() NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (remap_mutex_.try_lock()) - { - if (loaded_) - { - remap_mutex_.unlock(); - return error::load_loaded; - } - - // Updates fields. - if (!map_()) - { - remap_mutex_.unlock(); - return error::load_failure; - } - - remap_mutex_.unlock(); - return error::success; - } - - return error::load_locked; -} - -// Suspend writes before calling. -code map::reload() NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (remap_mutex_.try_lock()) - { - if (!loaded_) - { - remap_mutex_.unlock(); - return error::reload_unloaded; - } - - // Allow resume from disk full. - set_disk_space(zero); - - remap_mutex_.unlock(); - return error::success; - } - - return error::reload_locked; -} - -// Suspend writes before calling. -code map::flush() NOEXCEPT -{ - // Prevent unload, resize, remap. - std::shared_lock map_lock(remap_mutex_); - std::shared_lock field_lock(field_mutex_); - - if (!loaded_) - return error::flush_unloaded; - - // Reads fields and the memory map. - return flush_() ? error::success : error::flush_failure; -} - -// Suspend writes before calling. -code map::unload() NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (remap_mutex_.try_lock()) - { - if (!loaded_) - { - remap_mutex_.unlock(); - return error::success; - } - - BC_ASSERT_MSG(logical_ <= capacity_, "logical size exceeds capacity"); - - // Updates fields. - if (!unmap_()) - { - remap_mutex_.unlock(); - return error::unload_failure; - } - - remap_mutex_.unlock(); - return error::success; - } - - return error::unload_locked; -} - -// Suspend writes before calling. -code map::shrink() NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (remap_mutex_.try_lock()) - { - if (!loaded_) - { - remap_mutex_.unlock(); - return error::shrink_unloaded; - } - - BC_ASSERT_MSG(logical_ <= capacity_, "logical size exceeds capacity"); - - // Updates fields. - if (!unmap_()) - { - remap_mutex_.unlock(); - return error::shrink_unload_failure; - } - - // Updates fields. - if (!map_()) - { - remap_mutex_.unlock(); - return error::shrink_load_failure; - } - - remap_mutex_.unlock(); - return error::success; - } - - return error::shrink_locked; -} - -bool map::is_loaded() const NOEXCEPT -{ - std::shared_lock field_lock(field_mutex_); - return loaded_; -} - -// Used to copy headers in snapshot. -code map::dump(const std::filesystem::path& path) const NOEXCEPT -{ - const auto ptr = get(); - if (!ptr) - return error::unloaded_file; - - return file::create_file_ex(path, ptr->begin(), ptr->size()); -} - -// Interface. -// ---------------------------------------------------------------------------- - -size_t map::size() const NOEXCEPT -{ - std::shared_lock field_lock(field_mutex_); - return logical_; -} - -size_t map::capacity() const NOEXCEPT -{ - std::shared_lock field_lock(field_mutex_); - return capacity_; -} - -bool map::truncate(size_t size) NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (size > logical_) - return false; - - logical_ = size; - return true; -} - -bool map::expand(size_t size) NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (fault_ || !loaded_) - return false; - - if (size <= logical_) - return true; - - if (size > capacity_) - { - const auto capacity = to_capacity(size); - std::unique_lock remap_lock(remap_mutex_); - if (!remap_(capacity)) - return false; - } - - logical_ = size; - return true; -} - -bool map::reserve(size_t chunk) NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (fault_ || !loaded_ || is_add_overflow(logical_, chunk)) - return false; - - const auto end = logical_ + chunk; - if (end > capacity_) - { - const auto capacity = to_capacity(end); - std::unique_lock remap_lock(remap_mutex_); - if (!remap_(capacity)) - return false; - } - - // Same as allocate except logical does not change. - return true; -} - -// Waits until all access pointers are destructed. Will deadlock if any access -// pointer is waiting on allocation. Lock safety requires that access pointers -// are short-lived and do not block on allocation. -size_t map::allocate(size_t chunk) NOEXCEPT -{ - std::unique_lock field_lock(field_mutex_); - - if (fault_ || !loaded_ || is_add_overflow(logical_, chunk)) - return storage::eof; - - auto end = logical_ + chunk; - if (end > capacity_) - { - const auto capacity = to_capacity(end); - - // TODO: Could loop over a try lock here and log deadlock warning. - std::unique_lock remap_lock(remap_mutex_); - - // Disk full condition leaves store in valid state despite eof return. - if (!remap_(capacity)) - return storage::eof; - } - - std::swap(logical_, end); - return end; -} - -memory_ptr map::set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT -{ - { - std::unique_lock field_lock(field_mutex_); - - if (fault_ || !loaded_ || is_add_overflow(offset, size)) - return {}; - - const auto end = std::max(logical_, offset + size); - if (end > capacity_) - { - const auto capacity = to_capacity(end); - - // TODO: Could loop over a try lock here and log deadlock warning. - std::unique_lock remap_lock(remap_mutex_); - - // Disk full condition leaves store in valid state despite null. - if (!remap_(capacity)) - return {}; - - // Fill new capacity as offset may not be at end due to expansion. - BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) - std::fill_n(memory_map_ + logical_, capacity - logical_, backfill); - BC_POP_WARNING() - } - - logical_ = end; - } - - return get(offset); -} - -memory_ptr map::get(size_t offset) const NOEXCEPT -{ - // Obtaining size before access prevents mutual mutex wait (deadlock). - // The store could remap between here and next line, but capacity only - // increases. Close zeroizes capacity but file must be unloaded to do so. - // Truncate can reduce logical, but capacity is not affected. It is always - // safe to write past current logical within current capacity. - const auto allocated = size(); - - // Takes a shared lock on remap_mutex_ until destruct, blocking remap. - const auto ptr = std::make_shared(remap_mutex_); - - // loaded_ update is precluded by remap_mutex_, making this read atomic. - if (!loaded_ || is_null(ptr)) - return nullptr; - - // With offset > size the assignment is negative (stream is exhausted). - BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) - ptr->assign(memory_map_ + offset, memory_map_ + allocated); - BC_POP_WARNING() - return ptr; -} - -memory_ptr map::get_capacity(size_t offset) const NOEXCEPT -{ - // Same as get() but limited by capacity() vs. size(). - const auto allocated = capacity(); - const auto ptr = std::make_shared(remap_mutex_); - if (!loaded_ || is_null(ptr)) - return nullptr; - - BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) - ptr->assign(memory_map_ + offset, memory_map_ + allocated); - BC_POP_WARNING() - return ptr; -} - -memory::iterator map::get_raw(size_t offset) const NOEXCEPT -{ - // Pointer is otherwise unguarded, not remap safe (use for table heads). - if (offset > size()) - return nullptr; - - BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) - return memory_map_ + offset; - BC_POP_WARNING() -} - -code map::get_fault() const NOEXCEPT -{ - return error_.load(); -} - -size_t map::get_space() const NOEXCEPT -{ - return space_.load(); -} - -// protected -// ---------------------------------------------------------------------------- - -size_t map::to_capacity(size_t required) const NOEXCEPT -{ - BC_PUSH_WARNING(NO_STATIC_CAST) - const auto resize = required * ((expansion_ + 100.0) / 100.0); - const auto target = std::max(minimum_, static_cast(resize)); - BC_POP_WARNING() - - BC_ASSERT(target >= required); - return target; -} - -// Read-write protected by atomic, write-write protected by remap_mutex. -void map::set_first_code(const error::error_t& ec) NOEXCEPT -{ - if (!fault_) - { - // fault is not exposed so requires no atomic (fast read). - fault_ = true; - - // error is atomic for public read exposure. - error_.store(ec); - } -} - -void map::set_disk_space(size_t required) NOEXCEPT -{ - space_.store(required); -} - -// private, mman wrappers, not thread safe -// ---------------------------------------------------------------------------- - -// TODO: map_, flush_, unmap_, remap_ (resize_, finalize_) return codes. - -constexpr auto fail = -1; - -// Never results in unmapped. -bool map::flush_() NOEXCEPT -{ -#if defined(HAVE_MSC) - // unmap (and therefore msync) must be called before ftruncate. - // "To flush all the dirty pages plus the metadata for the file and ensure - // that they are physically written to disk..." - const auto success = (::msync(memory_map_, logical_, MS_SYNC) != fail) - && (::fsync(opened_) != fail); -#elif defined(F_FULLFSYNC) - // macOS msync fails with zero logical size (but we are no longer calling). - // non-standard macOS behavior: news.ycombinator.com/item?id=30372218 - const auto success = ::fcntl(opened_, F_FULLFSYNC, 0) != fail; -#else - // msync should not be required on modern linux, see linus et al. - // stackoverflow.com/questions/5902629/mmap-msync-and-linux-process-termination - // Linux: fsync "transfers ("flushes") all modified in-core data of - // (i.e., modified buffer cache pages for) the file referred to by the - // file descriptor fd to the disk device so all changed information - // can be retrieved even if the system crashes or is rebooted. This - // includes writing through or flushing a disk cache if present. The - // call blocks until the device reports that transfer has completed." - const auto success = ::fsync(opened_) != fail; -#endif - - if (!success) - set_first_code(error::fsync_failure); - - return success; -} - -// Always results in unmapped. -// Trims to logical size, can be zero. -bool map::unmap_() NOEXCEPT -{ -#if defined(HAVE_MSC) - const auto success = - (::msync(memory_map_, logical_, MS_SYNC) != fail) - && (::munmap(memory_map_, capacity_) != fail) - && (::ftruncate(opened_, logical_) != fail) - && (::fsync(opened_) != fail); -#else - const auto success = (::ftruncate(opened_, logical_) != fail) - #if defined(F_FULLFSYNC) - && (::fcntl(opened_, F_FULLFSYNC, 0) != fail) - #else - && (::fsync(opened_) != fail) - #endif - && (::munmap(memory_map_, capacity_) != fail); -#endif - if (!success) - set_first_code(error::munmap_failure); - - loaded_ = false; - capacity_ = zero; - memory_map_ = {}; - return success; -} - -// Mapping failure results in unmapped. -// Mapping has no effect on logical size, always maps max(logical, min) size. -bool map::map_() NOEXCEPT -{ - auto size = logical_; - - // Cannot map empty file, and want mininum capacity, so expand as required. - // disk_full: space is set but no code is set with false return. - if ((size < minimum_) && !resize_((size = minimum_))) - return false; - - memory_map_ = pointer_cast(::mmap(nullptr, size, - PROT_READ | PROT_WRITE, MAP_SHARED, opened_, 0)); - - return finalize_(size); -} - -// Remap failure results in unmapped. -// Remapping has no effect on logical size, sets map_/capacity_. -bool map::remap_(size_t size) NOEXCEPT -{ - BC_ASSERT(size >= logical_); - - // Cannot remap empty file, so expand to minimum capacity if zero. - if (is_zero(size)) - size = minimum_; - -#if !defined(HAVE_MSC) && !defined(MREMAP_MAYMOVE) - // macOS: unmap before ftruncate sets new size. - if (!unmap_()) - return false; - - // disk_full: unmap(ok), resize(fail for space), map(ok), return false. - // disk_full: if second unmap fails then code is set, and false return. - if (!resize_(size)) - { - /* bool */ map::map_(); - return false; - } -#else - // disk_full: space is set but no code is set with false return. - if (!resize_(size)) - return false; -#endif - -#if defined(HAVE_MSC) - // mman-win32 mremap hack (umap/map) requires flags and file descriptor. - memory_map_ = pointer_cast(::mremap_(memory_map_, capacity_, size, - PROT_READ | PROT_WRITE, MAP_SHARED, opened_)); -#elif defined(MREMAP_MAYMOVE) - memory_map_ = pointer_cast(::mremap(memory_map_, capacity_, size, - MREMAP_MAYMOVE)); -#else - // macOS: does not define mremap or MREMAP_MAYMOVE. - // TODO: see "MREMAP_MAYMOVE" in sqlite for map extension technique. - memory_map_ = pointer_cast(::mmap(nullptr, size, - PROT_READ | PROT_WRITE, MAP_SHARED, opened_, 0)); -#endif - - return finalize_(size); -} - -// disk_full: space is set but no code is set with false return. -bool map::resize_(size_t size) NOEXCEPT -{ - // Disk full detection, any other failure is an abort. -#if !defined (WITHOUT_FALLOCATE) - if (::fallocate(opened_, 0, capacity_, size - capacity_) == fail) -#else - if (::ftruncate(opened_, size) == fail) -#endif - { - // Disk full is the only restartable store failure (leave mapped). - if (errno == ENOSPC) - { - set_disk_space(size - capacity_); - return false; - } - - set_first_code(error::ftruncate_failure); - unmap_(); - return false; - } - - return true; -} - -// Finalize failure results in unmapped. -bool map::finalize_(size_t size) NOEXCEPT -{ - if (memory_map_ == MAP_FAILED) - { - loaded_ = false; - capacity_ = zero; - memory_map_ = {}; - - // mmap or mremap failure (not mapped). - set_first_code(error::mmap_failure); - return false; - } - -#if !defined (WITHOUT_MADVISE) -#if !defined(HAVE_MSC) - // Get page size (usually 4KB). - const int page_size = ::sysconf(_SC_PAGESIZE); - const auto page = possible_narrow_sign_cast(page_size); - - // If not one bit then page size is not a power of two as required. - if (page_size == fail || !is_one(ones_count(page))) - { - set_first_code(error::sysconf_failure); - unmap_(); - return false; - } - - // Align size up to page boundary. - const auto max = sub1(page); - const auto align = bit_and(ceilinged_add(size, max), bit_not(max)); - - // Use 1GB chunks to avoid large-length issues. - constexpr auto chunk = power2(30u); - const auto advice = (random_ ? MADV_RANDOM : MADV_SEQUENTIAL) | - MADV_WILLNEED; - - for (auto offset = zero; offset < align; offset += chunk) - { - BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) - const auto start = memory_map_ + offset; - BC_POP_WARNING() - - if (::madvise(start, std::min(chunk, align - offset), advice) == fail) - { - set_first_code(error::madvise_failure); - unmap_(); - return false; - } - } -#endif -#endif // WITHOUT_MADVISE - - loaded_ = true; - capacity_ = size; - return true; -} - -BC_POP_WARNING() - -} // namespace database -} // namespace libbitcoin - -#if defined(HAVE_APPLE) -// ::fallocate is not defined on macOS, so implement. Ignores mode (linux). -int fallocate(int fd, int, off_t offset, off_t len) NOEXCEPT -{ - constexpr auto fail = -1; - - fstore_t store - { - // Prefer contiguous allocation - .fst_flags = F_ALLOCATECONTIG, - - // Allocate from EOF - .fst_posmode = F_PEOFPOSMODE, - - // Start from current capacity - .fst_offset = offset, - - // Delta size - .fst_length = len, - - // Output: actual bytes allocated - .fst_bytesalloc = 0 - }; - - // Try contiguous allocation. - auto result = ::fcntl(fd, F_PREALLOCATE, &store); - - // Fallback to non-contiguous. - if ((result == fail) && (errno != ENOSPC)) - { - store.fst_flags = F_ALLOCATEALL; - result = ::fcntl(fd, F_PREALLOCATE, &store); - } - - if (result == fail) - return fail; - - // Extend file to new size (required for mmap). This is not required on - // Linux because fallocate(2) automatically extends file's logical size. - return ::ftruncate(fd, offset + len); -} -#endif // HAVE_APPLE diff --git a/src/memory/mman-win32/mman.cpp b/src/memory/mman.cpp similarity index 84% rename from src/memory/mman-win32/mman.cpp rename to src/memory/mman.cpp index bc9e7d852..7300d8f89 100644 --- a/src/memory/mman-win32/mman.cpp +++ b/src/memory/mman.cpp @@ -1,12 +1,11 @@ -// mman-win32 based on code.google.com/p/mman-win32 (MIT License). +// mman_win32 based on code.google.com/p/mman-win32 (MIT License). -#include "mman.hpp" +#include +#include -#ifdef _WIN32 +#if defined(HAVE_MSC) -#include #include -#include #include // local utilities @@ -283,4 +282,47 @@ int sysconf(int) noexcept return {}; } -#endif // _WIN32 +#elif defined(HAVE_APPLE) + +// ::fallocate is not defined on macOS, so implement. Ignores mode (linux). +int fallocate(int fd, int, off_t offset, off_t len) NOEXCEPT +{ + constexpr auto fail = -1; + + fstore_t store + { + // Prefer contiguous allocation + .fst_flags = F_ALLOCATECONTIG, + + // Allocate from EOF + .fst_posmode = F_PEOFPOSMODE, + + // Start from current capacity + .fst_offset = offset, + + // Delta size + .fst_length = len, + + // Output: actual bytes allocated + .fst_bytesalloc = 0 + }; + + // Try contiguous allocation. + auto result = ::fcntl(fd, F_PREALLOCATE, &store); + + // Fallback to non-contiguous. + if ((result == fail) && (errno != ENOSPC)) + { + store.fst_flags = F_ALLOCATEALL; + result = ::fcntl(fd, F_PREALLOCATE, &store); + } + + if (result == fail) + return fail; + + // Extend file to new size (required for mmap). This is not required on + // Linux because fallocate(2) automatically extends file's logical size. + return ::ftruncate(fd, offset + len); +} + +#endif // HAVE_MSC