diff options
Diffstat (limited to 'libs/libmdbx/src')
97 files changed, 41354 insertions, 24689 deletions
diff --git a/libs/libmdbx/src/.circleci/config.yml b/libs/libmdbx/src/.circleci/config.yml deleted file mode 100644 index 6d46ecefd3..0000000000 --- a/libs/libmdbx/src/.circleci/config.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/buildpack-deps:20.04 - environment: - - TESTDB: /tmp/test.db - - TESTLOG: /tmp/test.log - steps: - - checkout - - run: make all - - run: ulimit -c unlimited && make check - - run: - command: | - mkdir -p /tmp/artifacts - mv -t /tmp/artifacts $TESTLOG $TESTDB core.* - when: on_fail - - store_artifacts: - path: /tmp/artifacts - destination: test-artifacts diff --git a/libs/libmdbx/src/.cirrus.yml b/libs/libmdbx/src/.cirrus.yml deleted file mode 100644 index c08e316c37..0000000000 --- a/libs/libmdbx/src/.cirrus.yml +++ /dev/null @@ -1,6 +0,0 @@ -freebsd_instance: - image_family: freebsd-12-1-snap - -task: - install_script: pkg install -y gmake bash git - script: git fetch --tags && gmake check diff --git a/libs/libmdbx/src/.clang-format b/libs/libmdbx/src/.clang-format deleted file mode 100644 index 6c59ef3a39..0000000000 --- a/libs/libmdbx/src/.clang-format +++ /dev/null @@ -1,3 +0,0 @@ -BasedOnStyle: LLVM -Standard: Cpp11 -ReflowComments: true diff --git a/libs/libmdbx/src/.github/workflows/release-assets.yml b/libs/libmdbx/src/.github/workflows/release-assets.yml deleted file mode 100644 index 934ab93b39..0000000000 --- a/libs/libmdbx/src/.github/workflows/release-assets.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Based on the https://github.com/actions/upload-release-asset example - -on: - push: - # Sequence of patterns matched against refs/tags - tags: - - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 - -name: Upload Release Asset - -jobs: - build: - name: Upload Release Asset - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Build assets - run: | - make release-assets - - id: name - run: | - echo "::set-output name=tarball::$(ls *.tar.gz)" - echo "::set-output name=zip::$(ls *.zip)" - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: true - prerelease: true - - name: Upload tarball - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.name.outputs.tarball }} - asset_name: amalgamated.tar.gz - # asset_label: Amalgamated source tarball - asset_content_type: application/tar+gzip - - name: Upload zip - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.name.outputs.zip }} - asset_name: amalgamated.zip - # asset_label: Amalgamated source zip-archive - asset_content_type: application/zip diff --git a/libs/libmdbx/src/.gitignore b/libs/libmdbx/src/.gitignore deleted file mode 100644 index 906233ac58..0000000000 --- a/libs/libmdbx/src/.gitignore +++ /dev/null @@ -1,33 +0,0 @@ -*.[ao] -*.bak -*.exe -*.gcda -*.gcno -*.gcov -*.lo -*.orig -*.rej -*.so -*[~#] -.idea -.le.ini -.vs/ -cmake-build-* -@* -core -mdbx_example -libmdbx.creator.user -mdbx_chk -mdbx_copy -mdbx_dump -mdbx_load -mdbx_stat -mdbx_test -test.log -test/tmp.db -test/tmp.db-lck -tmp.db -tmp.db-lck -valgrind.* -dist/ -*.tar* diff --git a/libs/libmdbx/src/.travis.yml b/libs/libmdbx/src/.travis.yml deleted file mode 100644 index e736d01341..0000000000 --- a/libs/libmdbx/src/.travis.yml +++ /dev/null @@ -1,88 +0,0 @@ -language: c cpp -sudo: false - -env: - global: - - secure: "M+W+heGGyRQJoBq2W0uqWVrpL4KBXmL0MFL7FSs7f9vmAaDyEgziUXeZRj3GOKzW4kTef3LpIeiu9SmvqSMoQivGGiomZShqPVl045o/OUgRCAT7Al1RLzEZ0efSHpIPf0PZ6byEf6GR2ML76OfuL6JxTVdnz8iVyO2sgLE1HbX1VeB+wgd/jfMeOBhCCXskfK6MLyZihfMYsiYZYSaV98ZDhDLSlzuuRIgzb0bMi8aL6AErs0WLW0NelRBeHkKPYfAUc85pdQHscgrJw6Rh/zT6+8BQ/q5f4IgWhiu4xoRg3Ngl7SNoedRQh93ADM3UG2iGl6HDFpVORaXcFWKAtuYY+kHQ0HB84BRYpQmeBuXNpltsfxQ3d1Q3u0RlE45zRvmr2+X1mFnkcNUAWISLPbsOUlriDQM8irGwRpho77/uYnRC00bJsHW//s6+uPf9zrAw1nI4f0y3PAWukGF/xs6HAI3FZPsuSSnx18Tj3Opgbc9Spop+V3hkhdiJoPGpNKTkFX4ZRXfkPgoRVJmtp4PpbpH0Ps/mCriKjMEfGGi0HcVCi0pEGLXiecdqJ5KPg5+22zNycEujQBJcNTKd9shN+R3glrbmhAxTEzGdGwxXXJ2ybwJ2PWJLMYZ7g98nLyX+uQPaA3BlsbYJHNeS5283/9pJsd9DzfHKsN2nFSc=" - -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - cmake - - clang-format - update: true - -matrix: - include: - - os: linux - dist: focal - compiler: gcc - env: CC=cc CXX=c++ - - os: linux - dist: focal - compiler: clang - env: CC=clang CXX=clang++ - - os: linux - dist: bionic - compiler: gcc - env: CC=cc CXX=c++ - - os: linux - dist: bionic - compiler: clang - env: CC=clang CXX=clang++ - - os: linux - dist: xenial - compiler: gcc - env: CC=cc CXX=c++ - - os: linux - dist: xenial - compiler: clang - env: CC=clang CXX=clang++ - - os: osx - osx_image: xcode11.3 - env: CC=cc CXX=c++ - - os: osx - osx_image: xcode9.4 - env: CC=cc CXX=c++ - -before_script: | - if [ "${TRAVIS_BRANCH}" = "coverity_scan" ]; then - # call Coverity Scan manually of addons.coverity_scan for first job only - if [ "${TRAVIS_JOB_NUMBER}" = "${TRAVIS_BUILD_NUMBER}.1" ]; then - export COVERITY_SCAN_BRANCH=1 - echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- - curl -s 'https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh' -o coverity_scan.sh - else - echo 'echo "Skip CoverityScan for unrelated os/compiler"' > coverity_scan.sh - fi - fi - -script: | - ${CC} --version - ${CXX} --version - git fetch --unshallow --tags --prune || exit 1 - if [ ! -s ./coverity_scan.sh ]; then - make --keep-going all && MALLOC_CHECK_=7 MALLOC_PERTURB_=42 make --keep-going check - else - COVERITY_SCAN_PROJECT_NAME="ReOpen/libmdbx" \ - COVERITY_SCAN_NOTIFICATION_EMAIL="leo@yuriev.ru" \ - COVERITY_SCAN_BUILD_COMMAND_PREPEND="" \ - COVERITY_SCAN_BUILD_COMMAND="make MDBX_OPTIONS=-DMDBX_DEBUG=2 build-test" \ - COVERITY_SCAN_BRANCH_PATTERN="$TRAVIS_BRANCH" \ - bash ./coverity_scan.sh || cat cov-int/scm_log.txt - fi - -after_script: | - if [ "${TRAVIS_BRANCH}" != "coverity_scan" -a "${TRAVIS_JOB_NUMBER}" = "${TRAVIS_BUILD_NUMBER}.1" ] && make reformat && [[ -n $(git diff) ]]; then - echo "You must run 'make reformat' before submitting a pull request" - echo "-------------------------------------------------------------------------------" - git diff - sleep 1 - echo "-------------------------------------------------------------------------------" - sleep 1 - exit -1 - fi - echo "-------------------------------------------------------------------------------" - sleep 1 diff --git a/libs/libmdbx/src/AUTHORS b/libs/libmdbx/src/AUTHORS deleted file mode 100644 index 10910e5744..0000000000 --- a/libs/libmdbx/src/AUTHORS +++ /dev/null @@ -1,32 +0,0 @@ -Contributors -============ - -Alexey Naumov <alexey.naumov@gmail.com> -Chris Mikkelson <cmikk@qwest.net> -Claude Brisson <claude.brisson@gmail.com> -David Barbour <dmbarbour@gmail.com> -David Wilson <dw@botanicus.net> -dreamsxin <dreamsxin@126.com> -Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no> -Heiko Becker <heirecka@exherbo.org> -Howard Chu <hyc@openldap.org>, <hyc@symas.com> -Ignacio Casal Quinteiro <ignacio.casal@nice-software.com> -James Rouzier <rouzier@gmail.com> -Jean-Christophe DUBOIS <jcd@tribudubois.net> -John Hewson <john@jahewson.com> -Klaus Malorny <klaus.malorny@knipp.de> -Kurt Zeilenga <kurt.zeilenga@isode.com> -Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com> -Lorenz Bauer <lmb@cloudflare.com> -Luke Yeager <lyeager@nvidia.com> -Martin Hedenfalk <martin@bzero.se> -Ondrej Kuznik <ondrej.kuznik@acision.com> -Orivej Desh <orivej@gmx.fr> -Oskari Timperi <oskari.timperi@iki.fi> -Pavel Medvedev <pmedvedev@gmail.com> -Philipp Storz <philipp.storz@bareos.com> -Quanah Gibson-Mount <quanah@openldap.org> -Salvador Ortiz <sog@msg.com.mx> -Sebastien Launay <sebastien@slaunay.fr> -Vladimir Romanov <vromanov@gmail.com> -Zano Foundation <crypto.sowle@gmail.com> diff --git a/libs/libmdbx/src/CMakeLists.txt b/libs/libmdbx/src/CMakeLists.txt index e00f34034a..20a50a4537 100644 --- a/libs/libmdbx/src/CMakeLists.txt +++ b/libs/libmdbx/src/CMakeLists.txt @@ -62,7 +62,8 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/mdbx_chk.c") + EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/mdbx_chk.c" AND + EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/mdbx.c++") set(MDBX_AMALGAMATED_SOURCE FALSE) find_program(GIT git) if(NOT GIT) @@ -71,6 +72,7 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND set(MDBX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src") elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.c" AND + EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.c++" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/config.h.in" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/man1" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_chk.c") @@ -86,15 +88,35 @@ if(DEFINED PROJECT_NAME) if(NOT MDBX_AMALGAMATED_SOURCE AND NOT DEFINED BUILD_TESTING) set(BUILD_TESTING OFF) endif() + enable_language(C) else() set(SUBPROJECT OFF) set(NOT_SUBPROJECT ON) - project(libmdbx C CXX) + project(libmdbx C) if(NOT MDBX_AMALGAMATED_SOURCE AND NOT DEFINED BUILD_TESTING) set(BUILD_TESTING ON) endif() endif() +if(MDBX_AMALGAMATED_SOURCE AND DEFINED MDBX_ENABLE_TESTS AND MDBX_ENABLE_TESTS) + message(WARNING "MDBX_ENABLE_TESTS=${MDBX_ENABLE_TESTS}: But amalgamated source code don't includes tests.") + set(MDBX_ENABLE_TESTS OFF) +endif() + +# Try to find a C++ compiler unless sure that this is unnecessary. +if (NOT CMAKE_CXX_COMPILER_LOADED) + include(CheckLanguage) + if(NOT DEFINED MDBX_BUILD_CXX OR MDBX_BUILD_CXX + OR (NOT MDBX_AMALGAMATED_SOURCE AND (NOT DEFINED MDBX_ENABLE_TESTS OR MDBX_ENABLE_TESTS))) + check_language(CXX) + if(CMAKE_CXX_COMPILER) + enable_language(CXX) + endif() + endif() +else() + enable_language(CXX) +endif() + if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." @@ -105,16 +127,6 @@ if(NOT_SUBPROJECT AND (CMAKE_CROSSCOMPILING OR IOS)) set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) endif() -if(IOS) - set(MDBX_BUILD_TOOLS_DEFAULT OFF) - if(NOT_SUBPROJECT) - cmake_policy(SET CMP0006 OLD) - set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO") - endif() -else() - set(MDBX_BUILD_TOOLS_DEFAULT ON) -endif() - if(NOT "$ENV{TEAMCITY_PROCESS_FLOW_ID}" STREQUAL "") set(CI TEAMCITY) message(STATUS "TeamCity CI") @@ -161,10 +173,27 @@ if(NOT CMAKE_BUILD_TYPE) endif() string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE) +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) + include(cmake/utils.cmake) include(cmake/compiler.cmake) include(cmake/profile.cmake) +# Workaround for `-pthread` toolchain/cmake bug +if(NOT APPLE AND NOT MSVC + AND CMAKE_USE_PTHREADS_INIT AND NOT CMAKE_THREAD_LIBS_INIT + AND (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)) + check_compiler_flag("-pthread" CC_HAS_PTHREAD) + if(CC_HAS_PTHREAD AND NOT CMAKE_EXE_LINKER_FLAGS MATCHES "-pthread") + message(STATUS "Force add -pthread for linker flags to avoid troubles") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pthread") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -pthread") + endif() +endif() + CHECK_FUNCTION_EXISTS(pow NOT_NEED_LIBM) if(NOT_NEED_LIBM) set(LIB_MATH "") @@ -178,18 +207,16 @@ else() endif() endif() -find_package(Threads REQUIRED) - if(SUBPROJECT) if(NOT DEFINED BUILD_SHARED_LIBS) option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" OFF) endif() if(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE) - option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON) + option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independent (PIC)" ON) endif() else() option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" ON) - option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON) + option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independent (PIC)" ON) if (CC_HAS_ARCH_NATIVE) option(BUILD_FOR_NATIVE_CPU "Generate code for the compiling machine CPU" OFF) endif() @@ -294,14 +321,30 @@ else() endif(SUBPROJECT) list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11) +list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_11 HAS_CXX11) +list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_14 HAS_CXX14) +list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_17 HAS_CXX17) +list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_20 HAS_CXX20) +if(NOT DEFINED MDBX_CXX_STANDARD) + if(DEFINED CMAKE_CXX_STANDARD) + set(MDBX_CXX_STANDARD ${CMAKE_CXX_STANDARD}) + elseif(NOT HAS_CXX20 LESS 0) + set(MDBX_CXX_STANDARD 20) + elseif(NOT HAS_CXX17 LESS 0) + set(MDBX_CXX_STANDARD 17) + elseif(NOT HAS_CXX14 LESS 0) + set(MDBX_CXX_STANDARD 14) + elseif(NOT HAS_CXX11 LESS 0) + set(MDBX_CXX_STANDARD 11) + else() + set(MDBX_CXX_STANDARD 98) + endif() +endif() if(NOT HAS_C11 LESS 0) set(MDBX_C_STANDARD 11) else() set(MDBX_C_STANDARD 99) endif() -if(MDBX_C_STANDARD) - message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx") -endif() if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND EXISTS "${MDBX_SOURCE_DIR}/ntdll.def") if(MSVC) @@ -354,15 +397,6 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND EXISTS "${MDBX_SOURCE_DIR}/ntdll. endif() endif() -macro(add_mdbx_option NAME DESCRIPTION DEFAULT) - list(APPEND MDBX_BUILD_OPTIONS ${NAME}) - if(NOT ${DEFAULT} STREQUAL "AUTO") - option(${NAME} "${DESCRIPTION}" ${DEFAULT}) - elseif(NOT DEFINED ${NAME}) - set(${NAME}_AUTO ON) - endif() -endmacro() - ################################################################################ ################################################################################ # @@ -375,13 +409,33 @@ endmacro() # set(MDBX_BUILD_OPTIONS ENABLE_ASAN MDBX_USE_VALGRIND ENABLE_GPROF ENABLE_GCOV) +macro(add_mdbx_option NAME DESCRIPTION DEFAULT) + list(APPEND MDBX_BUILD_OPTIONS ${NAME}) + if(NOT ${DEFAULT} STREQUAL "AUTO") + option(${NAME} "${DESCRIPTION}" ${DEFAULT}) + elseif(NOT DEFINED ${NAME}) + set(${NAME}_AUTO ON) + endif() +endmacro() + +if(IOS) + set(MDBX_BUILD_TOOLS_DEFAULT OFF) + if(NOT_SUBPROJECT) + cmake_policy(SET CMP0006 OLD) + set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO") + endif() +else() + set(MDBX_BUILD_TOOLS_DEFAULT ON) +endif() + add_mdbx_option(MDBX_INSTALL_STATIC "Build and install libmdbx for static linking" OFF) add_mdbx_option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS}) add_mdbx_option(MDBX_BUILD_TOOLS "Build MDBX tools (mdbx_chk/stat/dump/load/copy)" ${MDBX_BUILD_TOOLS_DEFAULT}) +CMAKE_DEPENDENT_OPTION(MDBX_INSTALL_MANPAGES "Install man-pages for MDBX tools (mdbx_chk/stat/dump/load/copy)" ON MDBX_BUILD_TOOLS OFF) add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON) -add_mdbx_option(MDBX_TXN_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO) +add_mdbx_option(MDBX_ENV_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO) add_mdbx_option(MDBX_HUGE_TRANSACTIONS "Support for huge write-transactions" OFF) -mark_as_advanced(MDBX_TXN_CHECKPID) +mark_as_advanced(MDBX_ENV_CHECKPID) if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") add_mdbx_option(MDBX_DISABLE_GNU_SOURCE "Don't use GNU/Linux libc extensions" OFF) mark_as_advanced(MDBX_DISABLE_GNU_SOURCE) @@ -392,7 +446,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" OR IOS) endif() if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") if(MDBX_NTDLL_EXTRA_IMPLIB) - add_mdbx_option(MDBX_AVOID_CRT "Avoid dependence from MSVC CRT and use ntdll.dll instead" ${NOT_SUBPROJECT}) + add_mdbx_option(MDBX_AVOID_CRT "Avoid dependence from MSVC CRT and use ntdll.dll instead" OFF) endif() add_mdbx_option(MDBX_CONFIG_MANUAL_TLS_CALLBACK "Provide mdbx_dll_handler() for manual initialization" OFF) @@ -409,8 +463,7 @@ option(MDBX_FORCE_ASSERTIONS "Force enable assertion checking" OFF) if(NOT MDBX_AMALGAMATED_SOURCE) add_mdbx_option(MDBX_ALLOY_BUILD "Build MDBX library through single/alloyed object file" ON) - option(MDBX_ENABLE_TESTS "Build MDBX tests" ${BUILD_TESTING}) -endif(NOT MDBX_AMALGAMATED_SOURCE) +endif() if((MDBX_BUILD_TOOLS OR MDBX_ENABLE_TESTS) AND MDBX_BUILD_SHARED_LIBRARY) add_mdbx_option(MDBX_LINK_TOOLS_NONSTATIC "Link MDBX tools with non-static libmdbx" OFF) @@ -418,14 +471,36 @@ else() unset(MDBX_LINK_TOOLS_NONSTATIC CACHE) endif() +if(CMAKE_CXX_COMPILER_LOADED AND MDBX_CXX_STANDARD GREATER_EQUAL 11 AND MDBX_CXX_STANDARD LESS 83) + if(NOT MDBX_AMALGAMATED_SOURCE) + option(MDBX_ENABLE_TESTS "Build MDBX tests" ${BUILD_TESTING}) + endif() + if(NOT MDBX_AVOID_CRT + AND NOT (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) + AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4) + AND NOT (MSVC AND MSVC_VERSION LESS 1900)) + option(MDBX_BUILD_CXX "Build C++ portion" ON) + else() + set(MDBX_BUILD_CXX FALSE) + endif() +else() + set(MDBX_BUILD_CXX FALSE) + set(MDBX_ENABLE_TESTS FALSE) +endif() + ################################################################################ ################################################################################ +if(MDBX_BUILD_CXX AND NOT CMAKE_CXX_COMPILER_LOADED) + message(FATAL_ERROR "MDBX_BUILD_CXX=${MDBX_BUILD_CXX}: The C++ compiler is required to build the C++API.") +endif() + # Get version fetch_version(MDBX "${CMAKE_CURRENT_SOURCE_DIR}" FALSE) message(STATUS "libmdbx version is ${MDBX_VERSION}") # sources list +set(LIBMDBX_PUBLIC_HEADERS mdbx.h) set(LIBMDBX_SOURCES mdbx.h "${CMAKE_CURRENT_BINARY_DIR}/config.h") if(MDBX_AMALGAMATED_SOURCE) list(APPEND LIBMDBX_SOURCES mdbx.c) @@ -438,7 +513,7 @@ else() set(MDBX_BUILD_SOURCERY "${MDBX_SOURCERY_DIGEST}_${MDBX_SOURCERY_SUFFIX}") if(MDBX_ALLOY_BUILD) - list(APPEND LIBMDBX_SOURCES ${MDBX_SOURCE_DIR}/alloy.c) + list(APPEND LIBMDBX_SOURCES "${MDBX_SOURCE_DIR}/alloy.c") include_directories("${MDBX_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") else() list(APPEND LIBMDBX_SOURCES @@ -450,12 +525,25 @@ else() include_directories("${MDBX_SOURCE_DIR}") endif() endif(MDBX_AMALGAMATED_SOURCE) +if(MDBX_BUILD_CXX) + message(STATUS "Use C${MDBX_C_STANDARD} and C++${MDBX_CXX_STANDARD} for libmdbx") + list(APPEND LIBMDBX_PUBLIC_HEADERS mdbx.h++) + list(APPEND LIBMDBX_SOURCES "${MDBX_SOURCE_DIR}/mdbx.c++" mdbx.h++) +else() + message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx but C++ portion is disabled") +endif() macro(target_setup_options TARGET) if(DEFINED INTERPROCEDURAL_OPTIMIZATION) set_target_properties(${TARGET} PROPERTIES INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>) endif() + set_target_properties(${TARGET} PROPERTIES + C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) + if(MDBX_BUILD_CXX) + set_target_properties(${TARGET} PROPERTIES + CXX_STANDARD ${MDBX_CXX_STANDARD} CXX_STANDARD_REQUIRED ON) + endif() if(CC_HAS_FASTMATH) target_compile_options(${TARGET} PRIVATE "-ffast-math") endif() @@ -468,7 +556,7 @@ macro(target_setup_options TARGET) endmacro() macro(libmdbx_setup_libs TARGET MODE) - target_link_libraries(${TARGET} ${MODE} ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(${TARGET} ${MODE} Threads::Threads) if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") target_link_libraries(${TARGET} ${MODE} ntdll.lib) if(MDBX_NTDLL_EXTRA_IMPLIB AND MDBX_AVOID_CRT) @@ -479,6 +567,9 @@ macro(libmdbx_setup_libs TARGET MODE) elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(${TARGET} ${MODE} log) endif() + if(LIBCXX_FILESYSTEM AND MDBX_BUILD_CXX) + target_link_libraries(${TARGET} ${MODE} ${LIBCXX_FILESYSTEM}) + endif() endmacro() # build static library @@ -487,11 +578,7 @@ if(MDBX_INSTALL_STATIC) else() add_library(mdbx-static STATIC EXCLUDE_FROM_ALL ${LIBMDBX_SOURCES}) endif() -set_target_properties(mdbx-static PROPERTIES PUBLIC_HEADER mdbx.h) -if(MDBX_C_STANDARD) - set_target_properties(mdbx-static PROPERTIES - C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) -endif() +set_target_properties(mdbx-static PROPERTIES PUBLIC_HEADER "${LIBMDBX_PUBLIC_HEADERS}") target_compile_definitions(mdbx-static PRIVATE MDBX_BUILD_SHARED_LIBRARY=0) target_setup_options(mdbx-static) libmdbx_setup_libs(mdbx-static INTERFACE) @@ -506,11 +593,7 @@ endif() # build shared library if(MDBX_BUILD_SHARED_LIBRARY) add_library(mdbx SHARED ${LIBMDBX_SOURCES}) - set_target_properties(mdbx PROPERTIES PUBLIC_HEADER mdbx.h) - if(MDBX_C_STANDARD) - set_target_properties(mdbx PROPERTIES - C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) - endif() + set_target_properties(mdbx PROPERTIES PUBLIC_HEADER "${LIBMDBX_PUBLIC_HEADERS}") target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS MDBX_BUILD_SHARED_LIBRARY=1 INTERFACE LIBMDBX_IMPORTS) target_setup_options(mdbx) libmdbx_setup_libs(mdbx PRIVATE) @@ -579,19 +662,24 @@ endif() ################################################################################ # mdbx-shared-lib installation +if(NOT DEFINED MDBX_DLL_INSTALL_DESTINATION) + if(WIN32) + set(MDBX_DLL_INSTALL_DESTINATION bin) + else() + set(MDBX_DLL_INSTALL_DESTINATION lib) + endif() +endif() if(MDBX_BUILD_SHARED_LIBRARY) if(CMAKE_VERSION VERSION_LESS 3.12) install(TARGETS mdbx EXPORT libmdbx - RUNTIME DESTINATION bin COMPONENT runtime - LIBRARY DESTINATION bin COMPONENT runtime + LIBRARY DESTINATION ${MDBX_DLL_INSTALL_DESTINATION} COMPONENT runtime OBJECTS DESTINATION lib COMPONENT devel ARCHIVE DESTINATION lib COMPONENT devel PUBLIC_HEADER DESTINATION include COMPONENT devel INCLUDES DESTINATION include COMPONENT devel) else() install(TARGETS mdbx EXPORT libmdbx - RUNTIME DESTINATION bin COMPONENT runtime - LIBRARY DESTINATION bin COMPONENT runtime + LIBRARY DESTINATION ${MDBX_DLL_INSTALL_DESTINATION} COMPONENT runtime NAMELINK_COMPONENT devel OBJECTS DESTINATION lib COMPONENT devel ARCHIVE DESTINATION lib COMPONENT devel @@ -602,6 +690,9 @@ endif(MDBX_BUILD_SHARED_LIBRARY) # mdbx-tools installation if(MDBX_BUILD_TOOLS) + if(NOT DEFINED MDBX_TOOLS_INSTALL_DESTINATION) + set(MDBX_TOOLS_INSTALL_DESTINATION bin) + endif() install( TARGETS mdbx_chk @@ -610,33 +701,36 @@ if(MDBX_BUILD_TOOLS) mdbx_dump mdbx_load RUNTIME - DESTINATION bin + DESTINATION ${MDBX_TOOLS_INSTALL_DESTINATION} COMPONENT runtime) - install( - FILES - "${MDBX_SOURCE_DIR}/man1/mdbx_chk.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_stat.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_copy.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_dump.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_load.1" - DESTINATION man/man1 - COMPONENT doc) + if(MDBX_INSTALL_MANPAGES) + if(NOT DEFINED MDBX_MAN_INSTALL_DESTINATION) + set(MDBX_MAN_INSTALL_DESTINATION man/man1) + endif() + install( + FILES + "${MDBX_SOURCE_DIR}/man1/mdbx_chk.1" + "${MDBX_SOURCE_DIR}/man1/mdbx_stat.1" + "${MDBX_SOURCE_DIR}/man1/mdbx_copy.1" + "${MDBX_SOURCE_DIR}/man1/mdbx_dump.1" + "${MDBX_SOURCE_DIR}/man1/mdbx_load.1" + DESTINATION ${MDBX_MAN_INSTALL_DESTINATION} + COMPONENT doc) + endif() endif(MDBX_BUILD_TOOLS) # mdbx-static-lib installation if(MDBX_INSTALL_STATIC) if(CMAKE_VERSION VERSION_LESS 3.12) install(TARGETS mdbx-static EXPORT libmdbx - RUNTIME DESTINATION bin COMPONENT runtime - LIBRARY DESTINATION bin COMPONENT runtime + LIBRARY DESTINATION lib COMPONENT devel OBJECTS DESTINATION lib COMPONENT devel ARCHIVE DESTINATION lib COMPONENT devel PUBLIC_HEADER DESTINATION include COMPONENT devel INCLUDES DESTINATION include COMPONENT devel) else() install(TARGETS mdbx-static EXPORT libmdbx - RUNTIME DESTINATION bin COMPONENT runtime - LIBRARY DESTINATION bin COMPONENT runtime + LIBRARY DESTINATION lib COMPONENT devel NAMELINK_COMPONENT devel OBJECTS DESTINATION lib COMPONENT devel ARCHIVE DESTINATION lib COMPONENT devel @@ -650,10 +744,16 @@ endif(MDBX_INSTALL_STATIC) # collect options & build info string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC) set(MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS}) +if(MDBX_BUILD_CXX) + set(MDBX_BUILD_FLAGS ${CMAKE_CXX_FLAGS}) +endif() # append cmake's build-type flags and defines if(NOT CMAKE_CONFIGURATION_TYPES) list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}}) + if(MDBX_BUILD_CXX) + list(APPEND MDBX_BUILD_FLAGS ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}}) + endif() endif() # get definitions @@ -705,13 +805,15 @@ endif() # provide build-type if(CMAKE_CONFIGURATION_TYPES) + # via per-configuration define add_definitions(-DMDBX_BUILD_TYPE="$<CONFIG>") + set(MDBX_BUILD_TYPE "<CONFIGURATION DEPENDENT>") else() set(MDBX_BUILD_TYPE ${CMAKE_BUILD_TYPE}) endif() # options -set(options VERSION C_COMPILER CXX_COMPILER) +set(options VERSION C_COMPILER CXX_COMPILER MDBX_BUILD_TARGET MDBX_BUILD_TYPE) foreach(item IN LISTS options) if(DEFINED ${item}) set(value "${${item}}") @@ -735,9 +837,14 @@ add_definitions(-DMDBX_CONFIG_H="${CMAKE_CURRENT_BINARY_DIR}/config.h") ################################################################################ if(NOT MDBX_AMALGAMATED_SOURCE AND MDBX_ENABLE_TESTS) + if(NOT CMAKE_CXX_COMPILER_LOADED) + message(FATAL_ERROR "MDBX_ENABLE_TESTS=${MDBX_ENABLE_TESTS}: The C++ compiler is required to build the tests.") + endif() add_subdirectory(test) endif() +################################################################################ + set(PACKAGE "libmdbx") set(CPACK_PACKAGE_VERSION_MAJOR ${MDBX_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${MDBX_VERSION_MINOR}) diff --git a/libs/libmdbx/src/COPYRIGHT b/libs/libmdbx/src/COPYRIGHT deleted file mode 100644 index fc6d9eed24..0000000000 --- a/libs/libmdbx/src/COPYRIGHT +++ /dev/null @@ -1,22 +0,0 @@ -Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>. -Copyright 2011-2015 Howard Chu, Symas Corp. -Copyright 2015,2016 Peter-Service R&D LLC. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted only as authorized by the OpenLDAP -Public License. - -A copy of this license is available in the file LICENSE in the -top-level directory of the distribution or, alternatively, at -<http://www.OpenLDAP.org/license.html>. - -OpenLDAP is a registered trademark of the OpenLDAP Foundation. - -Individual files and/or contributed packages may be copyright by -other parties and/or subject to additional restrictions. - -This work also contains materials derived from public sources. - -Additional information about OpenLDAP can be obtained at -<http://www.openldap.org/>. diff --git a/libs/libmdbx/src/ChangeLog.md b/libs/libmdbx/src/ChangeLog.md index 2208b212ed..34be056b8d 100644 --- a/libs/libmdbx/src/ChangeLog.md +++ b/libs/libmdbx/src/ChangeLog.md @@ -1,9 +1,87 @@ -## v0.9.x (in the development): -- TODO: API for explicit threads (de)registration. -- TODO: Native bindings for C++. -- TODO: Packages for AltLinux, Fedora/RHEL, Debian/Ubuntu. +ChangeLog +--------- -## v0.8.2 2020-07-06: +## v0.9.2 (in development) + +TODO: + + - Rework/speedup the implementation of the dirty page list (lazy compactification, lazy sorting via merge). + - Finalize C++ API (few typos and trivia bugs are likely for now). + - Packages for ROSA Linux, ALT Linux, Fedora/RHEL, Debian/Ubuntu. + +Fixes: + + - Fixed missing installation of `mdbx.h++`. + - Fixed use of obsolete `__noreturn`. + - Fixed use of `yield` instruction on ARM if unsupported. + - Added pthread workaround for buggy toolchain/cmake/buildroot. + - Fixed use of `pthread_yield()` for non-GLIBC. + - Fixed use of `RegGetValueA()` on Windows 2000/XP. + - Fixed use of `GetTickCount64()` on Windows 2000/XP. + - Fixed opening DB on a network shares (in the exclusive mode). + - Fixed copy&paste typos. + - Fixed minor false-positive GCC warning. + +## v0.9.1 2020-09-30 + +Added features: + + - Preliminary C++ API with support for C++17 polymorphic allocators. + - [Online C++ API reference](https://erthink.github.io/libmdbx/) by Doxygen. + - Quick reference for Insert/Update/Delete operations. + - Explicit `MDBX_SYNC_DURABLE` to sync modes for API clarity. + - Explicit `MDBX_ALLDUPS` and `MDBX_UPSERT` for API clarity. + - Support for read transactions preparation (`MDBX_TXN_RDONLY_PREPARE` flag). + - Support for cursor preparation/(pre)allocation and reusing (`mdbx_cursor_create()` and `mdbx_cursor_bind()` functions). + - Support for checking database using specified meta-page (see `mdbx_chk -h`). + - Support for turn to the specific meta-page after checking (see `mdbx_chk -h`). + - Support for explicit reader threads (de)registration. + - The `mdbx_txn_break()` function to explicitly mark a transaction as broken. + - Improved handling of corrupted databases by `mdbx_chk` utility and `mdbx_walk_tree()` function. + - Improved DB corruption detection by checking parent-page-txnid. + - Improved opening large DB (> 4Gb) from 32-bit code. + - Provided `pure-function` and `const-function` attributes to C API. + - Support for user-settable context for transactions & cursors. + - Revised API and documentation related to Handle-Slow-Readers callback feature. + +Deprecated functions and flags: + + - For clarity and API simplification the `MDBX_MAPASYNC` flag is deprecated. + Just use `MDBX_SAFE_NOSYNC` or `MDBX_UTTERLY_NOSYNC` instead of it. + - `MDBX_oom_func`, `mdbx_env_set_oomfunc()` and `mdbx_env_get_oomfunc()` + replaced with `MDBX_hsr_func`, `mdbx_env_get_hsr` and `mdbx_env_get_hsr()`. + +Fixes: + + - Fix `mdbx_strerror()` for `MDBX_BUSY` error (no error description is returned). + - Fix update internal meta-geo information in read-only mode (`EACCESS` or `EBADFD` error). + - Fix `mdbx_page_get()` null-defer when DB corrupted (crash by `SIGSEGV`). + - Fix `mdbx_env_open()` for re-opening after non-fatal errors (`mdbx_chk` unexpected failures). + - Workaround for MSVC 19.27 `static_assert()` bug. + - Doxygen descriptions and refinement. + - Update Valgrind's suppressions. + - Workaround to avoid infinite loop of 'nested' testcase on MIPS under QEMU. + - Fix a lot of typos & spelling (Thanks to Josh Soref for PR). + - Fix `getopt()` messages for Windows (Thanks to Andrey Sporaw for reporting). + - Fix MSVC compiler version requirements (Thanks to Andrey Sporaw for reporting). + - Workarounds for QEMU's bugs to run tests for cross-builded library under QEMU. + - Now C++ compiler optional for building by CMake. + + +## v0.9.0 2020-07-31 (not a release, but API changes) + +Added features: + + - [Online C API reference](https://erthink.github.io/libmdbx/) by Doxygen. + - Separated enums for environment, sub-databases, transactions, copying and data-update flags. + +Deprecated functions and flags: + + - Usage of custom comparators and the `mdbx_dbi_open_ex()` are deprecated, since such databases couldn't be checked by the `mdbx_chk` utility. + Please use the value-to-key functions to provide keys that are compatible with the built-in libmdbx comparators. + + +## v0.8.2 2020-07-06 - Added support multi-opening the same DB in a process with SysV locking (BSD). - Fixed warnings & minors for LCC compiler (E2K). - Enabled to simultaneously open the same database from processes with and without the `MDBX_WRITEMAP` option. @@ -17,7 +95,8 @@ - Remapping on-the-fly and of the database file was implemented. Now remapping with a change of address is performed automatically if there are no dependent readers in the current process. -## v0.8.1 2020-06-12: + +## v0.8.1 2020-06-12 - Minor change versioning. The last number in the version now means the number of commits since last release/tag. - Provide ChangeLog file. - Fix for using libmdbx as a C-only sub-project with CMake. @@ -26,7 +105,8 @@ - Refine LTO (link time optimization) for clang. - Force enabling exceptions handling for MSVC (`/EHsc` option). -## v0.8.0 2020-06-05: + +## v0.8.0 2020-06-05 - Support for Android/Bionic. - Support for iOS. - Auto-handling `MDBX_NOSUBDIR` while opening for any existing database. @@ -70,7 +150,8 @@ - Minor fix/workaround to avoid UBSAN traps for `memcpy(ptr, NULL, 0)`. - Avoid some GCC-analyzer false-positive warnings. -## v0.7.0 2020-03-18: + +## v0.7.0 2020-03-18 - Workarounds for Wine (Windows compatibility layer for Linux). - `MDBX_MAP_RESIZED` renamed to `MDBX_UNABLE_EXTEND_MAPSIZE`. - Clarify API description, fix typos. @@ -81,7 +162,8 @@ - Refine/clarify error messages. - Avoids extra error messages "bad txn" from mdbx_chk when DB is corrupted. -## v0.6.0 2020-01-21: + +## v0.6.0 2020-01-21 - Fix `mdbx_load` utility for custom comparators. - Fix checks related to `MDBX_APPEND` flag inside `mdbx_cursor_put()`. - Refine/fix dbi_bind() internals. @@ -92,7 +174,8 @@ - Fix env_set_geometry() for large pagesize. - Clarify API description & comments, fix typos. -## v0.5.0 2019-12-31: + +## v0.5.0 2019-12-31 - Fix returning MDBX_RESULT_TRUE from page_alloc(). - Fix false-positive ASAN issue. - Fix assertion for `MDBX_NOTLS` option. @@ -106,7 +189,8 @@ - Avoid using `FILE_FLAG_NO_BUFFERING` for compatibility with small database pages. - Added install section for CMake. -## v0.4.0 2019-12-02: + +## v0.4.0 2019-12-02 - Support for Mac OSX, FreeBSD, NetBSD, OpenBSD, DragonFly BSD, OpenSolaris, OpenIndiana (AIX and HP-UX pending). - Use bootid for decisions of rollback. - Counting retired pages and extended transaction info. diff --git a/libs/libmdbx/src/GNUmakefile b/libs/libmdbx/src/GNUmakefile index 0123a36923..82d168b2b8 100644 --- a/libs/libmdbx/src/GNUmakefile +++ b/libs/libmdbx/src/GNUmakefile @@ -1,11 +1,19 @@ -# This makefile is for GNU Make, and nowadays provided +# This makefile is for GNU Make 3.80 or above, and nowadays provided # just for compatibility and preservation of traditions. -# Please use CMake in case of any difficulties or problems. # -# Preprocessor macros (for MDBX_OPTIONS) of interest... +# Please use CMake in case of any difficulties or +# problems with this old-school's magic. +# +################################################################################ +# +# Preprocessor macros (for MDBX_OPTIONS) of interest. +# # Note that the defaults should already be correct for most platforms; # you should not need to change any of these. Read their descriptions -# in README and source code if you do. There may be other macros of interest. +# in README and source code (see src/options.h) if you do. +# There may be other macros of interest. +# + SHELL := env bash # install sandbox @@ -24,6 +32,10 @@ LD ?= ld MDBX_OPTIONS ?= -DNDEBUG=1 CFLAGS ?= -O2 -g -Wall -Werror -Wextra -Wpedantic -ffunction-sections -fPIC -fvisibility=hidden -std=gnu11 -pthread -Wno-error=attributes $(CFLAGS_EXTRA) # -Wno-tautological-compare +CXX ?= g++ +# Choosing C++ standard with deferred simple variable expansion trick +CXXSTD ?= $(eval CXXSTD := $$(shell PROBE=$$$$([ -f mdbx.c++ ] && echo mdbx.c++ || echo src/mdbx.c++); for std in gnu++20 c++20 gnu++2a c++2a gnu++17 c++17 gnu++14 c++14 gnu+11 c++11; do $(CXX) -std=$$$${std} -c $$$${PROBE} -o /dev/null 2>/dev/null >/dev/null && echo "-std=$$$${std}" && exit; done))$(CXXSTD) +CXXFLAGS = $(CXXSTD) $(filter-out -std=gnu11,$(CFLAGS)) # HINT: Try append '--no-as-needed,-lrt' for ability to built with modern glibc, but then run with the old. LIBS ?= $(shell uname | grep -qi SunOS && echo "-lkstat") $(shell uname | grep -qi -e Darwin -e OpenBSD || echo "-lrt") $(shell uname | grep -qi Windows && echo "-lntdll") @@ -43,12 +55,12 @@ define uname2sosuffix endef SO_SUFFIX := $(shell $(uname2sosuffix)) -HEADERS := mdbx.h +HEADERS := mdbx.h mdbx.h++ LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX) TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1 -.PHONY: mdbx all install clean test dist check +.PHONY: mdbx all install clean all: $(LIBRARIES) $(TOOLS) @@ -64,15 +76,12 @@ clean: *.gcov *.log *.err src/*.o test/*.o mdbx_example dist \ config.h src/config.h src/version.c *.tar* -libmdbx.a: mdbx-static.o +libmdbx.a: mdbx-static.o mdbx++-static.o $(AR) rs $@ $? -libmdbx.$(SO_SUFFIX): mdbx-dylib.o - $(CC) $(CFLAGS) $^ -pthread -shared $(LDFLAGS) $(LIBS) -o $@ +libmdbx.$(SO_SUFFIX): mdbx-dylib.o mdbx++-dylib.o + $(CXX) $(CXXFLAGS) $^ -pthread -shared $(LDFLAGS) $(LIBS) -o $@ -#> dist-cutoff-begin -ifeq ($(wildcard mdbx.c),mdbx.c) -#< dist-cutoff-end ################################################################################ # Amalgamated source code, i.e. distributed after `make dists` @@ -80,268 +89,26 @@ MAN_SRCDIR := man1/ config.h: mdbx.c $(lastword $(MAKEFILE_LIST)) (echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \ - && echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS) $(LIBS)"' \ + && echo '#define MDBX_BUILD_FLAGS "$(CXXSTD) $(CFLAGS) $(LDFLAGS) $(LIBS)"' \ && echo '#define MDBX_BUILD_COMPILER "$(shell (LC_ALL=C $(CC) --version || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \ && echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; (LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || (LC_ALL=C $(CC) --version | grep -qi e2k && echo E2K) || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \ ) > $@ -mdbx-dylib.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST)) +mdbx-dylib.o: config.h mdbx.c mdbx.h $(lastword $(MAKEFILE_LIST)) $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c mdbx.c -o $@ -mdbx-static.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST)) +mdbx-static.o: config.h mdbx.c mdbx.h $(lastword $(MAKEFILE_LIST)) $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c mdbx.c -o $@ -mdbx_%: mdbx_%.c libmdbx.a - $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) $(LIBS) -o $@ - -#> dist-cutoff-begin -else -################################################################################ -# Plain (non-amalgamated) sources with test - -define uname2osal - case "$(UNAME)" in - CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;; - *) echo unix;; - esac -endef - -define uname2titer - case "$(UNAME)" in - Darwin*|Mach*) echo 2;; - *) echo 12;; - esac -endef - -DIST_EXTRA := LICENSE README.md CMakeLists.txt GNUmakefile Makefile ChangeLog.md VERSION config.h.in ntdll.def \ - $(addprefix man1/, $(MANPAGES)) cmake/compiler.cmake cmake/profile.cmake cmake/utils.cmake -DIST_SRC := mdbx.h mdbx.c $(addsuffix .c, $(TOOLS)) - -TEST_DB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.db -TEST_LOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.log.gz -TEST_OSAL := $(shell $(uname2osal)) -TEST_ITER := $(shell $(uname2titer)) -TEST_SRC := test/osal-$(TEST_OSAL).cc $(filter-out $(wildcard test/osal-*.cc), $(wildcard test/*.cc)) -TEST_INC := $(wildcard test/*.h) -TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC)) -CXX ?= g++ -CXXSTD ?= $(shell $(CXX) -std=c++17 -c test/test.cc -o /dev/null 2>/dev/null && echo -std=c++17 || echo -std=c++11) -CXXFLAGS := $(CXXSTD) $(filter-out -std=gnu11,$(CFLAGS)) -TAR ?= $(shell which gnu-tar || echo tar) -ZIP ?= $(shell which zip || echo "echo 'Please install zip'") -CLANG_FORMAT ?= $(shell (which clang-format-12 || which clang-format-11 || which clang-format-10 || which clang-format) 2>/dev/null) - -reformat: - @if [ -n "$(CLANG_FORMAT)" ]; then \ - git ls-files | grep -E '\.(c|cxx|cc|cpp|h|hxx|hpp)(\.in)?$$' | xargs -r $(CLANG_FORMAT) -i --style=file; \ - else \ - echo "clang-format version 8..12 not found for 'reformat'"; \ - fi - -MAN_SRCDIR := src/man1/ -ALLOY_DEPS := $(wildcard src/*) -MDBX_GIT_VERSION = ${shell set -o pipefail; git describe --tags | sed -n 's|^v*\([0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\)\(.*\)|\1|p' || echo 'Please fetch tags and/or install latest git version'} -MDBX_GIT_REVISION = $(shell git rev-list --count HEAD ^`git tag --sort=-version:refname | sed -n '/^\(v[0-9]\+\.[0-9]\+\.[0-9]\+\)*/p;q'`) -MDBX_GIT_TIMESTAMP = $(shell git show --no-patch --format=%cI HEAD || echo 'Please install latest get version') -MDBX_GIT_DESCRIBE = $(shell git describe --tags --long --dirty=-dirty || echo 'Please fetch tags and/or install latest git version') -MDBX_VERSION_SUFFIX = $(shell set -o pipefail; echo -n '$(MDBX_GIT_DESCRIBE)' | tr -c -s '[a-zA-Z0-9]' _) -MDBX_BUILD_SOURCERY = $(shell set -o pipefail; $(MAKE) -s src/version.c && (openssl dgst -r -sha256 src/version.c || sha256sum src/version.c || shasum -a 256 src/version.c) 2>/dev/null | cut -d ' ' -f 1 || echo 'Please install openssl or sha256sum or shasum')_$(MDBX_VERSION_SUFFIX) -MDBX_DIST_DIR = libmdbx-$(MDBX_VERSION_SUFFIX) - -check: test dist - -test: build-test - rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \ - (./mdbx_test --table=+data.integer --keygen.split=29 --datalen.min=min --datalen.max=max --progress --console=no --repeat=$(TEST_ITER) --pathname=$(TEST_DB) --dont-cleanup-after basic && \ - ./mdbx_test --mode=-writemap,-mapasync,-lifo --progress --console=no --repeat=12 --pathname=$(TEST_DB) --dont-cleanup-after basic) \ - | tee >(gzip --stdout > $(TEST_LOG)) | tail -n 42) \ - && ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy - -test-singleprocess: all mdbx_test - rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \ - (./mdbx_test --table=+data.integer --keygen.split=29 --datalen.min=min --datalen.max=max --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --hill && \ - ./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \ - ./mdbx_test --mode=-writemap,-mapasync,-lifo --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --nested) \ - | tee >(gzip --stdout > $(TEST_LOG)) | tail -n 42) \ - && ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy - -test-fault: all mdbx_test - rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --inject-writefault=42 --dump-config --dont-cleanup-after basic \ - | tee >(gzip --stdout > $(TEST_LOG)) | tail -n 42) \ - ; ./mdbx_chk -vvnw $(TEST_DB) && ([ ! -e $(TEST_DB)-copy ] || ./mdbx_chk -vvn $(TEST_DB)-copy) - -VALGRIND=valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt -memcheck test-valgrind: - $(MAKE) clean && $(MAKE) CFLAGS_EXTRA="-Ofast -DMDBX_USE_VALGRIND" build-test && \ - rm -f valgrind-*.log $(TEST_DB) $(TEST_LOG) && (set -o pipefail; ( \ - $(VALGRIND) ./mdbx_test --table=+data.integer --keygen.split=29 --datalen.min=min --datalen.max=max --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-after basic && \ - $(VALGRIND) ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \ - $(VALGRIND) ./mdbx_test --mode=-writemap,-mapasync,-lifo --progress --console=no --repeat=4 --pathname=$(TEST_DB) --dont-cleanup-after basic && \ - $(VALGRIND) ./mdbx_chk -vvn $(TEST_DB) && \ - $(VALGRIND) ./mdbx_chk -vvn $(TEST_DB)-copy \ - ) | tee >(gzip --stdout > $(TEST_LOG)) | tail -n 42) - -gcc-analyzer: - @echo "NOTE: There a lot of false-positive warnings at 2020-05-01 by pre-release GCC-10 (20200328, Red Hat 10.0.1-0.11)" - $(MAKE) --always-make CFLAGS_EXTRA="-Og -fanalyzer -Wno-error" build-test - -test-ubsan: - $(MAKE) clean && $(MAKE) CFLAGS_EXTRA="-Ofast -fsanitize=undefined -fsanitize-undefined-trap-on-error" check - -test-asan: - $(MAKE) clean && $(MAKE) CFLAGS_EXTRA="-Os -fsanitize=address" check - -test-leak: - $(MAKE) clean && $(MAKE) CFLAGS_EXTRA="-fsanitize=leak" check - -mdbx_example: mdbx.h example/example-mdbx.c libmdbx.$(SO_SUFFIX) - $(CC) $(CFLAGS) -I. example/example-mdbx.c ./libmdbx.$(SO_SUFFIX) -o $@ - -build-test: all mdbx_example mdbx_test - -define test-rule -$(patsubst %.cc,%.o,$(1)): $(1) $(TEST_INC) mdbx.h $(lastword $(MAKEFILE_LIST)) - $(CXX) $(CXXFLAGS) $(MDBX_OPTIONS) -c $(1) -o $$@ +mdbx++-dylib.o: config.h mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST)) + $(CXX) $(CXXFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c mdbx.c++ -o $@ -endef -$(foreach file,$(TEST_SRC),$(eval $(call test-rule,$(file)))) +mdbx++-static.o: config.h mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST)) + $(CXX) $(CXXFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c mdbx.c++ -o $@ -mdbx_%: src/mdbx_%.c libmdbx.a +mdbx_%: mdbx_%.c libmdbx.a $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) $(LIBS) -o $@ -mdbx_test: $(TEST_OBJ) libmdbx.$(SO_SUFFIX) - $(CXX) $(CXXFLAGS) $(TEST_OBJ) -Wl,-rpath . -L . -l mdbx $(EXE_LDFLAGS) $(LIBS) -o $@ - -git_DIR := $(shell if [ -d .git ]; then echo .git; elif [ -s .git -a -f .git ]; then grep '^gitdir: ' .git | cut -d ':' -f 2; else echo "Please use libmdbx as a git-submodule or the amalgamated source code" >&2 && echo git_directory; fi) - -src/version.c: src/version.c.in $(lastword $(MAKEFILE_LIST)) $(git_DIR)/HEAD $(git_DIR)/index $(git_DIR)/refs/tags - sed \ - -e "s|@MDBX_GIT_TIMESTAMP@|$(MDBX_GIT_TIMESTAMP)|" \ - -e "s|@MDBX_GIT_TREE@|$(shell git show --no-patch --format=%T HEAD || echo 'Please install latest get version')|" \ - -e "s|@MDBX_GIT_COMMIT@|$(shell git show --no-patch --format=%H HEAD || echo 'Please install latest get version')|" \ - -e "s|@MDBX_GIT_DESCRIBE@|$(MDBX_GIT_DESCRIBE)|" \ - -e "s|\$${MDBX_VERSION_MAJOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 1)|" \ - -e "s|\$${MDBX_VERSION_MINOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 2)|" \ - -e "s|\$${MDBX_VERSION_RELEASE}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 3)|" \ - -e "s|\$${MDBX_VERSION_REVISION}|$(MDBX_GIT_REVISION)|" \ - src/version.c.in > $@ - -src/config.h: src/version.c $(lastword $(MAKEFILE_LIST)) - (echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \ - && echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS) $(LIBS)"' \ - && echo '#define MDBX_BUILD_COMPILER "$(shell (LC_ALL=C $(CC) --version || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \ - && echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; (LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || (LC_ALL=C $(CC) --version | grep -qi e2k && echo E2K) || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \ - && echo '#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)' \ - ) > $@ - -mdbx-dylib.o: src/config.h src/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST)) - $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c src/alloy.c -o $@ - -mdbx-static.o: src/config.h src/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST)) - $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c src/alloy.c -o $@ - -.PHONY: dist release-assets -dist: libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz $(lastword $(MAKEFILE_LIST)) - -release-assets: libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz libmdbx-sources-$(MDBX_VERSION_SUFFIX).zip - -libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz: $(addprefix dist/, $(DIST_SRC) $(DIST_EXTRA)) - $(TAR) -c $(shell LC_ALL=C $(TAR) --help | grep -q -- '--owner' && echo '--owner=0 --group=0') -f - -C dist $(DIST_SRC) $(DIST_EXTRA) | gzip -c -9 > $@ \ - && rm dist/@tmp-shared_internals.inc - -libmdbx-sources-$(MDBX_VERSION_SUFFIX).zip: $(addprefix dist/, $(DIST_SRC) $(DIST_EXTRA)) - rm -rf $@ && (cd dist && $(ZIP) -9 ../$@ $(DIST_SRC) $(DIST_EXTRA)) || rm -rf $@ - -dist/mdbx.h: mdbx.h src/version.c $(lastword $(MAKEFILE_LIST)) - mkdir -p dist && cp $< $@ - -dist/@tmp-shared_internals.inc: src/version.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST)) - mkdir -p dist && sed \ - -e 's|#pragma once|#define MDBX_ALLOY 1\n#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)|' \ - -e 's|#include "../mdbx.h"|@INCLUDE "mdbx.h"|' \ - -e '/#include "defs.h"/r src/defs.h' \ - -e '/#include "osal.h"/r src/osal.h' \ - -e '/#include "options.h"/r src/options.h' \ - src/internals.h > $@ - -dist/mdbx.c: dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST)) - mkdir -p dist && (cat dist/@tmp-shared_internals.inc \ - && cat src/core.c src/osal.c src/version.c src/lck-windows.c src/lck-posix.c \ - ) | grep -v -e '#include "' -e '#pragma once' | sed 's|@INCLUDE|#include|' > $@ - -define dist-tool-rule -dist/$(1).c: src/$(1).c src/wingetopt.h src/wingetopt.c \ - dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST)) - mkdir -p dist && sed \ - -e '/#include "internals.h"/r dist/@tmp-shared_internals.inc' \ - -e '/#include "wingetopt.h"/r src/wingetopt.c' \ - src/$(1).c \ - | grep -v -e '#include "' -e '#pragma once' -e '#define MDBX_ALLOY' \ - | sed 's|@INCLUDE|#include|' > $$@ - -endef -$(foreach file,$(TOOLS),$(eval $(call dist-tool-rule,$(file)))) - -define dist-extra-rule -dist/$(1): $(1) - mkdir -p $$(dir $$@) && sed -e '/^#> dist-cutoff-begin/,/^#< dist-cutoff-end/d' $$< > $$@ - -endef -$(foreach file,$(filter-out man1/% VERSION %.in ntdll.def,$(DIST_EXTRA)),$(eval $(call dist-extra-rule,$(file)))) - -dist/VERSION: src/version.c - mkdir -p dist/ && echo "$(MDBX_GIT_VERSION).$(MDBX_GIT_REVISION)" > $@ - -dist/ntdll.def: src/ntdll.def - mkdir -p dist/cmake/ && cp $< $@ - -dist/config.h.in: src/config.h.in - mkdir -p dist/cmake/ && cp $< $@ - -dist/man1/mdbx_%.1: src/man1/mdbx_%.1 - mkdir -p dist/man1/ && cp $< $@ - -endif - -################################################################################ -# Cross-compilation simple test - -CROSS_LIST = mips-linux-gnu-gcc \ - powerpc64-linux-gnu-gcc powerpc-linux-gnu-gcc \ - arm-linux-gnueabihf-gcc aarch64-linux-gnu-gcc \ - sh4-linux-gnu-gcc mips64-linux-gnuabi64-gcc - -# hppa-linux-gnu-gcc - don't supported by current qemu release -# s390x-linux-gnu-gcc - qemu troubles (hang/abort) -# sparc64-linux-gnu-gcc - qemu troubles (fcntl for F_SETLK/F_GETLK) -# alpha-linux-gnu-gcc - qemu (or gcc) troubles (coredump) - -CROSS_LIST_NOQEMU = hppa-linux-gnu-gcc s390x-linux-gnu-gcc \ - sparc64-linux-gnu-gcc alpha-linux-gnu-gcc - -cross-gcc: - @echo "CORRESPONDING CROSS-COMPILERs ARE REQUIRED." - @echo "FOR INSTANCE: apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu" - @for CC in $(CROSS_LIST_NOQEMU) $(CROSS_LIST); do \ - echo "===================== $$CC"; \ - $(MAKE) clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) all || exit $$?; \ - done - -# Unfortunately qemu don't provide robust support for futexes. -# Therefore it is impossible to run full multi-process tests. -cross-qemu: - @echo "CORRESPONDING CROSS-COMPILERs AND QEMUs ARE REQUIRED." - @echo "FOR INSTANCE: " - @echo " 1) apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu" - @echo " 2) apt install binfmt-support qemu-user-static qemu-user qemu-system-arm qemu-system-mips qemu-system-misc qemu-system-ppc qemu-system-sparc" - @for CC in $(CROSS_LIST); do \ - echo "===================== $$CC + qemu"; \ - $(MAKE) clean && \ - CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static MDBX_OPTIONS="-DMDBX_SAFE4QEMU $(MDBX_OPTIONS)" \ - $(MAKE) test-singleprocess || exit $$?; \ - done - -#< dist-cutoff-end install: $(LIBRARIES) $(TOOLS) $(HEADERS) install -D -p -s -t $(DESTDIR)$(prefix)/bin$(suffix) $(TOOLS) && \ install -D -p -s -t $(DESTDIR)$(prefix)/lib$(suffix) $(filter-out libmdbx.a,$(LIBRARIES)) && \ @@ -363,6 +130,7 @@ IOARENA ?= $(shell \ (test -x ../../@BUILD/src/ioarena && echo ../../@BUILD/src/ioarena) || \ (test -x ../../src/ioarena && echo ../../src/ioarena) || which ioarena) NN ?= 25000000 +BENCH_CRUD_MODE ?= nosync ifneq ($(wildcard $(IOARENA)),) @@ -376,10 +144,10 @@ re-bench: clean-bench bench define bench-rule bench-$(1)_$(2).txt: $(3) $(IOARENA) $(lastword $(MAKEFILE_LIST)) LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \ - $(IOARENA) -D $(1) -B crud -m nosync -n $(2) \ + $(IOARENA) -D $(1) -B crud -m $(BENCH_CRUD_MODE) -n $(2) \ | tee $$@ | grep throughput && \ LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \ - $(IOARENA) -D $(1) -B get,iterate -m sync -r 4 -n $(2) \ + $(IOARENA) -D $(1) -B get,iterate -m $(BENCH_CRUD_MODE) -r 4 -n $(2) \ | tee -a $$@ | grep throughput \ || mv -f $$@ $$@.error @@ -408,5 +176,7 @@ bench: bench-mdbx_$(NN).txt bench-debug: bench-debug_10.txt bench-quartet: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt bench-rocksdb_$(NN).txt bench-wiredtiger_$(NN).txt +bench-triplet: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt bench-sqlite3_$(NN).txt +bench-couple: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt endif diff --git a/libs/libmdbx/src/Makefile b/libs/libmdbx/src/Makefile index 1898655ec2..bb4aa0531f 100644 --- a/libs/libmdbx/src/Makefile +++ b/libs/libmdbx/src/Makefile @@ -1,5 +1,7 @@ -all bench bench-quartet build-test check clean clean-bench cross-gcc cross-qemu dist gcc-analyzer install mdbx memcheck reformat release-assets strip test test-asan test-fault test-leak test-singleprocess test-ubsan test-valgrind tools: +# This is thunk-Makefile for calling GNU Make 3.80 or above + +all bench bench-quartet build-test check clean clean-bench cross-gcc cross-qemu dist doxygen gcc-analyzer install mdbx memcheck reformat release-assets strip test test-asan test-fault test-leak test-singleprocess test-ubsan test-valgrind tools: @CC=$(CC) \ CXX=`if test -n "$(CXX)" && which "$(CXX)" > /dev/null; then echo "$(CXX)"; elif test -n "$(CCC)" && which "$(CCC)" > /dev/null; then echo "$(CCC)"; else echo "c++"; fi` \ - `which gmake || which gnumake || echo 'echo "GNU Make is required"; exit 2;'` \ + `which gmake || which gnumake || echo 'echo "GNU Make 3.80 or above is required"; exit 2;'` \ $(MAKEFLAGS) -f GNUmakefile $@ diff --git a/libs/libmdbx/src/README.md b/libs/libmdbx/src/README.md index 2191c563fa..90f2924608 100644 --- a/libs/libmdbx/src/README.md +++ b/libs/libmdbx/src/README.md @@ -1,13 +1,19 @@ <!-- Required extensions: pymdownx.betterem, pymdownx.tilde, pymdownx.emoji, pymdownx.tasklist, pymdownx.superfences --> libmdbx -======= +======== +> Please refer to the online [documentation](https://erthink.github.io/libmdbx/) +> with [`C` API description](https://erthink.github.io/libmdbx/group__c__api.html) +> and pay attention to the preliminary [`C++` API](https://github.com/erthink/libmdbx/blob/devel/mdbx.h%2B%2B). +> Questions, feedback and suggestions are welcome to the [Telegram' group](https://t.me/libmdbx). + +<!-- section-begin overview --> _libmdbx_ is an extremely fast, compact, powerful, embedded, -transactional [key-value store](https://en.wikipedia.org/wiki/Key-value_database) -database, with [permissive license](LICENSE). -_MDBX_ has a specific set of properties and capabilities, -focused on creating unique lightweight solutions with extraordinary performance. +transactional [key-value database](https://en.wikipedia.org/wiki/Key-value_database), +with [permissive license](./LICENSE). +_libmdbx_ has a specific set of properties and capabilities, +focused on creating unique lightweight solutions. 1. Allows **a swarm of multi-threaded processes to [ACID]((https://en.wikipedia.org/wiki/ACID))ly read and update** several @@ -24,9 +30,9 @@ tree](https://en.wikipedia.org/wiki/B%2B_tree). [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging), but that might be a caveat for write-intensive workloads with durability requirements. -4. **Compact and friendly for fully embedding**. Only 25KLOC of `C11`, -64K x86 binary code, no internal threads neither processes, but -implements a simplified variant of the [Berkeley +4. **Compact and friendly for fully embedding**. Only ≈25KLOC of `C11`, +≈64K x86 binary code of core, no internal threads neither server process(es), +but implements a simplified variant of the [Berkeley DB](https://en.wikipedia.org/wiki/Berkeley_DB) and [dbm](https://en.wikipedia.org/wiki/DBM_(computing)) API. @@ -43,13 +49,15 @@ neglected in favour of write performance. 7. Supports Linux, Windows, MacOS, Android, iOS, FreeBSD, DragonFly, Solaris, OpenSolaris, OpenIndiana, NetBSD, OpenBSD and other systems compliant with **POSIX.1-2008**. +<!-- section-end --> -Historically, _MDBX_ is a deeply revised and extended descendant of the amazing +Historically, _libmdbx_ is a deeply revised and extended descendant of the amazing [Lightning Memory-Mapped Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database). -_MDBX_ inherits all benefits from _LMDB_, but resolves some issues and adds [a set of improvements](#improvements-beyond-lmdb). +_libmdbx_ inherits all benefits from _LMDB_, but resolves some issues and adds [a set of improvements](#improvements-beyond-lmdb). +<!-- section-begin mithril --> The next version is under active non-public development from scratch and will be -released as **_MithrilDB_** and `libmithrildb` for libraries & packages. +released as _**MithrilDB**_ and `libmithrildb` for libraries & packages. Admittedly mythical [Mithril](https://en.wikipedia.org/wiki/Mithril) is resembling silver but being stronger and lighter than steel. Therefore _MithrilDB_ is a rightly relevant name. @@ -58,6 +66,7 @@ _MithrilDB_ is a rightly relevant name. > License](https://www.apache.org/licenses/LICENSE-2.0). The goal of this > revolution is to provide a clearer and robust API, add more features and > new valuable properties of the database. +<!-- section-end --> [](https://t.me/libmdbx) [](https://travis-ci.org/erthink/libmdbx) @@ -71,10 +80,10 @@ _MithrilDB_ is a rightly relevant name. ----- ## Table of Contents -- [Overview](#overview) +- [Characteristics](#characteristics) - [Features](#features) - [Limitations](#limitations) - - [Caveats & Gotchas](#caveats--gotchas) + - [Gotchas](#gotchas) - [Comparison with other databases](#comparison-with-other-databases) - [Improvements beyond LMDB](#improvements-beyond-lmdb) - [History & Acknowledgments](#history) @@ -90,7 +99,9 @@ _MithrilDB_ is a rightly relevant name. - [Async-write mode](#async-write-mode) - [Cost comparison](#cost-comparison) -# Overview +# Characteristics + +<!-- section-begin characteristics --> ## Features @@ -146,30 +157,33 @@ transaction journal. No crash recovery needed. No maintenance is required. - **Database size**: up to `2147483648` pages (8 [TiB](https://en.wikipedia.org/wiki/Tebibyte) for default 4K pagesize, 128 [TiB](https://en.wikipedia.org/wiki/Tebibyte) for 64K pagesize). - **Maximum sub-databases**: `32765`. -## Caveats & Gotchas +## Gotchas 1. There cannot be more than one writer at a time, i.e. no more than one write transaction at a time. -2. MDBX is based on [B+ tree](https://en.wikipedia.org/wiki/B%2B_tree), so access to database pages is mostly random. +2. _libmdbx_ is based on [B+ tree](https://en.wikipedia.org/wiki/B%2B_tree), so access to database pages is mostly random. Thus SSDs provide a significant performance boost over spinning disks for large databases. -3. MDBX uses [shadow paging](https://en.wikipedia.org/wiki/Shadow_paging) instead of [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging). Thus syncing data to disk might be a bottleneck for write intensive workload. +3. _libmdbx_ uses [shadow paging](https://en.wikipedia.org/wiki/Shadow_paging) instead of [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging). Thus syncing data to disk might be a bottleneck for write intensive workload. -4. MDBX uses [copy-on-write](https://en.wikipedia.org/wiki/Copy-on-write) for [snapshot isolation](https://en.wikipedia.org/wiki/Snapshot_isolation) during updates, but read transactions prevents recycling an old retired/freed pages, since it read ones. Thus altering of data during a parallel +4. _libmdbx_ uses [copy-on-write](https://en.wikipedia.org/wiki/Copy-on-write) for [snapshot isolation](https://en.wikipedia.org/wiki/Snapshot_isolation) during updates, but read transactions prevents recycling an old retired/freed pages, since it read ones. Thus altering of data during a parallel long-lived read operation will increase the process work set, may exhaust entire free database space, the database can grow quickly, and result in performance degradation. Try to avoid long running read transactions. -5. MDBX is extraordinarily fast and provides minimal overhead for data access, +5. _libmdbx_ is extraordinarily fast and provides minimal overhead for data access, so you should reconsider using brute force techniques and double check your code. -On the one hand, in the case of MDBX, a simple linear search may be more profitable than complex indexes. +On the one hand, in the case of _libmdbx_, a simple linear search may be more profitable than complex indexes. On the other hand, if you make something suboptimally, you can notice detrimentally only on sufficiently large data. -### Comparison with other databases +## Comparison with other databases For now please refer to [chapter of "BoltDB comparison with other databases"](https://github.com/coreos/bbolt#comparison-with-other-databases) which is also (mostly) applicable to _libmdbx_. +<!-- section-end --> +<!-- section-begin improvements --> + Improvements beyond LMDB ======================== @@ -180,7 +194,7 @@ out-of-the-box, not silently and catastrophically break down. The list below is pruned down to the improvements most notable and obvious from the user's point of view. -### Added Features: +## Added Features 1. Keys could be more than 2 times longer than _LMDB_. > For DB with default page size _libmdbx_ support keys up to 1300 bytes @@ -210,7 +224,7 @@ the user's point of view. > _libmdbx_ database format depends only on the [endianness](https://en.wikipedia.org/wiki/Endianness) but not on the [bitness](https://en.wiktionary.org/wiki/bitness). 6. LIFO policy for Garbage Collection recycling. This can significantly increase write performance due write-back disk cache up to several times in a best case scenario. - > LIFO means that for reuse will be taken the latest becames unused pages. + > LIFO means that for reuse will be taken the latest becomes unused pages. > Therefore the loop of database pages circulation becomes as short as possible. > In other words, the set of pages, that are (over)written in memory and on disk during a series of write transactions, will be as small as possible. > Thus creates ideal conditions for the battery-backed or flash-backed disk cache efficiency. @@ -218,19 +232,20 @@ the user's point of view. 7. Fast estimation of range query result volume, i.e. how many items can be found between a `KEY1` and a `KEY2`. This is a prerequisite for build and/or optimize query execution plans. - > _libmdbx_ performs a rough estimate based on common B-tree pages of the paths from root to corresponding keys. + > _libmdbx_ performs a rough estimate based on common B-tree pages of the paths from root to corresponding keys. -8. `mdbx_chk` tool for database integrity check. +8. `mdbx_chk` utility for database integrity check. +Since version 0.9.1, the utility supports checking the database using any of the three meta pages and the ability to switch to it. 9. Automated steady sync-to-disk upon several thresholds and/or timeout via cheap polling. 10. Sequence generation and three persistent 64-bit markers. -11. Callback for lack-of-space condition of database that allows you to control and/or resolve such situations. +11. Handle-Slow-Readers callback to resolve a database full/overflow issues due to long-lived read transaction(s). 12. Support for opening databases in the exclusive mode, including on a network share. -### Added Abilities: +## Added Abilities 1. Zero-length for keys and values. @@ -248,11 +263,15 @@ pair, to the first, to the last, or not set to anything. > _libmdbx_ allows one _at once_ with getting previous value > and addressing the particular item from multi-value with the same key. -### Other fixes and specifics: +## Other fixes and specifics -1. Fixed more than 10 significant errors, in particular: page leaks, wrong sub-database statistics, segfault in several conditions, nonoptimal page merge strategy, updating an existing record with a change in data size (including for multimap), etc. +1. Fixed more than 10 significant errors, in particular: page leaks, +wrong sub-database statistics, segfault in several conditions, +nonoptimal page merge strategy, updating an existing record with +a change in data size (including for multimap), etc. -2. All cursors can be reused and should be closed explicitly, regardless ones were opened within a write or read transaction. +2. All cursors can be reused and should be closed explicitly, +regardless ones were opened within a write or read transaction. 3. Opening database handles are spared from race conditions and pre-opening is not needed. @@ -260,10 +279,9 @@ pre-opening is not needed. 4. Returning `MDBX_EMULTIVAL` error in case of ambiguous update or delete. 5. Guarantee of database integrity even in asynchronous unordered write-to-disk mode. - > _libmdbx_ propose additional trade-off by implementing append-like manner for updates - > in `MDBX_SAFE_NOSYNC` and `MDBX_WRITEMAP|MDBX_MAPASYNC` modes, that avoid database corruption after a system crash - > contrary to LMDB. Nevertheless, the `MDBX_UTTERLY_NOSYNC` mode is available to match LMDB behaviour, - > and for special use-cases. + > _libmdbx_ propose additional trade-off by `MDBX_SAFE_NOSYNC` with append-like manner for updates, + > that avoids database corruption after a system crash contrary to LMDB. + > Nevertheless, the `MDBX_UTTERLY_NOSYNC` mode is available to match behaviour of the `MDB_NOSYNC` in LMDB. 6. On **MacOS & iOS** the `fcntl(F_FULLFSYNC)` syscall is used _by default_ to synchronize data with the disk, as this is [the only way to @@ -282,7 +300,13 @@ against incompetent user actions (aka _libmdbx_ may be a little lag in performance tests from LMDB where the named mutexes are used. -### History +<!-- section-end --> +<!-- section-begin history --> + +# History + +Historically, _libmdbx_ is a deeply revised and extended descendant of the +[Lightning Memory-Mapped Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database). At first the development was carried out within the [ReOpenLDAP](https://github.com/erthink/ReOpenLDAP) project. About a year later _libmdbx_ was separated into a standalone project, which was @@ -292,18 +316,26 @@ conference](http://www.highload.ru/2015/abstracts/1831.html). Since 2017 _libmdbx_ is used in [Fast Positive Tables](https://github.com/erthink/libfpta), and development is funded by [Positive Technologies](https://www.ptsecurity.com). -### Acknowledgments +## Acknowledgments Howard Chu <hyc@openldap.org> is the author of LMDB, from which -originated the MDBX in 2015. +originated the _libmdbx_ in 2015. Martin Hedenfalk <martin@bzero.se> is the author of `btree.c` code, which was used to begin development of LMDB. +<!-- section-end --> + -------------------------------------------------------------------------------- Usage ===== +<!-- section-begin usage --> +Currently, libmdbx is only available in a +[source code](https://en.wikipedia.org/wiki/Source_code) form. +Packages support for common Linux distributions is planned in the future, +since release the version 1.0. + ## Source code embedding _libmdbx_ provides two official ways for integration in source code form: @@ -316,7 +348,7 @@ _libmdbx_ provides two official ways for integration in source code form: > This allows you to build as _libmdbx_ and testing tool. > On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool. -**_Please, avoid using any other techniques._** Otherwise, at least +_**Please, avoid using any other techniques.**_ Otherwise, at least don't ask for support and don't name such chimeras `libmdbx`. The amalgamated source code could be created from the original clone of git @@ -434,21 +466,30 @@ To build _libmdbx_ for iOS, we recommend using CMake with the "[toolchain file](https://cmake.org/cmake/help/latest/variable/CMAKE_TOOLCHAIN_FILE.html)" from the [ios-cmake](https://github.com/leetal/ios-cmake) project. +<!-- section-end --> + ## API description -For more information and API description see the [mdbx.h](mdbx.h) header. -Please do not hesitate to point out errors in the documentation, -including creating [PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/proposing-changes-to-your-work-with-pull-requests) with corrections and improvements. -## Bindings +Please refer to the online [_libmdbx_ API reference](https://erthink.github.io/libmdbx/) +and/or see the [mdbx.h](mdbx.h) header. + +<!-- section-begin bindings --> + +Bindings +======== - | Runtime | GitHub | Author | - | -------- | ------ | ------ | - | Rust | [mdbx-rs](https://github.com/Kerollmops/mdbx-rs) | [@Kerollmops](https://github.com/Kerollmops) | - | Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) | - | .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) | +| Runtime | GitHub | Author | +| ------- | ------ | ------ | +| Rust | [mdbx-rs](https://github.com/Kerollmops/mdbx-rs) | [Clément Renault](https://github.com/Kerollmops) | +| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) | +| .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) | + +<!-- section-end --> -------------------------------------------------------------------------------- +<!-- section-begin performance --> + Performance comparison ====================== @@ -585,6 +626,8 @@ syscall and by scanning the data directory.  +<!-- section-end --> + -------------------------------------------------------------------------------- #### This is a mirror of the origin repository that was moved to [abf.io](https://abf.io/erthink/) because of discriminatory restrictions for Russian Crimea. diff --git a/libs/libmdbx/src/VERSION b/libs/libmdbx/src/VERSION new file mode 100644 index 0000000000..4cef1f9b06 --- /dev/null +++ b/libs/libmdbx/src/VERSION @@ -0,0 +1 @@ +0.9.1.18 diff --git a/libs/libmdbx/src/appveyor.yml b/libs/libmdbx/src/appveyor.yml deleted file mode 100644 index 66323a6037..0000000000 --- a/libs/libmdbx/src/appveyor.yml +++ /dev/null @@ -1,99 +0,0 @@ -version: 0.8.2.{build} - -environment: - matrix: - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CMAKE_GENERATOR: Visual Studio 16 2019 - TOOLSET: 142 - MDBX_BUILD_SHARED_LIBRARY: OFF - MDBX_AVOID_CRT: OFF - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CMAKE_GENERATOR: Visual Studio 16 2019 - TOOLSET: 142 - MDBX_BUILD_SHARED_LIBRARY: ON - MDBX_AVOID_CRT: ON - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CMAKE_GENERATOR: Visual Studio 16 2019 - TOOLSET: 142 - MDBX_BUILD_SHARED_LIBRARY: OFF - MDBX_AVOID_CRT: ON - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CMAKE_GENERATOR: Visual Studio 16 2019 - TOOLSET: 142 - MDBX_BUILD_SHARED_LIBRARY: ON - MDBX_AVOID_CRT: OFF - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - CMAKE_GENERATOR: Visual Studio 15 2017 - TOOLSET: 141 - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - CMAKE_GENERATOR: Visual Studio 14 2015 - TOOLSET: 140 - -branches: - except: - - coverity_scan - -configuration: -- Debug -- Release - -platform: -- Win32 -- x64 - -before_build: -- git clean -x -f -d -- git submodule sync -- git fetch --tags --prune -- git submodule update --init --recursive -- git submodule foreach --recursive git fetch --tags --prune -- cmake --version - -build_script: -- ps: | - Write-Output "*******************************************************************************" - Write-Output "Configuration: $env:CONFIGURATION" - Write-Output "Platform: $env:PLATFORM" - Write-Output "Toolchain: $env:CMAKE_GENERATOR v$env:TOOLSET" - Write-Output "Options: MDBX_AVOID_CRT=$env:MDBX_AVOID_CRT MDBX_BUILD_SHARED_LIBRARY=$env:MDBX_BUILD_SHARED_LIBRARY" - Write-Output "*******************************************************************************" - - md _build -Force | Out-Null - cd _build - - $generator = $env:CMAKE_GENERATOR - if ($env:TOOLSET -lt 142) { - if ($env:PLATFORM -eq "x64") { - $generator = "$generator Win64" - } - & cmake -G "$generator" -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -D MDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY .. - } else { - & cmake -G "$generator" -A $env:PLATFORM -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -DMDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY .. - } - if ($LastExitCode -ne 0) { - throw "Exec: $ErrorMessage" - } - Write-Output "*******************************************************************************" - - & cmake --build . --config $env:CONFIGURATION - if ($LastExitCode -ne 0) { - throw "Exec: $ErrorMessage" - } - Write-Output "*******************************************************************************" - -test_script: -- ps: | - if (($env:PLATFORM -ne "ARM") -and ($env:PLATFORM -ne "ARM64")) { - & ./$env:CONFIGURATION/mdbx_test.exe --progress --console=no --pathname=test.db --dont-cleanup-after basic > test.log - Get-Content test.log | Select-Object -last 42 - if ($LastExitCode -ne 0) { - throw "Exec: $ErrorMessage" - } else { - & ./$env:CONFIGURATION/mdbx_chk.exe -nvv test.db | Tee-Object -file chk.log | Select-Object -last 42 - } - } - -on_failure: -- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.log -- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.db -- ps: Push-AppveyorArtifact \projects\libmdbx\_build\chk.log diff --git a/libs/libmdbx/src/cmake/compiler.cmake b/libs/libmdbx/src/cmake/compiler.cmake index e13225d63d..ab72b897d1 100644 --- a/libs/libmdbx/src/cmake/compiler.cmake +++ b/libs/libmdbx/src/cmake/compiler.cmake @@ -231,7 +231,7 @@ else() check_compiler_flag("-ggdb" CC_HAS_GGDB) check_compiler_flag("-fvisibility=hidden" CC_HAS_VISIBILITY) check_compiler_flag("-march=native" CC_HAS_ARCH_NATIVE) - check_compiler_flag("-Og" CC_HAS_DEBUG_FRENDLY_OPTIMIZATION) + check_compiler_flag("-Og" CC_HAS_DEBUG_FRIENDLY_OPTIMIZATION) check_compiler_flag("-Wall" CC_HAS_WALL) check_compiler_flag("-Ominimal" CC_HAS_OMINIMAL) check_compiler_flag("-ffunction-sections -fdata-sections" CC_HAS_SECTIONS) @@ -606,8 +606,9 @@ macro(setup_compile_flags) if(MSVC_VERSION LESS 1900) message(FATAL_ERROR "At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.") endif() - add_compile_flags("CXX" "/Zc:__cplusplus") - add_compile_flags("C;CXX" "/W4") + if(NOT MSVC_VERSION LESS 1910) + add_compile_flags("CXX" "/Zc:__cplusplus") + endif() add_compile_flags("C;CXX" "/utf-8") else() if(CC_HAS_WALL) @@ -771,14 +772,14 @@ endmacro(setup_compile_flags) if(CMAKE_CXX_COMPILER_LOADED) # determine library for for std::filesystem set(LIBCXX_FILESYSTEM "") - if(CMAKE_COMPILER_IS_GNUCXX) - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) - set(LIBCXX_FILESYSTEM "stdc++fs") - endif() - elseif(CMAKE_COMPILER_IS_CLANG) + if(CMAKE_COMPILER_IS_CLANG) if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) set(LIBCXX_FILESYSTEM "c++experimental") - else() + elseif(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) + set(LIBCXX_FILESYSTEM "stdc++fs") + endif() + elseif(CMAKE_COMPILER_IS_GNUCXX) + if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.3 AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) set(LIBCXX_FILESYSTEM "stdc++fs") endif() endif() diff --git a/libs/libmdbx/src/cmake/utils.cmake b/libs/libmdbx/src/cmake/utils.cmake index 8e98267f4e..dc6a240e1b 100644 --- a/libs/libmdbx/src/cmake/utils.cmake +++ b/libs/libmdbx/src/cmake/utils.cmake @@ -155,7 +155,7 @@ macro(fetch_version name source_root_directory parent_scope) if(NOT ${name}_GIT_VERSION OR NOT ${name}_GIT_TIMESTAMP OR ${name}_GIT_REVISION STREQUAL "") if(GIT AND EXISTS "${source_root_directory}/.git") - message(WARNING "Unable to retrive ${name} version from git.") + message(WARNING "Unable to retrieve ${name} version from git.") endif() set(${name}_GIT_VERSION "0;0;0;0") set(${name}_GIT_TIMESTAMP "") @@ -168,7 +168,7 @@ macro(fetch_version name source_root_directory parent_scope) endif() if(NOT ${name}_VERSION) - message(WARNING "Unable to retrive ${name} version from \"${version_file}\" file.") + message(WARNING "Unable to retrieve ${name} version from \"${version_file}\" file.") set(${name}_VERSION_LIST ${${name}_GIT_VERSION}) string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}") else() diff --git a/libs/libmdbx/src/config.h b/libs/libmdbx/src/config.h deleted file mode 100644 index 23fea98bd0..0000000000 --- a/libs/libmdbx/src/config.h +++ /dev/null @@ -1,53 +0,0 @@ -/* This is CMake-template for libmdbx's config.h - ******************************************************************************/ - -/* *INDENT-OFF* */ -/* clang-format off */ - -#define LTO_ENABLED -/* #undef MDBX_USE_VALGRIND */ -/* #undef ENABLE_GPROF */ -/* #undef ENABLE_GCOV */ -/* #undef ENABLE_ASAN */ -/* #undef MDBX_FORCE_ASSERTIONS */ - -/* Common */ -#define MDBX_HUGE_TRANSACTIONS 0 -#define MDBX_TXN_CHECKOWNER 1 -#define MDBX_TXN_CHECKPID_AUTO -#ifndef MDBX_TXN_CHECKPID_AUTO -#define MDBX_TXN_CHECKPID 0 -#endif -#define MDBX_LOCKING_AUTO -#ifndef MDBX_LOCKING_AUTO -/* #undef MDBX_LOCKING */ -#endif -#define MDBX_TRUST_RTC_AUTO -#ifndef MDBX_TRUST_RTC_AUTO -#define MDBX_TRUST_RTC 0 -#endif - -/* Windows */ -#define MDBX_CONFIG_MANUAL_TLS_CALLBACK 1 -#define MDBX_AVOID_CRT 0 - -/* MacOS & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY 0 - -/* POSIX */ -#define MDBX_DISABLE_GNU_SOURCE 0 -/* #undef MDBX_USE_OFDLOCKS_AUTO */ -#ifndef MDBX_USE_OFDLOCKS_AUTO -#define MDBX_USE_OFDLOCKS 0 -#endif - -/* Build Info */ -#define MDBX_BUILD_TIMESTAMP "2020-06-14T10:50:19Z" -#define MDBX_BUILD_TARGET "x86_64-Windows" -/* #undef MDBX_BUILD_TYPE */ -#define MDBX_BUILD_COMPILER "MSVC-19.26.28806.0" -#define MDBX_BUILD_FLAGS "/DWIN32 /D_WINDOWS /W3 /EHsc /Gy /W4 /utf-8 /WX /GL MDBX_BUILD_SHARED_LIBRARY=0" -#define MDBX_BUILD_SOURCERY d9a19ecb1e6bd3a67910c162b2e9c2dc476d665c134d041af742fa0196b19a3e_v0_8_1_1_g0c5496d - -/* *INDENT-ON* */ -/* clang-format on */ diff --git a/libs/libmdbx/src/src/config.h.in b/libs/libmdbx/src/config.h.in index 871944e115..5048d4e021 100644 --- a/libs/libmdbx/src/src/config.h.in +++ b/libs/libmdbx/src/config.h.in @@ -14,9 +14,9 @@ /* Common */ #cmakedefine01 MDBX_HUGE_TRANSACTIONS #cmakedefine01 MDBX_TXN_CHECKOWNER -#cmakedefine MDBX_TXN_CHECKPID_AUTO -#ifndef MDBX_TXN_CHECKPID_AUTO -#cmakedefine01 MDBX_TXN_CHECKPID +#cmakedefine MDBX_ENV_CHECKPID_AUTO +#ifndef MDBX_ENV_CHECKPID_AUTO +#cmakedefine01 MDBX_ENV_CHECKPID #endif #cmakedefine MDBX_LOCKING_AUTO #ifndef MDBX_LOCKING_AUTO @@ -42,11 +42,21 @@ #endif /* Build Info */ +#ifndef MDBX_BUILD_TIMESTAMP #cmakedefine MDBX_BUILD_TIMESTAMP "@MDBX_BUILD_TIMESTAMP@" +#endif +#ifndef MDBX_BUILD_TARGET #cmakedefine MDBX_BUILD_TARGET "@MDBX_BUILD_TARGET@" +#endif +#ifndef MDBX_BUILD_TYPE #cmakedefine MDBX_BUILD_TYPE "@MDBX_BUILD_TYPE@" +#endif +#ifndef MDBX_BUILD_COMPILER #cmakedefine MDBX_BUILD_COMPILER "@MDBX_BUILD_COMPILER@" +#endif +#ifndef MDBX_BUILD_FLAGS #cmakedefine MDBX_BUILD_FLAGS "@MDBX_BUILD_FLAGS@" +#endif #cmakedefine MDBX_BUILD_SOURCERY @MDBX_BUILD_SOURCERY@ /* *INDENT-ON* */ diff --git a/libs/libmdbx/src/example/CMakeLists.txt b/libs/libmdbx/src/example/CMakeLists.txt deleted file mode 100644 index d3e56e82e9..0000000000 --- a/libs/libmdbx/src/example/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(TARGET mdbx_example) -project(${TARGET}) - -add_executable(${TARGET} example-mdbx.c) - -target_link_libraries(${TARGET} mdbx) diff --git a/libs/libmdbx/src/example/README.md b/libs/libmdbx/src/example/README.md deleted file mode 100644 index b819cf4a40..0000000000 --- a/libs/libmdbx/src/example/README.md +++ /dev/null @@ -1 +0,0 @@ -See [example-mdbx.c](example-mdbx.c) as an example of using _libmdbx_, and do a line-by-line comparison of it with the [sample-bdb.txt](sample-bdb.txt) file. diff --git a/libs/libmdbx/src/example/example-mdbx.c b/libs/libmdbx/src/example/example-mdbx.c deleted file mode 100644 index 0a9e7ed601..0000000000 --- a/libs/libmdbx/src/example/example-mdbx.c +++ /dev/null @@ -1,112 +0,0 @@ -/* MDBX usage examle - * - * Do a line-by-line comparison of this and sample-bdb.txt - */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>. - * Copyright 2017 Ilya Shipitsin <chipitsine@gmail.com>. - * Copyright 2012-2015 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "mdbx.h" -#include <stdio.h> -#include <stdlib.h> - -int main(int argc, char *argv[]) { - (void)argc; - (void)argv; - - int rc; - MDBX_env *env = NULL; - MDBX_dbi dbi = 0; - MDBX_val key, data; - MDBX_txn *txn = NULL; - MDBX_cursor *cursor = NULL; - char sval[32]; - - rc = mdbx_env_create(&env); - if (rc != MDBX_SUCCESS) { - fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - rc = mdbx_env_open(env, "./example-db", - MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664); - if (rc != MDBX_SUCCESS) { - fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - rc = mdbx_txn_begin(env, NULL, 0, &txn); - if (rc != MDBX_SUCCESS) { - fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - rc = mdbx_dbi_open(txn, NULL, 0, &dbi); - if (rc != MDBX_SUCCESS) { - fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - key.iov_len = sizeof(int); - key.iov_base = sval; - data.iov_len = sizeof(sval); - data.iov_base = sval; - - sprintf(sval, "%03x %d foo bar", 32, 3141592); - rc = mdbx_put(txn, dbi, &key, &data, 0); - if (rc != MDBX_SUCCESS) { - fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - rc = mdbx_txn_commit(txn); - if (rc) { - fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - txn = NULL; - - rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); - if (rc != MDBX_SUCCESS) { - fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - rc = mdbx_cursor_open(txn, dbi, &cursor); - if (rc != MDBX_SUCCESS) { - fprintf(stderr, "mdbx_cursor_open: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - int found = 0; - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len, - (char *)key.iov_base, data.iov_base, (int)data.iov_len, - (char *)data.iov_base); - found += 1; - } - if (rc != MDBX_NOTFOUND || found == 0) { - fprintf(stderr, "mdbx_cursor_get: (%d) %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } else { - rc = MDBX_SUCCESS; - } -bailout: - if (cursor) - mdbx_cursor_close(cursor); - if (txn) - mdbx_txn_abort(txn); - if (dbi) - mdbx_dbi_close(env, dbi); - if (env) - mdbx_env_close(env); - return (rc != MDBX_SUCCESS) ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/libs/libmdbx/src/example/sample-bdb.txt b/libs/libmdbx/src/example/sample-bdb.txt deleted file mode 100644 index 0f5e190a5e..0000000000 --- a/libs/libmdbx/src/example/sample-bdb.txt +++ /dev/null @@ -1,77 +0,0 @@ -/* BerkeleyDB toy/sample - * - * Do a line-by-line comparison of this and example-mdbx.c - */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>. - * Copyright 2012-2015 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include <stdio.h> -#include <string.h> -#include <db.h> - -int main(int argc,char * argv[]) -{ - int rc; - DB_ENV *env; - DB *dbi; - DBT key, data; - DB_TXN *txn; - DBC *cursor; - char sval[32], kval[32]; - - /* Note: Most error checking omitted for simplicity */ - -#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD) - rc = db_env_create(&env, 0); - rc = env->open(env, "./testdb", FLAGS, 0664); - rc = db_create(&dbi, env, 0); - rc = env->txn_begin(env, NULL, &txn, 0); - rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664); - - memset(&key, 0, sizeof(DBT)); - memset(&data, 0, sizeof(DBT)); - key.size = sizeof(int); - key.data = sval; - data.size = sizeof(sval); - data.data = sval; - - sprintf(sval, "%03x %d foo bar", 32, 3141592); - rc = dbi->put(dbi, txn, &key, &data, 0); - rc = txn->commit(txn, 0); - if (rc) { - fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc)); - goto leave; - } - rc = env->txn_begin(env, NULL, &txn, 0); - rc = dbi->cursor(dbi, txn, &cursor, 0); - key.flags = DB_DBT_USERMEM; - key.data = kval; - key.ulen = sizeof(kval); - data.flags = DB_DBT_USERMEM; - data.data = sval; - data.ulen = sizeof(sval); - while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", - key.data, (int) key.size, (char *) key.data, - data.data, (int) data.size, (char *) data.data); - } - rc = cursor->c_close(cursor); - rc = txn->abort(txn); -leave: - rc = dbi->close(dbi, 0); - rc = env->close(env, 0); - return rc; -} diff --git a/libs/libmdbx/src/src/man1/mdbx_chk.1 b/libs/libmdbx/src/man1/mdbx_chk.1 index d6292f3384..0348f90b06 100644 --- a/libs/libmdbx/src/src/man1/mdbx_chk.1 +++ b/libs/libmdbx/src/man1/mdbx_chk.1 @@ -1,6 +1,6 @@ .\" Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_CHK 1 "2020-06-05" "MDBX 0.8.x" +.TH MDBX_CHK 1 "2020-09-25" "MDBX 0.9.1" .SH NAME mdbx_chk \- MDBX checking tool .SH SYNOPSIS @@ -44,9 +44,6 @@ during B-tree traversal and basic info of each GC record. If \fB\-vvvvv\fP is given, turn maximal verbosity, display the full list of page IDs in the GC records and size of each key-value pair of database(s). .TP -.BR \-n -Open MDBX environment(s) which do not use subdirectories. -.TP .BR \-q Be quiet; do not output anything even if an error was detected. .TP @@ -74,6 +71,20 @@ comparator(s) was used. .TP .BR \-s \ subdb Verify and show info only for a specific subdatabase. +.TP +.BR \-0 | \-1 | \-2 +Using specific meta-page 0, or 2 for checking. +.TP +.BR \-t +Turn to a specified meta-page on successful check. +.TP +.BR \-T +Turn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK! +.TP +.BR \-n +Open MDBX environment(s) which do not use subdirectories. +This is legacy option. For now MDBX handles this automatically. + .SH DIAGNOSTICS Exit status is zero if no errors occur. Errors result in a non-zero exit status and a diagnostic message being written to standard error diff --git a/libs/libmdbx/src/src/man1/mdbx_copy.1 b/libs/libmdbx/src/man1/mdbx_copy.1 index 6c587abdc3..83fbc904b9 100644 --- a/libs/libmdbx/src/src/man1/mdbx_copy.1 +++ b/libs/libmdbx/src/man1/mdbx_copy.1 @@ -2,7 +2,7 @@ .\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_COPY 1 "2020-06-05" "MDBX 0.8.x" +.TH MDBX_COPY 1 "2020-09-25" "MDBX 0.9.1" .SH NAME mdbx_copy \- MDBX environment copy tool .SH SYNOPSIS @@ -47,6 +47,7 @@ Currently it fails if the environment has suffered a page leak. .TP .BR \-n Open MDBX environment(s) which do not use subdirectories. +This is legacy option. For now MDBX handles this automatically. .SH DIAGNOSTICS Exit status is zero if no errors occur. diff --git a/libs/libmdbx/src/src/man1/mdbx_dump.1 b/libs/libmdbx/src/man1/mdbx_dump.1 index 698b7cf414..bdb4d55366 100644 --- a/libs/libmdbx/src/src/man1/mdbx_dump.1 +++ b/libs/libmdbx/src/man1/mdbx_dump.1 @@ -2,7 +2,7 @@ .\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_DUMP 1 "2020-06-05" "MDBX 0.8.x" +.TH MDBX_DUMP 1 "2020-09-25" "MDBX 0.9.1" .SH NAME mdbx_dump \- MDBX environment export tool .SH SYNOPSIS @@ -68,6 +68,8 @@ Rescure mode. Ignore some errors to dump corrupted DB. .TP .BR \-n Dump an MDBX database which does not use subdirectories. +This is legacy option. For now MDBX handles this automatically. + .SH DIAGNOSTICS Exit status is zero if no errors occur. Errors result in a non-zero exit status and diff --git a/libs/libmdbx/src/src/man1/mdbx_load.1 b/libs/libmdbx/src/man1/mdbx_load.1 index bc3d737e17..950ae0283e 100644 --- a/libs/libmdbx/src/src/man1/mdbx_load.1 +++ b/libs/libmdbx/src/man1/mdbx_load.1 @@ -2,7 +2,7 @@ .\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_LOAD 1 "2020-06-05" "MDBX 0.8.x" +.TH MDBX_LOAD 1 "2020-09-25" "MDBX 0.9.1" .SH NAME mdbx_load \- MDBX environment import tool .SH SYNOPSIS @@ -40,6 +40,19 @@ must be in the output format specified by the utility or as specified by the .B -T option below. + +A simple escape mechanism, where newline and backslash (\\) characters are special, is +applied to the text input. Newline characters are interpreted as record separators. +Backslash characters in the text will be interpreted in one of two ways: If the backslash +character precedes another backslash character, the pair will be interpreted as a literal +backslash. If the backslash character precedes any other character, the two characters +following the backslash will be interpreted as a hexadecimal specification of a single +character; for example, \\0a is a newline character in the ASCII character set. + +For this reason, any backslash or newline characters that naturally occur in the text +input must be escaped to avoid misinterpretation by +.BR mdbx_load . + .SH OPTIONS .TP .BR \-V @@ -74,18 +87,7 @@ Rescure mode. Ignore errors to load corrupted DB dump. .TP .BR \-n Load an MDBX database which does not use subdirectories. - -A simple escape mechanism, where newline and backslash (\\) characters are special, is -applied to the text input. Newline characters are interpreted as record separators. -Backslash characters in the text will be interpreted in one of two ways: If the backslash -character precedes another backslash character, the pair will be interpreted as a literal -backslash. If the backslash character precedes any other character, the two characters -following the backslash will be interpreted as a hexadecimal specification of a single -character; for example, \\0a is a newline character in the ASCII character set. - -For this reason, any backslash or newline characters that naturally occur in the text -input must be escaped to avoid misinterpretation by -.BR mdbx_load . +This is legacy option. For now MDBX handles this automatically. .SH DIAGNOSTICS Exit status is zero if no errors occur. diff --git a/libs/libmdbx/src/src/man1/mdbx_stat.1 b/libs/libmdbx/src/man1/mdbx_stat.1 index f510d4e013..ec0ffb1280 100644 --- a/libs/libmdbx/src/src/man1/mdbx_stat.1 +++ b/libs/libmdbx/src/man1/mdbx_stat.1 @@ -2,7 +2,7 @@ .\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_STAT 1 "2020-06-05" "MDBX 0.8.x" +.TH MDBX_STAT 1 "2020-09-25" "MDBX 0.9.1" .SH NAME mdbx_stat \- MDBX environment status tool .SH SYNOPSIS @@ -61,6 +61,9 @@ Display the status of a specific subdatabase. .TP .BR \-n Display the status of an MDBX database which does not use subdirectories. +This is legacy option. For now MDBX handles this automatically +for existing databases, but may be required while creating new. + .SH DIAGNOSTICS Exit status is zero if no errors occur. Errors result in a non-zero exit status and diff --git a/libs/libmdbx/src/src/core.c b/libs/libmdbx/src/mdbx.c index 62d7c8e38c..2fa4734952 100644 --- a/libs/libmdbx/src/src/core.c +++ b/libs/libmdbx/src/mdbx.c @@ -3,6 +3,3033 @@ * and other libmdbx authors: please see AUTHORS file. * All rights reserved. * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#define MDBX_ALLOY 1 +#define MDBX_BUILD_SOURCERY 3b5677a6062b714f1e138b0066c5590ee3c9ebf3bf8cfa3bb9503515ea0d1f02_v0_9_1_18_g1d31ebdc1c +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_FULL_VER < 190024234 + /* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual Studio 2015 Update 3). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for alignment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant aggregate initializer */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ + +#if defined(MDBX_TOOLS) +#undef MDBX_DEPRECATED +#define MDBX_DEPRECATED +#endif /* MDBX_TOOLS */ + +#include "mdbx.h" +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# else +# define __noinline +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(7, 0) && \ + (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +# define __fallthrough __attribute__((__fallthrough__)) +# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ + __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define __fallthrough [[clang::fallthrough]] +# else +# define __fallthrough +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline __maybe_unused const void* MDBX_PURE_FUNCTION + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include <valgrind/memcheck.h> +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include <sanitizer/asan_interface.h> +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template <typename T, size_t N> + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_STATIC_ASSERT) +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif defined(_MSC_VER) +# include <crtdbg.h> +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wnested-anon-types") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wnested-anon-types" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wnested-anon-types" +# else +# pragma warning disable "nested-anon-types" +# endif +#endif /* -Wnested-anon-types */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) && MDBX_AVOID_CRT +#define _NO_CRT_STDIO_INLINE +#endif +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include <assert.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +/* C11 stdalign.h */ +#if __has_include(<stdalign.h>) +#include <stdalign.h> +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif /* Apple OSX & iOS */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) +#include <sys/cdefs.h> +#include <sys/mount.h> +#include <sys/sysctl.h> +#include <sys/types.h> +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <vm/vm_param.h> +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include <uvm/uvm_param.h> +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#ifndef __MACH__ +#include <sys/vmmeter.h> +#endif +#else +#include <malloc.h> +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(_WIN32) || defined(_WIN64)) +#include <mntent.h> +#endif /* !Solaris */ +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) +#include <malloc_np.h> +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) +#include <malloc/malloc.h> +#endif /* MacOS */ + +#if defined(__MACH__) +#include <mach/host_info.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> +#include <uuid/uuid.h> +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include <linux/sysctl.h> +#include <sched.h> +#include <sys/sendfile.h> +#include <sys/statfs.h> +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include <utmpx.h> +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include <kstat.h> +#include <sys/mnttab.h> +/* On Solaris, it's easier to add a missing prototype rather than find a + * combination of #defines that break nothing. */ +__extern_C key_t ftok(const char *, int); +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <tlhelp32.h> +#include <windows.h> +#include <winnt.h> +#include <winternl.h> +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event[2]; +} mdbx_condpair_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <sys/file.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond[2]; +} mdbx_condpair_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +#ifdef __ANDROID_API__ +#include <android/log.h> +#if __ANDROID_API__ >= 21 +#include <sys/sendfile.h> +#endif +#endif /* Android */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) +#include <sys/stat.h> +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) +#include <sys/types.h> +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) +#include <sys/file.h> +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include <intrin.h> +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif /* __ia32__ */ +#if defined(__ia32__) +#include <cpuid.h> +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include <mbarrier.h> +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include <machine/sys/inline.h> +#elif defined(__IBMC__) && defined(__powerpc) +#include <atomic.h> +#elif defined(_AIX) +#include <builtins.h> +#include <sys/atomic_op.h> +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include <c_asm.h> +#include <machine/builtins.h> +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include <machine/inline.h> +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline size_t +mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; +#endif /* Windows */ + +#ifdef __cplusplus +extern void mdbx_osal_jitter(bool tiny); +#else +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +#if __has_include(<sys/cachectl.h>) +#include <sys/cachectl.h> +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include <sys/cachectl.h> +#endif + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC MDBX_PRINTF_ARGS(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +MDBX_INTERNAL_VAR bool + mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; +#endif /* Linux */ + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_NONE = 0, + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + const enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); + +enum mdbx_openfile_purpose { + MDBX_OPEN_DXB_READ = 0, + MDBX_OPEN_DXB_LAZY = 1, + MDBX_OPEN_DXB_DSYNC = 2, + MDBX_OPEN_LCK = 3, + MDBX_OPEN_COPY = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +#define MMAP_OPTION_TRUNCATE 1 +#define MMAP_OPTION_SEMAPHORE 2 +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const unsigned options); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna, const bool may_move); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { + mdbx_tid_t thunk; + STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); +#if defined(_WIN32) || defined(_WIN64) + thunk = GetCurrentThreadId(); +#else + thunk = pthread_self(); +#endif + return (uintptr_t)thunk; +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by argument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (especially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); +MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; + +static __inline bool mdbx_RunningUnderWine(void) { + return !mdbx_NtExtendSection; +} + +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, + LPCSTR lpValue, DWORD dwFlags, + LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData); +MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include <stdatomic.h> +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include <libkern/OSAtomic.h> +#else +#error FIXME atomic-ops +#endif + +#endif /* !__cplusplus */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/******************************************************************************* + ******************************************************************************* + ******************************************************************************* + * + * + * #### ##### ##### # #### # # #### + * # # # # # # # # ## # # + * # # # # # # # # # # # #### + * # # ##### # # # # # # # # + * # # # # # # # # ## # # + * #### # # # #### # # #### + * + * + */ + +/** \defgroup build_option Build options + * The libmdbx build options. + @{ */ + +#ifdef DOXYGEN +/* !!! Actually this is a fake definitions !!! + * !!! for documentation generation by Doxygen !!! */ + +/** Controls enabling of debugging features. + * + * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all, + * including logging and assertion controls. + * Logging level and corresponding debug flags changing + * by \ref mdbx_setup_debug() will not have effect. + * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging, + * assertions checking and internal audit). + * Simultaneously sets the default logging level + * to the `MDBX_DEBUG` value. + * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. + * + * \ingroup build_option */ +#define MDBX_DEBUG 0...7 + +/** Disables using of GNU libc extensions. */ +#define MDBX_DISABLE_GNU_SOURCE 0 or 1 + +#endif /* DOXYGEN */ + +/** Enables support for huge write-transactions */ +#ifndef MDBX_HUGE_TRANSACTIONS +#define MDBX_HUGE_TRANSACTIONS 0 +#endif /* MDBX_HUGE_TRANSACTIONS */ + +/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_DURABILITY 0 +/** Using fsync() with chance of data lost on power failure */ +#define MDBX_OSX_WANNA_SPEED 1 + +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED + * for OSX & iOS */ +#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ + +/** Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_ENV_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be omitted: + * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_ENV_CHECKPID 0 +#else +#define MDBX_ENV_CHECKPID 1 +#endif +#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_ENV_CHECKPID) +#else +#define MDBX_ENV_CHECKPID_CONFIG STRINGIFY(MDBX_ENV_CHECKPID) +#endif /* MDBX_ENV_CHECKPID */ + +/** Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +/** Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +//------------------------------------------------------------------------------ + +/** Win32 File Locking API for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_WIN32FILES -1 + +/** SystemV IPC semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_SYSV 5 + +/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX1988 1988 + +/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2001 2001 + +/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2008 2008 + +/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_BENAPHORE 1995 + +/** Advanced: Choices the locking implementation (autodetection by default). */ +#if defined(_WIN32) || defined(_WIN64) +#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES +#else +#ifndef MDBX_LOCKING +#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ + _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ + _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ + _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 +#else +#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 +#endif +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 +#else +#define MDBX_LOCKING MDBX_LOCKING_SYSV +#endif +#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) +#else +#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) +#endif /* MDBX_LOCKING */ +#endif /* !Windows */ + +/** Advanced: Using POSIX OFD-locks (autodetection by default). */ +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) && \ + !defined(__sun) /* OFD-lock are broken on Solaris */ +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/** Advanced: Using sendfile() syscall (autodetection by default). */ +#ifndef MDBX_USE_SENDFILE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) +#define MDBX_USE_SENDFILE 1 +#else +#define MDBX_USE_SENDFILE 0 +#endif +#endif /* MDBX_USE_SENDFILE */ + +/** Advanced: Using copy_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_COPYFILERANGE +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) +#define MDBX_USE_COPYFILERANGE 1 +#else +#define MDBX_USE_COPYFILERANGE 0 +#endif +#endif /* MDBX_USE_COPYFILERANGE */ + +/** Advanced: Using sync_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_SYNCFILERANGE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) +#define MDBX_USE_SYNCFILERANGE 1 +#else +#define MDBX_USE_SYNCFILERANGE 0 +#endif +#endif /* MDBX_USE_SYNCFILERANGE */ + +//------------------------------------------------------------------------------ + +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) || defined(DOXYGEN) +#define MDBX_CPU_WRITEBACK_INCOHERENT 0 +#else +#define MDBX_CPU_WRITEBACK_INCOHERENT 1 +#endif +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#ifdef __OpenBSD__ +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 +#else +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +/** @} end of build options */ +/******************************************************************************* + ******************************************************************************* + ******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) +#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) +#define INVALID_TXNID UINT64_MAX +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + MDBX_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + bin128_t mm_bootid; + +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + __anonymous_struct_extension__ struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + indx_t mp_ptrs[] /* dynamic size */; +#endif /* C99 */ +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES +#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) +typedef void mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t mdbx_ipclock_t; +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#define MDBX_CLOCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) +typedef sem_t mdbx_ipclock_t; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); +#endif /* MDBX_LOCKING */ + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Write transaction lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_wlock; +#endif /* MDBX_LOCKING > 0 */ + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Readeaders registration lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_rlock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[] /* dynamic size */; +#endif /* C99 */ +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + __anonymous_struct_extension__ struct { + pgno_t pgno; + MDBX_page *ptr; + }; + __anonymous_struct_extension__ struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#if MDBX_HUGE_TRANSACTIONS +#define MDBX_PNL_MAX \ + ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) +#else +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) +#endif /* MDBX_HUGE_TRANSACTIONS */ + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ + size_t md_klen_min, md_klen_max; /* min/max key length for the database */ + size_t md_vlen_min, + md_vlen_max; /* min/max value/data length for the database */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) +#define MDBX_TXN_RW_BEGIN_FLAGS \ + (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + +#define TXN_FLAGS \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ + MDBX_TXN_HAS_CHILD) + +#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ + ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ + MDBX_SHRINK_ALLOWED) +#error "Oops, some flags overlapped or wrong" +#endif + + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + + /* Transaction DBI Flags */ +#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ +#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ +#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ +#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ +#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ +#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* Array of flags for each DB */ + uint8_t *mt_dbistate; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + MDBX_canary mt_canary; + void *mt_userctx; /* User-settable context */ + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbistate for this database */ + uint8_t *mc_dbistate; + unsigned mc_snum; /* number of pushed pages */ + unsigned mc_top; /* index of top page, normally mc_snum-1 */ + + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + + /* Cursor checking flags. */ +#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ +#define C_UPDATING 0x200 /* update/rebalance pending */ +#define C_RETIRING 0x400 /* refs to child pages may be invalid */ +#define C_SKIPORD 0x800 /* don't check keys ordering */ + + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbistate for this Dup DB */ + uint8_t mx_dbistate; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + void *mc_userctx; /* User-settable context */ + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + uint32_t me_signature; + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + /* Legacy MDBX_MAPASYNC (prior v0.9) */ +#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) +#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) + uint32_t me_flags; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_lazy_fd me_dxb_mmap.fd + mdbx_filehandle_t me_dsync_fd; + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + unsigned me_psize; /* DB page size, inited from me_os_psize */ + uint8_t me_psize2log; /* log2 of DB page size */ + int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ + + /* write-txn lock */ +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING > 0 + mdbx_ipclock_t *me_wlock; +#endif /* MDBX_LOCKING > 0 */ + + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + unsigned me_branch_nodemax; /* max size of a branch-node */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ + struct { +#if MDBX_LOCKING > 0 + mdbx_ipclock_t wlock; +#endif /* MDBX_LOCKING > 0 */ + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +#ifndef __cplusplus +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +extern uint8_t mdbx_runtime_flags; +extern uint8_t mdbx_loglevel; +extern MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) + mdbx_debug_log(int level, const char *function, int line, const char *fmt, + ...) MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function, + int line, const char *fmt, + va_list args); + +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#endif /* MDBX_DEBUG */ + +#if !MDBX_DEBUG && defined(__ANDROID_API__) +#define mdbx_assert_fail(env, msg, func, line) \ + __android_log_assert(msg, "mdbx", "%s:%u", func, line) +#else +void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, + int line); +#endif + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ + +#if MDBX_CPU_WRITEBACK_INCOHERENT +#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() +#else +#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +static __maybe_unused __inline void +mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = + (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + mdbx_assert(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#if MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +#endif /* !__cplusplus */ + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) + +/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + }; +#endif /* __BYTE_ORDER__ */ + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + uint8_t mn_data[] /* key and data are appended here */; +#endif /* C99 */ +} MDBX_node; + +#define DB_PERSISTENT_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP) + +/* mdbx_dbi_open() flags */ +#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) + +#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define DB_INTERNAL_FLAGS DB_VALID + +#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif +#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused bool +is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define ENV_CHANGEABLE_FLAGS \ + (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ + MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) +#define ENV_CHANGELESS_FLAGS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) +#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) + +#if !defined(__cplusplus) || defined(__cpp_constexpr) +static __maybe_unused void static_checks(void) { + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, + "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & + (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, + "Oops, some flags overlapped or wrong"); +} +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +#ifdef __cplusplus +} +#endif +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>. + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * * This code is derived from "LMDB engine" written by * Howard Chu (Symas Corporation), which itself derived from btree.c * written by Martin Hedenfalk. @@ -35,12 +3062,11 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include "internals.h" /*------------------------------------------------------------------------------ * Internal inlines */ -static __pure_function unsigned log2n(size_t value) { +MDBX_NOTHROW_CONST_FUNCTION static unsigned log2n(size_t value) { assert(value > 0 && value < INT32_MAX && is_powerof2(value)); assert((value & -(int32_t)value) == value); #if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl) @@ -60,14 +3086,14 @@ static __pure_function unsigned log2n(size_t value) { /*------------------------------------------------------------------------------ * Unaligned access */ -static __pure_function __maybe_unused __always_inline unsigned +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __always_inline unsigned field_alignment(unsigned alignment_baseline, size_t field_offset) { unsigned merge = alignment_baseline | (unsigned)field_offset; return merge & -(int)merge; } /* read-thunk for UB-sanitizer */ -static __pure_function __always_inline uint8_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t peek_u8(const uint8_t *const __restrict ptr) { return *ptr; } @@ -78,7 +3104,7 @@ static __always_inline void poke_u8(uint8_t *const __restrict ptr, *ptr = v; } -static __pure_function __always_inline uint16_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint16_t unaligned_peek_u16(const unsigned expected_alignment, const void *const ptr) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(uint16_t)) == 0) @@ -100,7 +3126,7 @@ unaligned_poke_u16(const unsigned expected_alignment, memcpy(ptr, &v, sizeof(v)); } -static __pure_function __always_inline uint32_t unaligned_peek_u32( +MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t unaligned_peek_u32( const unsigned expected_alignment, const void *const __restrict ptr) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(uint32_t)) == 0) @@ -132,7 +3158,7 @@ unaligned_poke_u32(const unsigned expected_alignment, memcpy(ptr, &v, sizeof(v)); } -static __pure_function __always_inline uint64_t unaligned_peek_u64( +MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t unaligned_peek_u64( const unsigned expected_alignment, const void *const __restrict ptr) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(uint64_t)) == 0) @@ -185,7 +3211,7 @@ unaligned_poke_u64(const unsigned expected_alignment, unaligned_poke_u64(1, (char *)(ptr) + offsetof(struct, field), value) /* Get the page number pointed to by a branch node */ -static __pure_function __always_inline pgno_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t node_pgno(const MDBX_node *const __restrict node) { pgno_t pgno = UNALIGNED_PEEK_32(node, MDBX_node, mn_pgno32); if (sizeof(pgno) > 4) @@ -205,7 +3231,7 @@ static __always_inline void node_set_pgno(MDBX_node *const __restrict node, } /* Get the size of the data in a leaf node */ -static __pure_function __always_inline size_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t node_ds(const MDBX_node *const __restrict node) { return UNALIGNED_PEEK_32(node, MDBX_node, mn_dsize); } @@ -218,7 +3244,7 @@ static __always_inline void node_set_ds(MDBX_node *const __restrict node, } /* The size of a key in a node */ -static __pure_function __always_inline size_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t node_ks(const MDBX_node *const __restrict node) { return UNALIGNED_PEEK_16(node, MDBX_node, mn_ksize); } @@ -230,7 +3256,7 @@ static __always_inline void node_set_ks(MDBX_node *const __restrict node, UNALIGNED_POKE_16(node, MDBX_node, mn_ksize, (uint16_t)size); } -static __pure_function __always_inline uint8_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t node_flags(const MDBX_node *const __restrict node) { return UNALIGNED_PEEK_8(node, MDBX_node, mn_flags); } @@ -244,29 +3270,29 @@ static __always_inline void node_set_flags(MDBX_node *const __restrict node, #define NODESIZE offsetof(MDBX_node, mn_data) /* Address of the key for the node */ -static __pure_function __always_inline void * +MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * node_key(const MDBX_node *const __restrict node) { return (char *)node + NODESIZE; } /* Address of the data for a node */ -static __pure_function __always_inline void * +MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * node_data(const MDBX_node *const __restrict node) { return (char *)node_key(node) + node_ks(node); } /* Size of a node in a leaf page with a given key and data. * This is node header plus key plus data size. */ -static __pure_function __always_inline size_t +MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t node_size_len(const size_t key_len, const size_t value_len) { return NODESIZE + EVEN(key_len + value_len); } -static __pure_function __always_inline size_t node_size(const MDBX_val *key, - const MDBX_val *value) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t +node_size(const MDBX_val *key, const MDBX_val *value) { return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0); } -static __pure_function __always_inline pgno_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t peek_pgno(const void *const __restrict ptr) { if (sizeof(pgno_t) == sizeof(uint32_t)) return (pgno_t)unaligned_peek_u32(1, ptr); @@ -289,7 +3315,7 @@ static __always_inline void poke_pgno(void *const __restrict ptr, memcpy(ptr, &pgno, sizeof(pgno)); } -static __pure_function __always_inline pgno_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t node_largedata_pgno(const MDBX_node *const __restrict node) { assert(node_flags(node) & F_BIGDATA); return peek_pgno(node_data(node)); @@ -319,7 +3345,7 @@ node_largedata_pgno(const MDBX_node *const __restrict node) { * leaf-page, since dupsort value couldn't be placed on a large/overflow * page. * - * - So, the simpliest solution is to use half of branch.maxkey as + * - So, the simplest solution is to use half of branch.maxkey as * a common maxkey value. Nevertheless, the actual values of maxkey are: * nondupsort.maxkey = even_floor(pageroom / 3) * - sizeof(indx_t) - node_hdr_len; @@ -338,14 +3364,16 @@ __cold int mdbx_env_get_maxkeysize(const MDBX_env *env) { return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT); } -__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, unsigned flags) { +__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, + MDBX_db_flags_t flags) { if (unlikely(!env || env->me_signature != MDBX_ME_SIGNATURE)) return -1; return (int)mdbx_limits_keysize_max((intptr_t)env->me_psize, flags); } -__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, unsigned flags) { +__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, + MDBX_db_flags_t flags) { if (pagesize < 1) pagesize = (intptr_t)mdbx_syspagesize(); if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || @@ -370,14 +3398,16 @@ __cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, unsigned flags) { return BRANCH_NODEMAX(pagesize) - NODESIZE - sizeof(pgno_t); } -__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, unsigned flags) { +__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, + MDBX_db_flags_t flags) { if (unlikely(!env || env->me_signature != MDBX_ME_SIGNATURE)) return -1; return (int)mdbx_limits_valsize_max((intptr_t)env->me_psize, flags); } -__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, unsigned flags) { +__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, + MDBX_db_flags_t flags) { if (pagesize < 1) pagesize = (intptr_t)mdbx_syspagesize(); if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || @@ -407,12 +3437,12 @@ __cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, unsigned flags) { * size will only include the key and not the data. Sizes are always * rounded up to an even number of bytes, to guarantee 2-byte alignment * of the MDBX_node headers. */ -static __pure_function __always_inline size_t leaf_size(const MDBX_env *env, - const MDBX_val *key, - const MDBX_val *data) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t +leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) { size_t node_bytes = node_size(key, data); /* NOTE: The actual limit is LEAF_NODEMAX(env->me_psize), but it reasonable to - * use env->me_branch_nodemax (which is 3 times less) as the treshold because: + * use env->me_branch_nodemax (which is 3 times less) as the threshold + * because: * - Large threshold implies that any insertion/update could result split * a single leaf page to THREE, which requires TWO insertion into parent * branch page, then could leads to split parent page and so on up to @@ -451,8 +3481,8 @@ static __pure_function __always_inline size_t leaf_size(const MDBX_env *env, * [in] key The key for the node. * * Returns The number of bytes needed to store the node. */ -static __pure_function __always_inline size_t branch_size(const MDBX_env *env, - const MDBX_val *key) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t +branch_size(const MDBX_env *env, const MDBX_val *key) { /* Size of a node in a branch page with a given key. * This is just the node header plus the key, there is no data. */ size_t node_bytes = node_size(key, nullptr); @@ -467,7 +3497,7 @@ static __pure_function __always_inline size_t branch_size(const MDBX_env *env, return node_bytes + sizeof(indx_t); } -static __pure_function __always_inline uint16_t +MDBX_NOTHROW_CONST_FUNCTION static __always_inline uint16_t flags_db2sub(uint16_t db_flags) { uint16_t sub_flags = db_flags & MDBX_DUPFIXED; @@ -488,96 +3518,111 @@ flags_db2sub(uint16_t db_flags) { /*----------------------------------------------------------------------------*/ -static __pure_function __always_inline size_t pgno2bytes(const MDBX_env *env, - pgno_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t +pgno2bytes(const MDBX_env *env, pgno_t pgno) { mdbx_assert(env, (1u << env->me_psize2log) == env->me_psize); return ((size_t)pgno) << env->me_psize2log; } -static __pure_function __always_inline MDBX_page *pgno2page(const MDBX_env *env, - pgno_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_page * +pgno2page(const MDBX_env *env, pgno_t pgno) { return (MDBX_page *)(env->me_map + pgno2bytes(env, pgno)); } -static __pure_function __always_inline pgno_t bytes2pgno(const MDBX_env *env, - size_t bytes) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t +bytes2pgno(const MDBX_env *env, size_t bytes) { mdbx_assert(env, (env->me_psize >> env->me_psize2log) == 1); return (pgno_t)(bytes >> env->me_psize2log); } -static __pure_function size_t pgno_align2os_bytes(const MDBX_env *env, - pgno_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION static size_t +pgno_align2os_bytes(const MDBX_env *env, pgno_t pgno) { return ceil_powerof2(pgno2bytes(env, pgno), env->me_os_psize); } -static __pure_function pgno_t pgno_align2os_pgno(const MDBX_env *env, - pgno_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION static pgno_t pgno_align2os_pgno(const MDBX_env *env, + pgno_t pgno) { return bytes2pgno(env, pgno_align2os_bytes(env, pgno)); } -static __pure_function size_t bytes_align2os_bytes(const MDBX_env *env, - size_t bytes) { +MDBX_NOTHROW_PURE_FUNCTION static size_t +bytes_align2os_bytes(const MDBX_env *env, size_t bytes) { return ceil_powerof2(ceil_powerof2(bytes, env->me_psize), env->me_os_psize); } /* Address of first usable data byte in a page, after the header */ -static __pure_function __always_inline void *page_data(const MDBX_page *mp) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * +page_data(const MDBX_page *mp) { return (char *)mp + PAGEHDRSZ; } -static __pure_function __always_inline const MDBX_page * +MDBX_NOTHROW_PURE_FUNCTION static __always_inline const MDBX_page * data_page(const void *data) { return container_of(data, MDBX_page, mp_ptrs); } -static __pure_function __always_inline MDBX_meta *page_meta(MDBX_page *mp) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_meta * +page_meta(MDBX_page *mp) { return (MDBX_meta *)page_data(mp); } /* Number of nodes on a page */ -static __pure_function __always_inline unsigned +MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned page_numkeys(const MDBX_page *mp) { return mp->mp_lower >> 1; } /* The amount of space remaining in the page */ -static __pure_function __always_inline unsigned page_room(const MDBX_page *mp) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned +page_room(const MDBX_page *mp) { return mp->mp_upper - mp->mp_lower; } -static __pure_function __always_inline unsigned +MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned page_space(const MDBX_env *env) { STATIC_ASSERT(PAGEHDRSZ % 2 == 0); return env->me_psize - PAGEHDRSZ; } -static __pure_function __always_inline unsigned page_used(const MDBX_env *env, - const MDBX_page *mp) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned +page_used(const MDBX_env *env, const MDBX_page *mp) { return page_space(env) - page_room(mp); } /* The percentage of space used in the page, in a percents. */ -static __pure_function __maybe_unused __inline double +MDBX_NOTHROW_PURE_FUNCTION static __maybe_unused __inline double page_fill(const MDBX_env *env, const MDBX_page *mp) { return page_used(env, mp) * 100.0 / page_space(env); } -static __pure_function __inline bool -page_fill_enough(const MDBX_page *mp, unsigned spaceleft_threshold, - unsigned minkeys_threshold) { - return page_room(mp) < spaceleft_threshold && - page_numkeys(mp) >= minkeys_threshold; -} - /* The number of overflow pages needed to store the given size. */ -static __pure_function __always_inline pgno_t +MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t number_of_ovpages(const MDBX_env *env, size_t bytes) { return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1; } +__cold static int MDBX_PRINTF_ARGS(2, 3) + bad_page(const MDBX_page *mp, const char *fmt, ...) { + if (mdbx_log_enabled(MDBX_LOG_ERROR)) { + static const MDBX_page *prev; + if (prev != mp) { + prev = mp; + mdbx_debug_log(MDBX_LOG_ERROR, "badpage", 0, + "corrupted page #%u, mod-txnid %" PRIaTXN " \n", + mp->mp_pgno, mp->mp_txnid); + } + + va_list args; + va_start(args, fmt); + mdbx_debug_log_va(MDBX_LOG_ERROR, "badpage", 0, fmt, args); + va_end(args); + } + return MDBX_CORRUPTED; +} + /* Address of node i in page p */ -static __pure_function __always_inline MDBX_node *page_node(const MDBX_page *mp, - unsigned i) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_node * +page_node(const MDBX_page *mp, unsigned i) { assert((mp->mp_flags & (P_LEAF2 | P_OVERFLOW | P_META)) == 0); assert(page_numkeys(mp) > (unsigned)(i)); assert(mp->mp_ptrs[i] % 2 == 0); @@ -587,7 +3632,7 @@ static __pure_function __always_inline MDBX_node *page_node(const MDBX_page *mp, /* The address of a key in a LEAF2 page. * LEAF2 pages are used for MDBX_DUPFIXED sorted-duplicate sub-DBs. * There are no node headers, keys are stored contiguously. */ -static __pure_function __always_inline void * +MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * page_leaf2key(const MDBX_page *mp, unsigned i, size_t keysize) { assert((mp->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_META)) == (P_LEAF | P_LEAF2)); @@ -737,7 +3782,8 @@ static __always_inline void atomic_yield(void) { #else __asm__ __volatile__("hint @pause"); #endif -#elif defined(__arm__) || defined(__aarch64__) +#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || \ + defined(__ARM_ARCH_6K__) #ifdef __CC_ARM __yield(); #else @@ -750,7 +3796,9 @@ static __always_inline void atomic_yield(void) { defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ defined(__MWERKS__) || defined(__sgi) __asm__ __volatile__(".word 0x00000140"); -#else +#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE) + sched_yield(); +#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS) pthread_yield(); #endif } @@ -874,7 +3922,7 @@ static __always_inline void safe64_reset(mdbx_safe64_t *ptr, static __always_inline bool safe64_reset_compare(mdbx_safe64_t *ptr, txnid_t compare) { mdbx_compiler_barrier(); - /* LY: This function is used to reset `mr_txnid` from OOM-kick in case + /* LY: This function is used to reset `mr_txnid` from hsr-handler in case * the asynchronously cancellation of read transaction. Therefore, * there may be a collision between the cleanup performed here and * asynchronous termination and restarting of the read transaction @@ -991,7 +4039,7 @@ static void __cold workaround_glibc_bug21031(void) { /* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031 * * Due race between pthread_key_delete() and __nptl_deallocate_tsd() - * The destructor(s) of thread-local-storate object(s) may be running + * The destructor(s) of thread-local-storage object(s) may be running * in another thread(s) and be blocked or not finished yet. * In such case we get a SEGFAULT after unload this library DSO. * @@ -1059,11 +4107,11 @@ static void thread_rthc_set(mdbx_thread_key_t key, const void *value) { mdbx_ensure(nullptr, TlsSetValue(key, (void *)value)); #else #define MDBX_THREAD_RTHC_ZERO 0 -#define MDBX_THREAD_RTHC_REGISTERD 1 +#define MDBX_THREAD_RTHC_REGISTERED 1 #define MDBX_THREAD_RTHC_COUNTED 2 static __thread uint32_t thread_registration_state; if (value && unlikely(thread_registration_state == MDBX_THREAD_RTHC_ZERO)) { - thread_registration_state = MDBX_THREAD_RTHC_REGISTERD; + thread_registration_state = MDBX_THREAD_RTHC_REGISTERED; mdbx_trace("thread registered 0x%" PRIxPTR, mdbx_thread_self()); if (&__cxa_thread_atexit_impl == nullptr || __cxa_thread_atexit_impl(mdbx_rthc_thread_dtor, @@ -1358,7 +4406,7 @@ static __inline void lcklist_unlock(void) { #endif } -static uint64_t rrxmrrxmsx_0(uint64_t v) { +MDBX_NOTHROW_CONST_FUNCTION static uint64_t rrxmrrxmsx_0(uint64_t v) { /* Pelle Evensen's mixer, https://bit.ly/2HOfynt */ v ^= (v << 39 | v >> 25) ^ (v << 14 | v >> 50); v *= UINT64_C(0xA24BAED4963EE407); @@ -3039,18 +6087,17 @@ static __always_inline void mdbx_dpl_clear(MDBX_DPL dl) { /*----------------------------------------------------------------------------*/ -#ifndef MDBX_ALLOY uint8_t mdbx_runtime_flags = MDBX_RUNTIME_FLAGS_INIT; -uint8_t mdbx_loglevel = MDBX_DEBUG; +uint8_t mdbx_loglevel = MDBX_LOG_FATAL; MDBX_debug_func *mdbx_debug_logger; -#endif /* MDBX_ALLOY */ static bool mdbx_refund(MDBX_txn *txn); static __must_check_result int mdbx_page_retire(MDBX_cursor *mc, MDBX_page *mp); static __must_check_result int mdbx_page_loose(MDBX_txn *txn, MDBX_page *mp); static int mdbx_page_alloc(MDBX_cursor *mc, const unsigned num, MDBX_page **const mp, int flags); -static txnid_t mdbx_oomkick(MDBX_env *env, const txnid_t laggard); +static txnid_t mdbx_kick_longlived_readers(MDBX_env *env, + const txnid_t laggard); static int mdbx_page_new(MDBX_cursor *mc, uint32_t flags, unsigned num, MDBX_page **mp); @@ -3080,10 +6127,11 @@ enum { static int mdbx_txn_end(MDBX_txn *txn, unsigned mode); static int __must_check_result mdbx_page_get(MDBX_cursor *mc, pgno_t pgno, - MDBX_page **mp, int *lvl); + MDBX_page **mp, int *lvl, + const txnid_t pp_txnid); static int __must_check_result mdbx_page_search_root(MDBX_cursor *mc, const MDBX_val *key, - int modify); + int flags); #define MDBX_PS_MODIFY 1 #define MDBX_PS_ROOTONLY 2 @@ -3130,7 +6178,8 @@ static void mdbx_node_shrink(MDBX_page *mp, unsigned indx); static int __must_check_result mdbx_node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, int fromleft); static int __must_check_result mdbx_node_read(MDBX_cursor *mc, MDBX_node *leaf, - MDBX_val *data); + MDBX_val *data, + const txnid_t pp_txnid); static int __must_check_result mdbx_rebalance(MDBX_cursor *mc); static int __must_check_result mdbx_update_key(MDBX_cursor *mc, const MDBX_val *key); @@ -3143,7 +6192,7 @@ static int __must_check_result mdbx_audit_ex(MDBX_txn *txn, bool dont_filter_gc); static __maybe_unused __always_inline int __must_check_result mdbx_audit(MDBX_txn *txn) { - return mdbx_audit_ex(txn, 0, (txn->mt_flags & MDBX_RDONLY) != 0); + return mdbx_audit_ex(txn, 0, (txn->mt_flags & MDBX_TXN_RDONLY) != 0); } static int __must_check_result mdbx_page_check(MDBX_cursor *const mc, @@ -3175,10 +6224,11 @@ static int __must_check_result mdbx_cursor_init(MDBX_cursor *mc, MDBX_txn *txn, MDBX_dbi dbi); static int __must_check_result mdbx_xcursor_init0(MDBX_cursor *mc); static int __must_check_result mdbx_xcursor_init1(MDBX_cursor *mc, - MDBX_node *node); + MDBX_node *node, + const MDBX_page *mp); static int __must_check_result mdbx_xcursor_init2(MDBX_cursor *mc, MDBX_xcursor *src_mx, - int force); + bool new_dupdata); static void mdbx_cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst); static int __must_check_result mdbx_drop0(MDBX_cursor *mc, int subs); @@ -3193,7 +6243,7 @@ static MDBX_cmp_func cmp_lexical, cmp_reverse, cmp_int_align4, cmp_int_align2, static __inline MDBX_cmp_func *get_default_keycmp(unsigned flags); static __inline MDBX_cmp_func *get_default_datacmp(unsigned flags); -static const char *__mdbx_strerr(int errnum) { +__cold const char *mdbx_liberr2str(int errnum) { /* Table of descriptions for MDBX errors */ static const char *const tbl[] = { "MDBX_KEYEXIST: Key/data pair already exists", @@ -3230,7 +6280,7 @@ static const char *__mdbx_strerr(int errnum) { " or environment is already used while opening with MDBX_EXCLUSIVE flag", }; - if (errnum >= MDBX_KEYEXIST && errnum <= MDBX_LAST_LMDB_ERRCODE) { + if (errnum >= MDBX_KEYEXIST && errnum <= MDBX_BUSY) { int i = errnum - MDBX_KEYEXIST; return tbl[i]; } @@ -3266,14 +6316,14 @@ static const char *__mdbx_strerr(int errnum) { } const char *__cold mdbx_strerror_r(int errnum, char *buf, size_t buflen) { - const char *msg = __mdbx_strerr(errnum); + const char *msg = mdbx_liberr2str(errnum); if (!msg && buflen > 0 && buflen < INT_MAX) { #if defined(_WIN32) || defined(_WIN64) const DWORD size = FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, NULL); - return size ? buf : NULL; + return size ? buf : "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed"; #elif defined(_GNU_SOURCE) && defined(__GLIBC__) /* GNU-specific */ if (errnum > 0) @@ -3305,7 +6355,7 @@ const char *__cold mdbx_strerror(int errnum) { static char buf[1024]; return mdbx_strerror_r(errnum, buf, sizeof(buf)); #else - const char *msg = __mdbx_strerr(errnum); + const char *msg = mdbx_liberr2str(errnum); if (!msg) { if (errnum > 0) msg = strerror(errnum); @@ -3321,13 +6371,17 @@ const char *__cold mdbx_strerror(int errnum) { #if defined(_WIN32) || defined(_WIN64) /* Bit of madness for Windows */ const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen) { - const char *msg = __mdbx_strerr(errnum); + const char *msg = mdbx_liberr2str(errnum); if (!msg && buflen > 0 && buflen < INT_MAX) { const DWORD size = FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, NULL); - if (size && CharToOemBuffA(buf, buf, size)) + if (!size) + msg = "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed"; + else if (!CharToOemBuffA(buf, buf, size)) + msg = "CharToOemBuffA() failed"; + else msg = buf; } return msg; @@ -3339,11 +6393,8 @@ const char *mdbx_strerror_ANSI2OEM(int errnum) { } #endif /* Bit of madness for Windows */ -void __cold mdbx_debug_log(int level, const char *function, int line, - const char *fmt, ...) { - va_list args; - - va_start(args, fmt); +void __cold mdbx_debug_log_va(int level, const char *function, int line, + const char *fmt, va_list args) { if (mdbx_debug_logger) mdbx_debug_logger(level, function, line, fmt, args); else { @@ -3379,6 +6430,13 @@ void __cold mdbx_debug_log(int level, const char *function, int line, fflush(stderr); #endif } +} + +void __cold mdbx_debug_log(int level, const char *function, int line, + const char *fmt, ...) { + va_list args; + va_start(args, fmt); + mdbx_debug_log_va(level, function, line, fmt, args); va_end(args); } @@ -3387,17 +6445,18 @@ const char *mdbx_dump_val(const MDBX_val *key, char *const buf, const size_t bufsize) { if (!key) return "<null>"; - if (!buf || bufsize < 4) - return nullptr; if (!key->iov_len) return "<empty>"; + if (!buf || bufsize < 4) + return nullptr; - const uint8_t *const data = key->iov_base; bool is_ascii = true; - unsigned i; - for (i = 0; is_ascii && i < key->iov_len; i++) - if (data[i] < ' ' || data[i] > 127) + const uint8_t *const data = key->iov_base; + for (unsigned i = 0; i < key->iov_len; i++) + if (data[i] < ' ' || data[i] > '~') { is_ascii = false; + break; + } if (is_ascii) { int len = @@ -3409,7 +6468,7 @@ const char *mdbx_dump_val(const MDBX_val *key, char *const buf, char *const detent = buf + bufsize - 2; char *ptr = buf; *ptr++ = '<'; - for (i = 0; i < key->iov_len; i++) { + for (unsigned i = 0; i < key->iov_len; i++) { const ptrdiff_t left = detent - ptr; assert(left > 0); int len = snprintf(ptr, left, "%02x", data[i]); @@ -3534,17 +6593,18 @@ static __maybe_unused void mdbx_page_list(MDBX_page *mp) { do { \ mdbx_cassert(&(mn), \ mn.mc_txn->mt_cursors != NULL /* must be not rdonly txt */); \ - MDBX_cursor mc_dummy, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ + MDBX_cursor mc_dummy; \ + MDBX_cursor **tracking_head = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ MDBX_cursor *tracked = &(mn); \ if ((mn).mc_flags & C_SUB) { \ mc_dummy.mc_flags = C_INITIALIZED; \ mc_dummy.mc_xcursor = (MDBX_xcursor *)&(mn); \ tracked = &mc_dummy; \ } \ - tracked->mc_next = *tp; \ - *tp = tracked; \ + tracked->mc_next = *tracking_head; \ + *tracking_head = tracked; \ { act; } \ - *tp = tracked->mc_next; \ + *tracking_head = tracked->mc_next; \ } while (0) int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, @@ -3787,8 +6847,8 @@ static void mdbx_refund_loose(MDBX_txn *txn) { most -= 1; } const unsigned refunded = txn->mt_next_pgno - most; - mdbx_verbose("refund-suitable %u pages %" PRIaPGNO " -> %" PRIaPGNO, - refunded, most, txn->mt_next_pgno); + mdbx_debug("refund-suitable %u pages %" PRIaPGNO " -> %" PRIaPGNO, + refunded, most, txn->mt_next_pgno); txn->tw.loose_count -= refunded; txn->tw.dirtyroom += refunded; txn->mt_next_pgno = most; @@ -3827,7 +6887,7 @@ static void mdbx_refund_loose(MDBX_txn *txn) { while (dl->length && dl[dl->length].pgno == txn->mt_next_pgno - 1 && dl[dl->length].ptr->mp_flags == (P_LOOSE | P_DIRTY)) { MDBX_page *dp = dl[dl->length].ptr; - mdbx_verbose("refund-sorted page %" PRIaPGNO, dp->mp_pgno); + mdbx_debug("refund-sorted page %" PRIaPGNO, dp->mp_pgno); mdbx_tassert(txn, dp->mp_pgno == dl[dl->length].pgno); dl->length -= 1; } @@ -3959,7 +7019,8 @@ static int mdbx_page_loose(MDBX_txn *txn, MDBX_page *mp) { mdbx_debug("loosen page %" PRIaPGNO, pgno); const bool is_dirty = IS_DIRTY(mp); - if (MDBX_DEBUG || unlikely((txn->mt_env->me_flags & MDBX_PAGEPERTURB) != 0)) { + if (MDBX_DEBUG != 0 || + unlikely((txn->mt_env->me_flags & MDBX_PAGEPERTURB) != 0)) { mdbx_kill_page(txn->mt_env, mp, pgno, npages); VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ); } @@ -4068,12 +7129,12 @@ mdbx_retire_pgno(MDBX_cursor *mc, const pgno_t pgno) { if (mdbx_audit_enabled()) { const unsigned save_flags = mc->mc_flags; mc->mc_flags |= C_RETIRING; - rc = mdbx_page_get(mc, pgno, &mp, NULL); + rc = mdbx_page_get(mc, pgno, &mp, NULL, mc->mc_txn->mt_txnid); if (likely(rc == MDBX_SUCCESS)) rc = mdbx_page_retire(mc, mp); mc->mc_flags = (mc->mc_flags & ~C_RETIRING) | (save_flags & C_RETIRING); } else { - rc = mdbx_page_get(mc, pgno, &mp, NULL); + rc = mdbx_page_get(mc, pgno, &mp, NULL, mc->mc_txn->mt_txnid); if (likely(rc == MDBX_SUCCESS)) rc = mdbx_page_retire(mc, mp); } @@ -4133,8 +7194,8 @@ mark_done: if (pgno == P_INVALID) continue; int level; - if (unlikely((rc = mdbx_page_get(m0, pgno, &dp, &level)) != - MDBX_SUCCESS)) + if (unlikely((rc = mdbx_page_get(m0, pgno, &dp, &level, + txn->mt_txnid)) != MDBX_SUCCESS)) break; if ((dp->mp_flags & Mask) == pflags && level <= 1) dp->mp_flags ^= P_KEEP; @@ -4279,10 +7340,10 @@ static __always_inline bool meta_bootid_match(const MDBX_meta *meta) { } static bool meta_weak_acceptable(const MDBX_env *env, const MDBX_meta *meta, - const int lck_exlusive) { - return lck_exlusive ? /* exclusive lock */ meta_bootid_match(meta) - : /* db already opened */ env->me_lck && - (env->me_lck->mti_envmode & MDBX_RDONLY) == 0; + const int lck_exclusive) { + return lck_exclusive ? /* exclusive lock */ meta_bootid_match(meta) + : /* db already opened */ env->me_lck && + (env->me_lck->mti_envmode & MDBX_RDONLY) == 0; } #define METAPAGE(env, n) page_meta(pgno2page(env, n)) @@ -4476,7 +7537,7 @@ static const char *mdbx_durable_str(const MDBX_meta *const meta) { /* Find oldest txnid still referenced. */ static txnid_t mdbx_find_oldest(const MDBX_txn *txn) { - mdbx_tassert(txn, (txn->mt_flags & MDBX_RDONLY) == 0); + mdbx_tassert(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); MDBX_env *env = txn->mt_env; const txnid_t edge = mdbx_recent_steady_txnid(env); mdbx_tassert(txn, edge <= txn->mt_txnid); @@ -4695,7 +7756,7 @@ static __cold int mdbx_mapresize(MDBX_env *env, const pgno_t used_pgno, goto bailout; /* 1) Windows allows only extending a read-write section, but not a - * corresponing mapped view. Therefore in other cases we must suspend + * corresponding mapped view. Therefore in other cases we must suspend * the local threads for safe remap. * 2) At least on Windows 10 1803 the entire mapped section is unavailable * for short time during NtExtendSection() or VirtualAlloc() execution. @@ -4831,11 +7892,11 @@ bailout: env->me_dxb_mmap.current, size_bytes, env->me_dxb_mmap.limit, limit_bytes, rc); } else { - mdbx_notice("unable resize datafile/mapping: " - "present %" PRIuPTR " -> %" PRIuPTR ", " - "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", - env->me_dxb_mmap.current, size_bytes, env->me_dxb_mmap.limit, - limit_bytes, rc); + mdbx_warning("unable resize datafile/mapping: " + "present %" PRIuPTR " -> %" PRIuPTR ", " + "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", + env->me_dxb_mmap.current, size_bytes, env->me_dxb_mmap.limit, + limit_bytes, rc); } if (!env->me_dxb_mmap.address) { env->me_flags |= MDBX_FATAL_ERROR; @@ -4883,10 +7944,10 @@ static __cold int mdbx_mapresize_implicit(MDBX_env *env, const pgno_t used_pgno, static int mdbx_meta_unsteady(MDBX_env *env, const txnid_t last_steady, MDBX_meta *const meta) { const uint64_t wipe = MDBX_DATASIGN_NONE; - if (META_IS_STEADY(meta) && + if (unlikely(META_IS_STEADY(meta)) && mdbx_meta_txnid_stable(env, meta) <= last_steady) { - mdbx_notice("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, last_steady, - data_page(meta)->mp_pgno); + mdbx_warning("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, last_steady, + data_page(meta)->mp_pgno); if (env->me_flags & MDBX_WRITEMAP) meta->mm_datasync_sign = wipe; else @@ -4914,17 +7975,25 @@ __cold static int mdbx_wipe_steady(MDBX_env *env, const txnid_t last_steady) { if (unlikely(err != MDBX_SUCCESS)) return err; } else { -#if (defined(__linux__) || defined(__gnu_linux__)) && \ - (!defined(__ANDROID_API__) || __ANDROID_API__ >= 26) && \ - defined(_GNU_SOURCE) && !defined(MDBX_SAFE4QEMU) - if (sync_file_range(env->me_lazy_fd, 0, pgno2bytes(env, NUM_METAS), - SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER)) +#if MDBX_USE_SYNCFILERANGE + static bool syncfilerange_unavailable; + if (likely(!syncfilerange_unavailable)) { + if (likely(!sync_file_range( + env->me_lazy_fd, 0, pgno2bytes(env, NUM_METAS), + SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER))) + goto done_filesync; err = errno; -#else - err = mdbx_filesync(env->me_lazy_fd, MDBX_SYNC_DATA); -#endif + if (ignore_enosys(err) != MDBX_RESULT_TRUE) + return err; + syncfilerange_unavailable = true; + } +#endif /* MDBX_USE_SYNCFILERANGE */ + err = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA); if (unlikely(err != MDBX_SUCCESS)) return err; +#if MDBX_USE_SYNCFILERANGE + done_filesync: +#endif /* MDBX_USE_SYNCFILERANGE */ mdbx_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS), env->me_os_psize); } @@ -4935,6 +8004,21 @@ __cold static int mdbx_wipe_steady(MDBX_env *env, const txnid_t last_steady) { return MDBX_SUCCESS; } +static __inline txnid_t pp_txnid4chk(const MDBX_page *mp, const MDBX_txn *txn) { + return IS_DIRTY(mp) + ? txn->mt_txnid - 1 + : (/* maybe zero in legacy DB */ mp->mp_txnid ? mp->mp_txnid + : MIN_TXNID); +} + +static __inline txnid_t pp_txnid2chk(const MDBX_txn *txn) { +#ifdef MDBX_DEBUG_LEGACY + if (txn->mt_txnid < 2222) + return 0; +#endif + return txn->mt_txnid; +} + /* Allocate page numbers and memory for writing. Maintain mt_last_reclaimed, * mt_reclaimed_pglist and mt_next_pgno. Set MDBX_TXN_ERROR on failure. * @@ -5011,7 +8095,7 @@ skip_cache: txnid_t oldest = 0, last = 0; const unsigned wanna_range = num - 1; - while (true) { /* oom-kick retry loop */ + while (true) { /* hsr-kick retry loop */ /* If our dirty list is already full, we can't do anything */ if (unlikely(txn->tw.dirtyroom == 0)) { rc = MDBX_TXN_FULL; @@ -5147,7 +8231,7 @@ skip_cache: if (unlikely((rc = mdbx_node_read( &recur.outer, page_node(np, recur.outer.mc_ki[recur.outer.mc_top]), - &data)) != MDBX_SUCCESS)) + &data, pp_txnid4chk(np, txn))) != MDBX_SUCCESS)) goto fail; if ((flags & MDBX_LIFORECLAIM) && !txn->tw.lifo_reclaimed) { @@ -5289,7 +8373,7 @@ skip_cache: ((autosync_threshold | autosync_period) == 0 || next >= steady->mm_geo.now)) { /* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode - * without any auto-sync treshold(s). */ + * without any auto-sync threshold(s). */ rc = mdbx_wipe_steady(env, oldest); mdbx_debug("gc-wipe-steady, rc %d", rc); mdbx_assert(env, steady != mdbx_meta_steady(env)); @@ -5314,7 +8398,7 @@ skip_cache: /* it is reasonable check/kick lagging reader(s) here, * since we made a new steady point or wipe the last. */ if (oldest < txn->mt_txnid - MDBX_TXNID_STEP && - mdbx_oomkick(env, oldest) > oldest) + mdbx_kick_longlived_readers(env, oldest) > oldest) continue; } else if (unlikely(rc != MDBX_RESULT_TRUE)) goto fail; @@ -5326,7 +8410,7 @@ skip_cache: if ((flags & MDBX_ALLOC_NEW) && next <= txn->mt_end_pgno) goto done; if ((flags & MDBX_ALLOC_GC) && oldest < txn->mt_txnid - MDBX_TXNID_STEP && - mdbx_oomkick(env, oldest) > oldest) + mdbx_kick_longlived_readers(env, oldest) > oldest) continue; rc = MDBX_NOTFOUND; @@ -5611,8 +8695,8 @@ fail: return rc; } -__cold static int mdbx_env_sync_internal(MDBX_env *env, int force, - int nonblock) { +__cold static int mdbx_env_sync_internal(MDBX_env *env, bool force, + bool nonblock) { unsigned flags = env->me_flags & ~MDBX_NOMETASYNC; if (unlikely(flags & (MDBX_RDONLY | MDBX_FATAL_ERROR))) return MDBX_EACCESS; @@ -5645,7 +8729,7 @@ __cold static int mdbx_env_sync_internal(MDBX_env *env, int force, if (outside_txn) { if (unsynced_pages > /* FIXME: define threshold */ 16 && - (flags & (MDBX_SAFE_NOSYNC | MDBX_MAPASYNC)) == 0) { + (flags & MDBX_SAFE_NOSYNC) == 0) { mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); const size_t usedbytes = pgno_align2os_bytes(env, head->mm_geo.next); @@ -5654,7 +8738,7 @@ __cold static int mdbx_env_sync_internal(MDBX_env *env, int force, /* LY: pre-sync without holding lock to reduce latency for writer(s) */ int err = (flags & MDBX_WRITEMAP) ? mdbx_msync(&env->me_dxb_mmap, 0, usedbytes, false) - : mdbx_filesync(env->me_lazy_fd, MDBX_SYNC_DATA); + : mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA); if (unlikely(err != MDBX_SUCCESS)) return err; @@ -5672,7 +8756,7 @@ __cold static int mdbx_env_sync_internal(MDBX_env *env, int force, } if (!META_IS_STEADY(head) || - ((flags & (MDBX_SAFE_NOSYNC | MDBX_MAPASYNC)) == 0 && unsynced_pages)) { + ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) { mdbx_debug("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIaPGNO, data_page(head)->mp_pgno, mdbx_durable_str(head), unsynced_pages); @@ -5693,11 +8777,10 @@ fastpath: if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_NOMETASYNC) != 0) { const txnid_t head_txnid = mdbx_recent_committed_txnid(env); if (*env->me_meta_sync_txnid != (uint32_t)head_txnid) { - rc = - (flags & MDBX_WRITEMAP) - ? mdbx_msync(&env->me_dxb_mmap, 0, - pgno_align2os_bytes(env, NUM_METAS), false) - : mdbx_filesync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + rc = (flags & MDBX_WRITEMAP) + ? mdbx_msync(&env->me_dxb_mmap, 0, + pgno_align2os_bytes(env, NUM_METAS), false) + : mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); if (likely(rc == MDBX_SUCCESS)) *env->me_meta_sync_txnid = (uint32_t)head_txnid; } @@ -5707,19 +8790,30 @@ fastpath: return rc; } -__cold int mdbx_env_sync_ex(MDBX_env *env, int force, int nonblock) { +static __inline int check_env(const MDBX_env *env) { if (unlikely(!env)) return MDBX_EINVAL; if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) return MDBX_EBADSIGN; -#if MDBX_TXN_CHECKPID +#if MDBX_ENV_CHECKPID if (unlikely(env->me_pid != mdbx_getpid())) { - env->me_flags |= MDBX_FATAL_ERROR; + ((MDBX_env *)env)->me_flags |= MDBX_FATAL_ERROR; return MDBX_PANIC; } -#endif /* MDBX_TXN_CHECKPID */ +#endif /* MDBX_ENV_CHECKPID */ + + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) + return MDBX_PANIC; + + return MDBX_SUCCESS; +} + +__cold int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; return mdbx_env_sync_internal(env, force, nonblock); } @@ -5782,9 +8876,9 @@ static void mdbx_cursors_eot(MDBX_txn *txn, unsigned merge) { for (mc = cursors[i]; mc; mc = next) { unsigned stage = mc->mc_signature; mdbx_ensure(txn->mt_env, - stage == MDBX_MC_SIGNATURE || stage == MDBX_MC_WAIT4EOT); + stage == MDBX_MC_LIVE || stage == MDBX_MC_WAIT4EOT); next = mc->mc_next; - mdbx_tassert(txn, !next || next->mc_signature == MDBX_MC_SIGNATURE || + mdbx_tassert(txn, !next || next->mc_signature == MDBX_MC_LIVE || next->mc_signature == MDBX_MC_WAIT4EOT); if ((bk = mc->mc_backup) != NULL) { if (merge) { @@ -5891,17 +8985,155 @@ static void mdbx_txn_valgrind(MDBX_env *env, MDBX_txn *txn) { } #endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ +typedef struct { + int err; + MDBX_reader *rslot; +} bind_rslot_result; + +static bind_rslot_result bind_rslot(MDBX_env *env, const uintptr_t tid) { + mdbx_assert(env, env->me_lck); + mdbx_assert(env, env->me_lck->mti_magic_and_version == MDBX_LOCK_MAGIC); + mdbx_assert(env, env->me_lck->mti_os_and_format == MDBX_LOCK_FORMAT); + + bind_rslot_result result = {mdbx_rdt_lock(env), nullptr}; + if (unlikely(MDBX_IS_ERROR(result.err))) + return result; + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { + mdbx_rdt_unlock(env); + result.err = MDBX_PANIC; + return result; + } + if (unlikely(!env->me_map)) { + mdbx_rdt_unlock(env); + result.err = MDBX_EPERM; + return result; + } + + if (unlikely(env->me_live_reader != env->me_pid)) { + result.err = mdbx_rpid_set(env); + if (unlikely(result.err != MDBX_SUCCESS)) { + mdbx_rdt_unlock(env); + return result; + } + env->me_live_reader = env->me_pid; + } + + result.err = MDBX_SUCCESS; + unsigned slot, nreaders; + while (1) { + nreaders = env->me_lck->mti_numreaders; + for (slot = 0; slot < nreaders; slot++) + if (env->me_lck->mti_readers[slot].mr_pid == 0) + break; + + if (likely(slot < env->me_maxreaders)) + break; + + result.err = mdbx_cleanup_dead_readers(env, true, NULL); + if (result.err != MDBX_RESULT_TRUE) { + mdbx_rdt_unlock(env); + result.err = + (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err; + return result; + } + } + + result.rslot = &env->me_lck->mti_readers[slot]; + /* Claim the reader slot, carefully since other code + * uses the reader table un-mutexed: First reset the + * slot, next publish it in lck->mti_numreaders. After + * that, it is safe for mdbx_env_close() to touch it. + * When it will be closed, we can finally claim it. */ + result.rslot->mr_pid = 0; + safe64_reset(&result.rslot->mr_txnid, true); + if (slot == nreaders) + env->me_lck->mti_numreaders = ++nreaders; + result.rslot->mr_tid = (env->me_flags & MDBX_NOTLS) ? 0 : tid; + result.rslot->mr_pid = env->me_pid; + mdbx_rdt_unlock(env); + + if (likely(env->me_flags & MDBX_ENV_TXKEY)) { + mdbx_assert(env, env->me_live_reader == env->me_pid); + thread_rthc_set(env->me_txkey, result.rslot); + } + return result; +} + +__cold int mdbx_thread_register(const MDBX_env *env) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!env->me_lck)) + return (env->me_flags & MDBX_EXCLUSIVE) ? MDBX_EINVAL : MDBX_EPERM; + + if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) { + mdbx_assert(env, !env->me_lck || (env->me_flags & MDBX_NOTLS)); + return MDBX_EINVAL /* MDBX_NOTLS mode */; + } + + mdbx_assert(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY | + MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY); + MDBX_reader *r = thread_rthc_get(env->me_txkey); + if (unlikely(r != NULL)) { + mdbx_assert(env, r->mr_pid == env->me_pid); + mdbx_assert(env, r->mr_tid == mdbx_thread_self()); + if (unlikely(r->mr_pid != env->me_pid)) + return MDBX_BAD_RSLOT; + return MDBX_RESULT_TRUE /* already registered */; + } + + const uintptr_t tid = mdbx_thread_self(); + if (env->me_txn0 && unlikely(env->me_txn0->mt_owner == tid)) + return MDBX_TXN_OVERLAPPING; + return bind_rslot((MDBX_env *)env, tid).err; +} + +__cold int mdbx_thread_unregister(const MDBX_env *env) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!env->me_lck)) + return MDBX_RESULT_TRUE; + + if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) { + mdbx_assert(env, !env->me_lck || (env->me_flags & MDBX_NOTLS)); + return MDBX_RESULT_TRUE /* MDBX_NOTLS mode */; + } + + mdbx_assert(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY | + MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY); + MDBX_reader *r = thread_rthc_get(env->me_txkey); + if (unlikely(r == NULL)) + return MDBX_RESULT_TRUE /* not registered */; + + mdbx_assert(env, r->mr_pid == env->me_pid); + mdbx_assert(env, r->mr_tid == mdbx_thread_self()); + if (unlikely(r->mr_pid != env->me_pid || r->mr_tid != mdbx_thread_self())) + return MDBX_BAD_RSLOT; + + if (unlikely(r->mr_txnid.inconsistent < SAFE64_INVALID_THRESHOLD)) + return MDBX_BUSY /* transaction is still active */; + + r->mr_pid = 0; + mdbx_compiler_barrier(); + env->me_lck->mti_readers_refresh_flag = true; + thread_rthc_set(env->me_txkey, nullptr); + return MDBX_SUCCESS; +} + /* Common code for mdbx_txn_begin() and mdbx_txn_renew(). */ -static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { +static int mdbx_txn_renew0(MDBX_txn *txn, const unsigned flags) { MDBX_env *env = txn->mt_env; int rc; -#if MDBX_TXN_CHECKPID +#if MDBX_ENV_CHECKPID if (unlikely(env->me_pid != mdbx_getpid())) { env->me_flags |= MDBX_FATAL_ERROR; return MDBX_PANIC; } -#endif /* MDBX_TXN_CHECKPID */ +#endif /* MDBX_ENV_CHECKPID */ STATIC_ASSERT(sizeof(MDBX_reader) == 32); #if MDBX_LOCKING > 0 @@ -5916,20 +9148,25 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE == 0); - mdbx_assert(env, (flags & ~(MDBX_TXN_BEGIN_FLAGS | MDBX_TXN_SPILLS | - MDBX_WRITEMAP)) == 0); const uintptr_t tid = mdbx_thread_self(); - if (flags & MDBX_RDONLY) { + if (flags & MDBX_TXN_RDONLY) { + mdbx_assert(env, (flags & ~(MDBX_TXN_RO_BEGIN_FLAGS | MDBX_WRITEMAP)) == 0); txn->mt_flags = - MDBX_RDONLY | (env->me_flags & (MDBX_NOTLS | MDBX_WRITEMAP)); + MDBX_TXN_RDONLY | (env->me_flags & (MDBX_NOTLS | MDBX_WRITEMAP)); MDBX_reader *r = txn->to.reader; STATIC_ASSERT(sizeof(uintptr_t) == sizeof(r->mr_tid)); if (likely(env->me_flags & MDBX_ENV_TXKEY)) { mdbx_assert(env, !(env->me_flags & MDBX_NOTLS)); r = thread_rthc_get(env->me_txkey); if (likely(r)) { - mdbx_assert(env, r->mr_pid == env->me_pid); - mdbx_assert(env, r->mr_tid == mdbx_thread_self()); + if (unlikely(!r->mr_pid) && + (mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) { + thread_rthc_set(env->me_txkey, nullptr); + r = nullptr; + } else { + mdbx_assert(env, r->mr_pid == env->me_pid); + mdbx_assert(env, r->mr_tid == mdbx_thread_self()); + } } } else { mdbx_assert(env, !env->me_lck || (env->me_flags & MDBX_NOTLS)); @@ -5940,106 +9177,85 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { r->mr_txnid.inconsistent < SAFE64_INVALID_THRESHOLD)) return MDBX_BAD_RSLOT; } else if (env->me_lck) { - unsigned slot, nreaders; - mdbx_assert(env, env->me_lck->mti_magic_and_version == MDBX_LOCK_MAGIC); - mdbx_assert(env, env->me_lck->mti_os_and_format == MDBX_LOCK_FORMAT); - - rc = mdbx_rdt_lock(env); - if (unlikely(MDBX_IS_ERROR(rc))) - return rc; - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { - mdbx_rdt_unlock(env); - return MDBX_PANIC; - } -#if defined(_WIN32) || defined(_WIN64) - if (unlikely(!env->me_map)) { - mdbx_rdt_unlock(env); - return MDBX_EPERM; + bind_rslot_result brs = bind_rslot(env, tid); + if (unlikely(brs.err != MDBX_SUCCESS)) + return brs.err; + r = brs.rslot; + } + txn->to.reader = r; + if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) { + mdbx_assert(env, txn->mt_txnid == 0); + mdbx_assert(env, txn->mt_owner == 0); + mdbx_assert(env, txn->mt_numdbs == 0); + if (likely(r)) { + mdbx_assert(env, r->mr_snapshot_pages_used == 0); + mdbx_assert(env, r->mr_txnid.inconsistent >= SAFE64_INVALID_THRESHOLD); + r->mr_snapshot_pages_used = 0; } -#endif /* Windows */ - rc = MDBX_SUCCESS; + txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED; + return MDBX_SUCCESS; + } - if (unlikely(env->me_live_reader != env->me_pid)) { - rc = mdbx_rpid_set(env); - if (unlikely(rc != MDBX_SUCCESS)) { - mdbx_rdt_unlock(env); - return rc; + /* Seek & fetch the last meta */ + if (likely(/* not recovery mode */ env->me_stuck_meta < 0)) { + while (1) { + MDBX_meta *const meta = mdbx_meta_head(env); + mdbx_jitter4testing(false); + const txnid_t snap = mdbx_meta_txnid_fluid(env, meta); + mdbx_jitter4testing(false); + if (likely(r)) { + safe64_reset(&r->mr_txnid, false); + r->mr_snapshot_pages_used = meta->mm_geo.next; + r->mr_snapshot_pages_retired = meta->mm_pages_retired; + safe64_write(&r->mr_txnid, snap); + mdbx_jitter4testing(false); + mdbx_assert(env, r->mr_pid == mdbx_getpid()); + mdbx_assert( + env, r->mr_tid == + ((env->me_flags & MDBX_NOTLS) ? 0 : mdbx_thread_self())); + mdbx_assert(env, r->mr_txnid.inconsistent == snap); + mdbx_compiler_barrier(); + env->me_lck->mti_readers_refresh_flag = true; + mdbx_flush_incoherent_cpu_writeback(); } - env->me_live_reader = env->me_pid; - } + mdbx_jitter4testing(true); - while (1) { - nreaders = env->me_lck->mti_numreaders; - for (slot = 0; slot < nreaders; slot++) - if (env->me_lck->mti_readers[slot].mr_pid == 0) - break; + /* Snap the state from current meta-head */ + txn->mt_txnid = snap; + txn->mt_geo = meta->mm_geo; + memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db)); + txn->mt_canary = meta->mm_canary; - if (likely(slot < env->me_maxreaders)) + /* LY: Retry on a race, ITS#7970. */ + mdbx_compiler_barrier(); + if (likely(meta == mdbx_meta_head(env) && + snap == mdbx_meta_txnid_fluid(env, meta) && + snap >= *env->me_oldest)) { + mdbx_jitter4testing(false); break; - - rc = mdbx_reader_check0(env, true, NULL); - if (rc != MDBX_RESULT_TRUE) { - mdbx_rdt_unlock(env); - return (rc == MDBX_SUCCESS) ? MDBX_READERS_FULL : rc; } } - - r = &env->me_lck->mti_readers[slot]; - /* Claim the reader slot, carefully since other code - * uses the reader table un-mutexed: First reset the - * slot, next publish it in lck->mti_numreaders. After - * that, it is safe for mdbx_env_close() to touch it. - * When it will be closed, we can finally claim it. */ - r->mr_pid = 0; - safe64_reset(&r->mr_txnid, true); - if (slot == nreaders) - env->me_lck->mti_numreaders = ++nreaders; - r->mr_tid = (env->me_flags & MDBX_NOTLS) ? 0 : tid; - r->mr_pid = env->me_pid; - mdbx_rdt_unlock(env); - - if (likely(env->me_flags & MDBX_ENV_TXKEY)) { - mdbx_assert(env, env->me_live_reader == env->me_pid); - thread_rthc_set(env->me_txkey, r); - } - } - - while (1) { - MDBX_meta *const meta = mdbx_meta_head(env); - mdbx_jitter4testing(false); - const txnid_t snap = mdbx_meta_txnid_fluid(env, meta); - mdbx_jitter4testing(false); + } else { + /* recovery mode */ + MDBX_meta *const meta = METAPAGE(env, env->me_stuck_meta); + txn->mt_txnid = mdbx_meta_txnid_stable(env, meta); + txn->mt_geo = meta->mm_geo; + memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db)); + txn->mt_canary = meta->mm_canary; if (likely(r)) { - safe64_reset(&r->mr_txnid, false); r->mr_snapshot_pages_used = meta->mm_geo.next; r->mr_snapshot_pages_retired = meta->mm_pages_retired; - safe64_write(&r->mr_txnid, snap); + r->mr_txnid.inconsistent = txn->mt_txnid; mdbx_jitter4testing(false); mdbx_assert(env, r->mr_pid == mdbx_getpid()); mdbx_assert( env, r->mr_tid == ((env->me_flags & MDBX_NOTLS) ? 0 : mdbx_thread_self())); - mdbx_assert(env, r->mr_txnid.inconsistent == snap); + mdbx_assert(env, r->mr_txnid.inconsistent == txn->mt_txnid); mdbx_compiler_barrier(); env->me_lck->mti_readers_refresh_flag = true; mdbx_flush_incoherent_cpu_writeback(); } - mdbx_jitter4testing(true); - - /* Snap the state from current meta-head */ - txn->mt_txnid = snap; - txn->mt_geo = meta->mm_geo; - memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db)); - txn->mt_canary = meta->mm_canary; - - /* LY: Retry on a race, ITS#7970. */ - mdbx_compiler_barrier(); - if (likely(meta == mdbx_meta_head(env) && - snap == mdbx_meta_txnid_fluid(env, meta) && - snap >= *env->me_oldest)) { - mdbx_jitter4testing(false); - break; - } } if (unlikely(txn->mt_txnid < MIN_TXNID || txn->mt_txnid > MAX_TXNID)) { @@ -6048,13 +9264,15 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { goto bailout; } mdbx_assert(env, txn->mt_txnid >= *env->me_oldest); - txn->to.reader = r; txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ mdbx_ensure(env, txn->mt_txnid >= /* paranoia is appropriate here */ *env->me_oldest); txn->mt_numdbs = env->me_numdbs; } else { - if (unlikely(txn->mt_owner == tid)) + mdbx_assert(env, (flags & ~(MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS | + MDBX_WRITEMAP)) == 0); + if (unlikely(txn->mt_owner == tid || + /* not recovery mode */ env->me_stuck_meta >= 0)) return MDBX_BUSY; MDBX_lockinfo *const lck = env->me_lck; if (lck && (env->me_flags & MDBX_NOTLS) == 0 && @@ -6072,7 +9290,7 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { /* Not yet touching txn == env->me_txn0, it may be active */ mdbx_jitter4testing(false); - rc = mdbx_txn_lock(env, F_ISSET(flags, MDBX_TRYTXN)); + rc = mdbx_txn_lock(env, F_ISSET(flags, MDBX_TXN_TRY)); if (unlikely(rc)) return rc; if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { @@ -6093,7 +9311,7 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { const txnid_t snap = mdbx_meta_txnid_stable(env, meta); txn->mt_txnid = safe64_txnid_next(snap); if (unlikely(txn->mt_txnid > MAX_TXNID)) { - mdbx_debug("%s", "txnid overflow!"); + mdbx_error("%s", "txnid overflow!"); rc = MDBX_TXN_FULL; goto bailout; } @@ -6135,7 +9353,9 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { mdbx_warning("%s", "environment had fatal error, must shutdown!"); rc = MDBX_PANIC; } else { - const size_t size = pgno2bytes(env, txn->mt_end_pgno); + const size_t size = + pgno2bytes(env, (txn->mt_flags & MDBX_TXN_RDONLY) ? txn->mt_next_pgno + : txn->mt_end_pgno); if (unlikely(size > env->me_dxb_mmap.limit)) { if (txn->mt_geo.upper > MAX_PAGENO || bytes2pgno(env, pgno2bytes(env, txn->mt_geo.upper)) != @@ -6145,11 +9365,11 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { } rc = mdbx_mapresize(env, txn->mt_next_pgno, txn->mt_end_pgno, txn->mt_geo.upper, - (txn->mt_flags & MDBX_RDONLY) ? true : false); + (txn->mt_flags & MDBX_TXN_RDONLY) ? true : false); if (rc != MDBX_SUCCESS) goto bailout; } - if (txn->mt_flags & MDBX_RDONLY) { + if (txn->mt_flags & MDBX_TXN_RDONLY) { #if defined(_WIN32) || defined(_WIN64) if ((size > env->me_dbgeo.lower && env->me_dbgeo.shrink) || (mdbx_RunningUnderWine() && @@ -6209,7 +9429,7 @@ static __always_inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) { if (unlikely(txn->mt_flags & bad_bits)) return MDBX_BAD_TXN; - if (unlikely(F_ISSET(txn->mt_flags, MDBX_RDONLY))) + if (unlikely(F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY))) return MDBX_EACCESS; #if MDBX_TXN_CHECKOWNER @@ -6227,7 +9447,7 @@ int mdbx_txn_renew(MDBX_txn *txn) { if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE)) return MDBX_EBADSIGN; - if (unlikely((txn->mt_flags & MDBX_RDONLY) == 0)) + if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) return MDBX_EINVAL; int rc; @@ -6237,73 +9457,76 @@ int mdbx_txn_renew(MDBX_txn *txn) { return rc; } - rc = mdbx_txn_renew0(txn, MDBX_RDONLY); + rc = mdbx_txn_renew0(txn, MDBX_TXN_RDONLY); if (rc == MDBX_SUCCESS) { txn->mt_owner = mdbx_thread_self(); mdbx_debug("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, - txn->mt_txnid, (txn->mt_flags & MDBX_RDONLY) ? 'r' : 'w', + txn->mt_txnid, (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root, txn->mt_dbs[FREE_DBI].md_root); } return rc; } -int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags, +int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret) { + return mdbx_txn_begin_ex(env, parent, flags, ret, nullptr); +} + +int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + txn->mt_userctx = ctx; + return MDBX_SUCCESS; +} + +void *mdbx_txn_get_userctx(const MDBX_txn *txn) { + return check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD) + ? nullptr + : txn->mt_userctx; +} + +int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, + MDBX_txn **ret, void *context) { MDBX_txn *txn; - int rc; unsigned size, tsize; if (unlikely(!ret)) return MDBX_EINVAL; *ret = NULL; - if (unlikely(!env)) + if (unlikely((flags & ~MDBX_TXN_RW_BEGIN_FLAGS) && + (flags & ~MDBX_TXN_RO_BEGIN_FLAGS))) return MDBX_EINVAL; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; - -#if MDBX_TXN_CHECKPID - if (unlikely(env->me_pid != mdbx_getpid())) - env->me_flags |= MDBX_FATAL_ERROR; -#endif /* MDBX_TXN_CHECKPID */ - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) - return MDBX_PANIC; - -#if !defined(_WIN32) && !defined(_WIN64) - /* Don't check env->me_map until lock to - * avoid race with re-mapping for shrinking */ - if (unlikely(!env->me_map)) - return MDBX_EPERM; -#endif /* Windows */ - - if (unlikely(flags & ~MDBX_TXN_BEGIN_FLAGS)) - return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; if (unlikely(env->me_flags & MDBX_RDONLY & ~flags)) /* write txn in RDONLY env */ return MDBX_EACCESS; + if (unlikely(!env->me_map)) + return MDBX_EPERM; + flags |= env->me_flags & MDBX_WRITEMAP; if (parent) { /* Nested transactions: Max 1 child, write txns only, no writemap */ - rc = check_txn_rw(parent, MDBX_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED); + rc = check_txn_rw(parent, + MDBX_TXN_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; -#if defined(_WIN32) || defined(_WIN64) - if (unlikely(!env->me_map)) - return MDBX_EPERM; -#endif /* Windows */ - - flags |= parent->mt_flags & (MDBX_TXN_BEGIN_FLAGS | MDBX_TXN_SPILLS); + flags |= parent->mt_flags & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS); /* Child txns save MDBX_pgstate and use own copy of cursors */ size = env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) + 1); size += tsize = sizeof(MDBX_txn); - } else if (flags & MDBX_RDONLY) { + } else if (flags & MDBX_TXN_RDONLY) { if (env->me_txn0 && unlikely(env->me_txn0->mt_owner == mdbx_thread_self()) && (mdbx_runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) @@ -6345,7 +9568,7 @@ int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags, MDBX_PNL_SIZEOF(parent->tw.reclaimed_pglist)); mdbx_assert(env, mdbx_pnl_check4assert( txn->tw.reclaimed_pglist, - (txn->mt_next_pgno /* LY: intentional assigment here, + (txn->mt_next_pgno /* LY: intentional assignment here, only for assertion */ = parent->mt_next_pgno))); @@ -6382,7 +9605,7 @@ int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags, rc = mdbx_cursor_shadow(parent, txn); if (unlikely(rc != MDBX_SUCCESS)) mdbx_txn_end(txn, MDBX_END_FAIL_BEGINCHILD); - } else { /* MDBX_RDONLY */ + } else { /* MDBX_TXN_RDONLY */ txn->mt_dbiseqs = env->me_dbiseqs; renew: rc = mdbx_txn_renew0(txn, flags); @@ -6392,23 +9615,30 @@ int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags, if (txn != env->me_txn0) mdbx_free(txn); } else { - mdbx_assert(env, - (txn->mt_flags & ~(MDBX_NOTLS | MDBX_RDONLY | MDBX_WRITEMAP | - MDBX_SHRINK_ALLOWED | MDBX_NOMETASYNC | - MDBX_SAFE_NOSYNC | MDBX_MAPASYNC)) == 0); + if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) + mdbx_assert(env, txn->mt_flags == (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED)); + else if (flags & MDBX_TXN_RDONLY) + mdbx_assert(env, (txn->mt_flags & + ~(MDBX_NOTLS | MDBX_TXN_RDONLY | MDBX_WRITEMAP | + /* Win32: SRWL flag */ MDBX_SHRINK_ALLOWED)) == 0); + else + mdbx_assert(env, + (txn->mt_flags & ~(MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED | + MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC)) == 0); txn->mt_signature = MDBX_MT_SIGNATURE; + txn->mt_userctx = context; *ret = txn; mdbx_debug("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, - txn->mt_txnid, (flags & MDBX_RDONLY) ? 'r' : 'w', (void *)txn, - (void *)env, txn->mt_dbs[MAIN_DBI].md_root, + txn->mt_txnid, (flags & MDBX_TXN_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root, txn->mt_dbs[FREE_DBI].md_root); } return rc; } -int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, int scan_rlt) { +int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -6417,17 +9647,17 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, int scan_rlt) { return MDBX_EINVAL; MDBX_env *const env = txn->mt_env; -#if MDBX_TXN_CHECKPID +#if MDBX_ENV_CHECKPID if (unlikely(env->me_pid != mdbx_getpid())) { env->me_flags |= MDBX_FATAL_ERROR; return MDBX_PANIC; } -#endif /* MDBX_TXN_CHECKPID */ +#endif /* MDBX_ENV_CHECKPID */ info->txn_id = txn->mt_txnid; info->txn_space_used = pgno2bytes(env, txn->mt_geo.next); - if (txn->mt_flags & MDBX_RDONLY) { + if (txn->mt_flags & MDBX_TXN_RDONLY) { const MDBX_meta *head_meta; txnid_t head_txnid; uint64_t head_retired; @@ -6588,17 +9818,17 @@ static int mdbx_txn_end(MDBX_txn *txn, unsigned mode) { MDBX_env *env = txn->mt_env; static const char *const names[] = MDBX_END_NAMES; -#if MDBX_TXN_CHECKPID +#if MDBX_ENV_CHECKPID if (unlikely(txn->mt_env->me_pid != mdbx_getpid())) { env->me_flags |= MDBX_FATAL_ERROR; return MDBX_PANIC; } -#endif /* MDBX_TXN_CHECKPID */ +#endif /* MDBX_ENV_CHECKPID */ mdbx_debug("%s txn %" PRIaTXN "%c %p on mdbenv %p, root page %" PRIaPGNO "/%" PRIaPGNO, names[mode & MDBX_END_OPMASK], txn->mt_txnid, - (txn->mt_flags & MDBX_RDONLY) ? 'r' : 'w', (void *)txn, + (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root, txn->mt_dbs[FREE_DBI].md_root); @@ -6606,7 +9836,7 @@ static int mdbx_txn_end(MDBX_txn *txn, unsigned mode) { /* paranoia is appropriate here */ *env->me_oldest); int rc = MDBX_SUCCESS; - if (F_ISSET(txn->mt_flags, MDBX_RDONLY)) { + if (F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)) { if (txn->to.reader) { MDBX_reader *slot = txn->to.reader; mdbx_assert(env, slot->mr_pid == env->me_pid); @@ -6637,7 +9867,7 @@ static int mdbx_txn_end(MDBX_txn *txn, unsigned mode) { mdbx_srwlock_ReleaseShared(&env->me_remap_guard); #endif txn->mt_numdbs = 0; /* prevent further DBI activity */ - txn->mt_flags = MDBX_RDONLY | MDBX_TXN_FINISHED; + txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED; txn->mt_owner = 0; } else if (!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED)) { #if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) @@ -6696,10 +9926,10 @@ static int mdbx_txn_end(MDBX_txn *txn, unsigned mode) { if (rc == MDBX_RESULT_TRUE) { /* unable undo resize (it is regular for Windows), * therefore promote size changes from child to the parent txn */ - mdbx_notice("unable undo resize performed by child txn, promote to " - "the parent (%u->%u, %u->%u)", - txn->mt_geo.now, parent->mt_geo.now, txn->mt_geo.upper, - parent->mt_geo.upper); + mdbx_warning("unable undo resize performed by child txn, promote to " + "the parent (%u->%u, %u->%u)", + txn->mt_geo.now, parent->mt_geo.now, txn->mt_geo.upper, + parent->mt_geo.upper); parent->mt_geo.now = txn->mt_geo.now; parent->mt_geo.upper = txn->mt_geo.upper; rc = MDBX_SUCCESS; @@ -6730,7 +9960,7 @@ int mdbx_txn_reset(MDBX_txn *txn) { return rc; /* This call is only valid for read-only txns */ - if (unlikely((txn->mt_flags & MDBX_RDONLY) == 0)) + if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) return MDBX_EINVAL; /* LY: don't close DBI-handles */ @@ -6742,12 +9972,25 @@ int mdbx_txn_reset(MDBX_txn *txn) { return rc; } +int mdbx_txn_break(MDBX_txn *txn) { + do { + int rc = check_txn(txn, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + txn->mt_flags |= MDBX_TXN_ERROR; + if (txn->mt_flags & MDBX_TXN_RDONLY) + break; + txn = txn->mt_child; + } while (txn); + return MDBX_SUCCESS; +} + int mdbx_txn_abort(MDBX_txn *txn) { int rc = check_txn(txn, 0); if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (F_ISSET(txn->mt_flags, MDBX_RDONLY)) + if (F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY)) /* LY: don't close DBI-handles */ return mdbx_txn_end(txn, MDBX_END_ABORT | MDBX_END_UPDATE | MDBX_END_SLOT | MDBX_END_FREE); @@ -6764,7 +10007,7 @@ int mdbx_txn_abort(MDBX_txn *txn) { static __cold int mdbx_audit_ex(MDBX_txn *txn, unsigned retired_stored, bool dont_filter_gc) { pgno_t pending = 0; - if ((txn->mt_flags & MDBX_RDONLY) == 0) { + if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) { pending = txn->tw.loose_count + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) + (MDBX_PNL_SIZE(txn->tw.retired_pages) - retired_stored) + txn->tw.retired2parent_count; @@ -6821,11 +10064,11 @@ static __cold int mdbx_audit_ex(MDBX_txn *txn, unsigned retired_stored, for (unsigned j = 0; j < page_numkeys(mp); j++) { MDBX_node *node = page_node(mp, j); if (node_flags(node) == F_SUBDATA) { - if (unlikely(node_ds(node) < sizeof(MDBX_db))) + if (unlikely(node_ds(node) != sizeof(MDBX_db))) return MDBX_CORRUPTED; MDBX_db db_copy, *db; memcpy(db = &db_copy, node_data(node), sizeof(db_copy)); - if ((txn->mt_flags & MDBX_RDONLY) == 0) { + if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) { for (MDBX_dbi k = txn->mt_numdbs; --k > MAIN_DBI;) { if ((txn->mt_dbistate[k] & DBI_VALID) && /* txn->mt_dbxs[k].md_name.iov_len > 0 && */ @@ -6870,7 +10113,7 @@ static __cold int mdbx_audit_ex(MDBX_txn *txn, unsigned retired_stored, if (pending + freecount + count + NUM_METAS == txn->mt_next_pgno) return MDBX_SUCCESS; - if ((txn->mt_flags & MDBX_RDONLY) == 0) + if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) mdbx_error("audit @%" PRIaTXN ": %u(pending) = %u(loose-count) + " "%u(reclaimed-list) + %u(retired-pending) - %u(retired-stored) " "+ %u(retired2parent)", @@ -7197,7 +10440,7 @@ retry_noaccount: continue; } - /* handle reclaimed and loost pages - merge and store both into gc */ + /* handle reclaimed and lost pages - merge and store both into gc */ mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno)); mdbx_tassert(txn, txn->tw.loose_count == 0); @@ -7671,7 +10914,7 @@ __hot static int mdbx_page_flush(MDBX_txn *txn, const unsigned keep) { (flush_end > dp->mp_pgno + npages) ? flush_end : dp->mp_pgno + npages; *env->me_unsynced_pages += npages; dp->mp_flags &= ~P_DIRTY; - dp->mp_txnid = txn->mt_txnid; + dp->mp_txnid = pp_txnid2chk(txn); if ((env->me_flags & MDBX_WRITEMAP) == 0) { const size_t size = pgno2bytes(env, npages); @@ -7781,17 +11024,17 @@ int mdbx_txn_commit(MDBX_txn *txn) { } MDBX_env *env = txn->mt_env; -#if MDBX_TXN_CHECKPID +#if MDBX_ENV_CHECKPID if (unlikely(env->me_pid != mdbx_getpid())) { env->me_flags |= MDBX_FATAL_ERROR; return MDBX_PANIC; } -#endif /* MDBX_TXN_CHECKPID */ +#endif /* MDBX_ENV_CHECKPID */ /* mdbx_txn_end() mode for a commit which writes nothing */ unsigned end_mode = MDBX_END_EMPTY_COMMIT | MDBX_END_UPDATE | MDBX_END_SLOT | MDBX_END_FREE; - if (unlikely(F_ISSET(txn->mt_flags, MDBX_RDONLY))) + if (unlikely(F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY))) goto done; if (txn->mt_child) { @@ -8091,7 +11334,7 @@ int mdbx_txn_commit(MDBX_txn *txn) { goto fail; } MDBX_db *db = &txn->mt_dbs[i]; - db->md_mod_txnid = txn->mt_txnid; + db->md_mod_txnid = pp_txnid2chk(txn); data.iov_base = db; WITH_CURSOR_TRACKING(couple.outer, rc = mdbx_cursor_put(&couple.outer, @@ -8116,7 +11359,7 @@ int mdbx_txn_commit(MDBX_txn *txn) { rc = mdbx_page_flush(txn, 0); if (likely(rc == MDBX_SUCCESS)) { if (txn->mt_dbs[MAIN_DBI].md_flags & DBI_DIRTY) - txn->mt_dbs[MAIN_DBI].md_mod_txnid = txn->mt_txnid; + txn->mt_dbs[MAIN_DBI].md_mod_txnid = pp_txnid2chk(txn); MDBX_meta meta, *head = mdbx_meta_head(env); meta.mm_magic_and_version = head->mm_magic_and_version; @@ -8179,8 +11422,8 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, /* LY: check pagesize */ if (!is_powerof2(meta->mm_psize) || meta->mm_psize < MIN_PAGESIZE || meta->mm_psize > MAX_PAGESIZE) { - mdbx_notice("meta[%u] has invalid pagesize (%u), skip it", meta_number, - meta->mm_psize); + mdbx_warning("meta[%u] has invalid pagesize (%u), skip it", meta_number, + meta->mm_psize); return is_powerof2(meta->mm_psize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID; } @@ -8196,9 +11439,9 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, /* LY: check signature as a checksum */ if (META_IS_STEADY(meta) && meta->mm_datasync_sign != mdbx_meta_sign(meta)) { - mdbx_notice("meta[%u] has invalid steady-checksum (0x%" PRIx64 - " != 0x%" PRIx64 "), skip it", - meta_number, meta->mm_datasync_sign, mdbx_meta_sign(meta)); + mdbx_warning("meta[%u] has invalid steady-checksum (0x%" PRIx64 + " != 0x%" PRIx64 "), skip it", + meta_number, meta->mm_datasync_sign, mdbx_meta_sign(meta)); return MDBX_RESULT_TRUE; } @@ -8213,23 +11456,23 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, /* LY: check min-pages value */ if (meta->mm_geo.lower < MIN_PAGENO || meta->mm_geo.lower > MAX_PAGENO) { - mdbx_notice("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.lower); + mdbx_warning("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.lower); return MDBX_INVALID; } /* LY: check max-pages value */ if (meta->mm_geo.upper < MIN_PAGENO || meta->mm_geo.upper > MAX_PAGENO || meta->mm_geo.upper < meta->mm_geo.lower) { - mdbx_notice("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.upper); + mdbx_warning("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.upper); return MDBX_INVALID; } /* LY: check last_pgno */ if (meta->mm_geo.next < MIN_PAGENO || meta->mm_geo.next - 1 > MAX_PAGENO) { - mdbx_notice("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.next); + mdbx_warning("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.next); return MDBX_CORRUPTED; } @@ -8241,21 +11484,36 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, if (unlikely(err != MDBX_SUCCESS)) return err; if (used_bytes > *filesize) { - mdbx_notice("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 - "), skip it", - meta_number, used_bytes, *filesize); + mdbx_warning("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 + "), skip it", + meta_number, used_bytes, *filesize); return MDBX_CORRUPTED; } } + if (meta->mm_geo.next - 1 > MAX_PAGENO || used_bytes > MAX_MAPSIZE) { + mdbx_warning("meta[%u] has too large used-space (%" PRIu64 "), skip it", + meta_number, used_bytes); + return MDBX_TOO_LARGE; + } /* LY: check mapsize limits */ const uint64_t mapsize_min = meta->mm_geo.lower * (uint64_t)meta->mm_psize; STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MAX_PAGESIZE); STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); if (mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE) { - mdbx_notice("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", - meta_number, mapsize_min); - return MDBX_VERSION_MISMATCH; + if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE && + mapsize_min <= MAX_MAPSIZE64) { + mdbx_assert(env, meta->mm_geo.next - 1 <= MAX_PAGENO && + used_bytes <= MAX_MAPSIZE); + mdbx_warning("meta[%u] has too large min-mapsize (%" PRIu64 "), " + "but size of used space still acceptable (%" PRIu64 ")", + meta_number, mapsize_min, used_bytes); + meta->mm_geo.lower = (pgno_t)(MAX_MAPSIZE / meta->mm_psize); + } else { + mdbx_warning("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", + meta_number, mapsize_min); + return MDBX_VERSION_MISMATCH; + } } const uint64_t mapsize_max = meta->mm_geo.upper * (uint64_t)meta->mm_psize; @@ -8263,16 +11521,12 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, if (mapsize_max > MAX_MAPSIZE || MAX_PAGENO < ceil_powerof2((size_t)mapsize_max, env->me_os_psize) / (size_t)meta->mm_psize) { - if (meta->mm_geo.next - 1 > MAX_PAGENO || used_bytes > MAX_MAPSIZE) { - mdbx_notice("meta[%u] has too large max-mapsize (%" PRIu64 "), skip it", - meta_number, mapsize_max); - return MDBX_TOO_LARGE; - } - /* allow to open large DB from a 32-bit environment */ - mdbx_notice("meta[%u] has too large max-mapsize (%" PRIu64 "), " - "but size of used space still acceptable (%" PRIu64 ")", - meta_number, mapsize_max, used_bytes); + mdbx_assert(env, meta->mm_geo.next - 1 <= MAX_PAGENO && + used_bytes <= MAX_MAPSIZE); + mdbx_warning("meta[%u] has too large max-mapsize (%" PRIu64 "), " + "but size of used space still acceptable (%" PRIu64 ")", + meta_number, mapsize_max, used_bytes); meta->mm_geo.upper = (pgno_t)(MAX_MAPSIZE / meta->mm_psize); } @@ -8290,9 +11544,9 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, meta->mm_geo.now = meta->mm_geo.upper; if (meta->mm_geo.next > meta->mm_geo.now) { - mdbx_notice("meta[%u] next-pageno (%" PRIaPGNO - ") is beyond end-pgno (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.next, meta->mm_geo.now); + mdbx_warning("meta[%u] next-pageno (%" PRIaPGNO + ") is beyond end-pgno (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.next, meta->mm_geo.now); return MDBX_CORRUPTED; } @@ -8302,12 +11556,12 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, meta->mm_dbs[FREE_DBI].md_depth || meta->mm_dbs[FREE_DBI].md_entries || meta->mm_dbs[FREE_DBI].md_leaf_pages || meta->mm_dbs[FREE_DBI].md_overflow_pages) { - mdbx_notice("meta[%u] has false-empty GC, skip it", meta_number); + mdbx_warning("meta[%u] has false-empty GC, skip it", meta_number); return MDBX_CORRUPTED; } } else if (meta->mm_dbs[FREE_DBI].md_root >= meta->mm_geo.next) { - mdbx_notice("meta[%u] has invalid GC-root %" PRIaPGNO ", skip it", - meta_number, meta->mm_dbs[FREE_DBI].md_root); + mdbx_warning("meta[%u] has invalid GC-root %" PRIaPGNO ", skip it", + meta_number, meta->mm_dbs[FREE_DBI].md_root); return MDBX_CORRUPTED; } @@ -8317,12 +11571,12 @@ static int __cold mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta, meta->mm_dbs[MAIN_DBI].md_depth || meta->mm_dbs[MAIN_DBI].md_entries || meta->mm_dbs[MAIN_DBI].md_leaf_pages || meta->mm_dbs[MAIN_DBI].md_overflow_pages) { - mdbx_notice("meta[%u] has false-empty maindb", meta_number); + mdbx_warning("meta[%u] has false-empty maindb", meta_number); return MDBX_CORRUPTED; } } else if (meta->mm_dbs[MAIN_DBI].md_root >= meta->mm_geo.next) { - mdbx_notice("meta[%u] has invalid maindb-root %" PRIaPGNO ", skip it", - meta_number, meta->mm_dbs[MAIN_DBI].md_root); + mdbx_warning("meta[%u] has invalid maindb-root %" PRIaPGNO ", skip it", + meta_number, meta->mm_dbs[MAIN_DBI].md_root); return MDBX_CORRUPTED; } @@ -8369,8 +11623,8 @@ static int __cold mdbx_read_header(MDBX_env *env, MDBX_meta *dest, if (err != MDBX_SUCCESS) { if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 && *filesize == 0 && (env->me_flags & MDBX_RDONLY) == 0) - mdbx_notice("read meta: empty file (%d, %s)", err, - mdbx_strerror(err)); + mdbx_warning("read meta: empty file (%d, %s)", err, + mdbx_strerror(err)); else mdbx_error("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err, mdbx_strerror(err)); @@ -8403,7 +11657,9 @@ static int __cold mdbx_read_header(MDBX_env *env, MDBX_meta *dest, if (rc != MDBX_SUCCESS) continue; - if (mdbx_meta_ot(prefer_steady, env, dest, meta)) { + if ((env->me_stuck_meta < 0) + ? mdbx_meta_ot(prefer_steady, env, dest, meta) + : (meta_number == (unsigned)env->me_stuck_meta)) { *dest = *meta; if (!META_IS_STEADY(dest)) loop_limit += 1; /* LY: should re-read to hush race with update */ @@ -8412,8 +11668,10 @@ static int __cold mdbx_read_header(MDBX_env *env, MDBX_meta *dest, } if (dest->mm_psize == 0 || - (!META_IS_STEADY(dest) && - !meta_weak_acceptable(env, dest, lck_exclusive))) { + ((env->me_stuck_meta < 0) + ? (!META_IS_STEADY(dest) && + !meta_weak_acceptable(env, dest, lck_exclusive)) + : false)) { mdbx_error("%s", "no usable meta-pages, database is corrupted"); if (rc == MDBX_SUCCESS) { /* TODO: try to restore the database by fully checking b-tree structure @@ -8473,7 +11731,7 @@ static MDBX_page *__cold mdbx_meta_model(const MDBX_env *env, MDBX_page *model, } /* Fill in most of the zeroed meta-pages for an empty database environment. - * Return pointer to recenly (head) meta-page. */ + * Return pointer to recently (head) meta-page. */ static MDBX_meta *__cold mdbx_init_metas(const MDBX_env *env, void *buffer) { MDBX_page *page0 = (MDBX_page *)buffer; MDBX_page *page1 = mdbx_meta_model(env, page0, 0); @@ -8492,6 +11750,7 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags, MDBX_meta *const meta1 = METAPAGE(env, 1); MDBX_meta *const meta2 = METAPAGE(env, 2); MDBX_meta *const head = mdbx_meta_head(env); + int rc; mdbx_assert(env, mdbx_meta_eq_mask(env) == 0); mdbx_assert(env, @@ -8499,7 +11758,7 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags, mdbx_assert(env, (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0); mdbx_assert(env, pending->mm_geo.next <= pending->mm_geo.now); - if (flags & (MDBX_SAFE_NOSYNC | MDBX_MAPASYNC)) { + if (flags & MDBX_SAFE_NOSYNC) { /* Check auto-sync conditions */ const pgno_t autosync_threshold = *env->me_autosync_threshold; const uint64_t autosync_period = *env->me_autosync_period; @@ -8529,20 +11788,20 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags, #if defined(MADV_DONTNEED) const size_t largest_bytes = pgno2bytes(env, largest_pgno); /* threshold to avoid unreasonable frequent madvise() calls */ - const size_t madvise_treshold = (largest_bytes < 65536 * 256) - ? 65536 - : (largest_bytes > MEGABYTE * 4 * 256) - ? MEGABYTE * 4 - : largest_bytes >> 10; + const size_t madvise_threshold = (largest_bytes < 65536 * 256) + ? 65536 + : (largest_bytes > MEGABYTE * 4 * 256) + ? MEGABYTE * 4 + : largest_bytes >> 10; const size_t discard_edge_bytes = bytes_align2os_bytes( env, ((MDBX_RDONLY & (env->me_lck ? env->me_lck->mti_envmode : env->me_flags)) ? largest_bytes - : largest_bytes + madvise_treshold)); + : largest_bytes + madvise_threshold)); const pgno_t discard_edge_pgno = bytes2pgno(env, discard_edge_bytes); const pgno_t prev_discarded_pgno = *env->me_discarded_tail; if (prev_discarded_pgno >= - discard_edge_pgno + bytes2pgno(env, madvise_treshold)) { + discard_edge_pgno + bytes2pgno(env, madvise_threshold)) { mdbx_notice("open-MADV_%s %u..%u", "DONTNEED", *env->me_discarded_tail, largest_pgno); *env->me_discarded_tail = discard_edge_pgno; @@ -8585,49 +11844,44 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags, flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED; shrink = pending->mm_geo.now - bottom; pending->mm_geo.now = bottom; - if (mdbx_meta_txnid_stable(env, head) == - pending->mm_txnid_a.inconsistent) - mdbx_meta_set_txnid( - env, pending, - safe64_txnid_next(pending->mm_txnid_a.inconsistent)); + if (unlikely(mdbx_meta_txnid_stable(env, head) == + pending->mm_txnid_a.inconsistent)) { + const txnid_t txnid = + safe64_txnid_next(pending->mm_txnid_a.inconsistent); + if (unlikely(txnid > MAX_TXNID)) { + mdbx_error("%s", "txnid overflow!"); + rc = MDBX_TXN_FULL; + goto fail; + } + mdbx_meta_set_txnid(env, pending, txnid); + } } } } } /* LY: step#1 - sync previously written/updated data-pages */ - int rc = *env->me_unsynced_pages ? MDBX_RESULT_TRUE /* carry non-steady */ - : MDBX_RESULT_FALSE /* carry steady */; - if (rc != MDBX_RESULT_FALSE && (flags & MDBX_SAFE_NOSYNC) == 0) { + rc = MDBX_RESULT_FALSE /* carry steady */; + if (*env->me_unsynced_pages) { mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); - MDBX_meta *const recent_steady_meta = mdbx_meta_steady(env); - if (flags & MDBX_WRITEMAP) { - const size_t begin = - floor_powerof2(pgno2bytes(env, NUM_METAS), env->me_os_psize); - const size_t end = pgno_align2os_bytes(env, pending->mm_geo.next); - if (end > begin) { - rc = mdbx_msync(&env->me_dxb_mmap, begin, end - begin, - flags & MDBX_MAPASYNC); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } - rc = MDBX_RESULT_TRUE /* carry non-steady */; - if ((flags & MDBX_MAPASYNC) == 0) { - if (unlikely(pending->mm_geo.next > recent_steady_meta->mm_geo.now)) { - rc = mdbx_filesync(env->me_lazy_fd, MDBX_SYNC_SIZE); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } - rc = MDBX_RESULT_FALSE /* carry steady */; - } - } else { - rc = mdbx_filesync(env->me_lazy_fd, - (pending->mm_geo.next > recent_steady_meta->mm_geo.now) - ? MDBX_SYNC_DATA | MDBX_SYNC_SIZE - : MDBX_SYNC_DATA); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; + enum mdbx_syncmode_bits mode_bits = MDBX_SYNC_NONE; + if ((flags & MDBX_SAFE_NOSYNC) == 0) { + mode_bits = MDBX_SYNC_DATA; + if (pending->mm_geo.next > mdbx_meta_steady(env)->mm_geo.now) + mode_bits |= MDBX_SYNC_SIZE; + if (flags & MDBX_NOMETASYNC) + mode_bits |= MDBX_SYNC_IODQ; } + if (flags & MDBX_WRITEMAP) + rc = + mdbx_msync(&env->me_dxb_mmap, 0, + pgno_align2os_bytes(env, pending->mm_geo.next), mode_bits); + else + rc = mdbx_fsync(env->me_lazy_fd, mode_bits); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + rc = (flags & MDBX_SAFE_NOSYNC) ? MDBX_RESULT_TRUE /* carry non-steady */ + : MDBX_RESULT_FALSE /* carry steady */; } /* Steady or Weak */ @@ -8740,24 +11994,13 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags, target->mm_datasync_sign = pending->mm_datasync_sign; mdbx_flush_incoherent_cpu_writeback(); mdbx_jitter4testing(true); - if ((flags & MDBX_SAFE_NOSYNC) == 0) { - /* sync meta-pages */ - const bool weak = (flags & (MDBX_MAPASYNC | MDBX_NOMETASYNC)) != 0; - rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), - weak); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - if (!weak) { -#if defined(__APPLE__) && \ - MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY - rc = likely(fcntl(env->me_lazy_fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS - : errno; - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; -#endif /* MacOS */ - *env->me_meta_sync_txnid = pending->mm_txnid_a.low; - } - } + /* sync meta-pages */ + rc = + mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), + (flags & MDBX_NOMETASYNC) ? MDBX_SYNC_NONE + : MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; } else { const MDBX_meta undo_meta = *target; const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE) @@ -8775,16 +12018,17 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags, goto fail; } mdbx_flush_incoherent_mmap(target, sizeof(MDBX_meta), env->me_os_psize); - if ((flags & (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC)) == 0) { - /* sync meta-pages */ - if (fd == env->me_lazy_fd) { - rc = mdbx_filesync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); - if (rc != MDBX_SUCCESS) - goto undo; - } - *env->me_meta_sync_txnid = pending->mm_txnid_a.low; + /* sync meta-pages */ + if ((flags & MDBX_NOMETASYNC) == 0 && fd == env->me_lazy_fd) { + rc = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + if (rc != MDBX_SUCCESS) + goto undo; } } + if (flags & MDBX_NOMETASYNC) + *env->me_unsynced_pages += 1; + else + *env->me_meta_sync_txnid = pending->mm_txnid_a.low; /* LY: shrink datafile if needed */ if (unlikely(shrink)) { @@ -8831,16 +12075,7 @@ static void __cold mdbx_setup_pagesize(MDBX_env *env, const size_t pagesize) { mdbx_ensure(env, branch_nodemax > 42 && branch_nodemax < (int)UINT16_MAX && branch_nodemax % 2 == 0); env->me_branch_nodemax = (unsigned)branch_nodemax; - env->me_maxkey_nd = (uint16_t)mdbx_limits_keysize_max(pagesize, 0); - env->me_maxkey_ds = (uint16_t)mdbx_limits_keysize_max(pagesize, MDBX_DUPSORT); - env->me_maxval_nd = (unsigned)mdbx_limits_valsize_max(pagesize, 0); - env->me_maxval_ds = (unsigned)mdbx_limits_valsize_max(pagesize, MDBX_DUPSORT); - mdbx_ensure(env, env->me_maxkey_nd == - env->me_branch_nodemax - NODESIZE - sizeof(pgno_t)); - mdbx_ensure(env, env->me_maxkey_ds == - env->me_branch_nodemax - NODESIZE - sizeof(MDBX_db)); - - env->me_psize2log = log2n(pagesize); + env->me_psize2log = (uint8_t)log2n(pagesize); mdbx_assert(env, pgno2bytes(env, 1) == pagesize); mdbx_assert(env, bytes2pgno(env, pagesize + pagesize) == 2); } @@ -8856,6 +12091,7 @@ int __cold mdbx_env_create(MDBX_env **penv) { env->me_dsync_fd = INVALID_HANDLE_VALUE; env->me_lfd = INVALID_HANDLE_VALUE; env->me_pid = mdbx_getpid(); + env->me_stuck_meta = -1; int rc; const size_t os_psize = mdbx_syspagesize(); @@ -8906,19 +12142,9 @@ __cold LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper, intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; - -#if MDBX_TXN_CHECKPID - if (unlikely(env->me_pid != mdbx_getpid())) - env->me_flags |= MDBX_FATAL_ERROR; -#endif /* MDBX_TXN_CHECKPID */ - - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) - return MDBX_PANIC; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; const bool inside_txn = (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self()); @@ -8931,7 +12157,7 @@ mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, #endif bool need_unlock = false; - int rc = MDBX_PROBLEM; + rc = MDBX_PROBLEM; if (env->me_map) { /* env already mapped */ if (unlikely(env->me_flags & MDBX_RDONLY)) @@ -9234,8 +12460,14 @@ mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, env->me_txn->mt_flags |= MDBX_TXN_DIRTY; } else { meta.mm_geo = new_geo; - mdbx_meta_set_txnid( - env, &meta, safe64_txnid_next(mdbx_meta_txnid_stable(env, head))); + const txnid_t txnid = + safe64_txnid_next(mdbx_meta_txnid_stable(env, head)); + if (unlikely(txnid > MAX_TXNID)) { + mdbx_error("%s", "txnid overflow!"); + rc = MDBX_TXN_FULL; + goto bailout; + } + mdbx_meta_set_txnid(env, &meta, txnid); rc = mdbx_sync_locked(env, env->me_flags, &meta); } } @@ -9254,15 +12486,13 @@ int __cold mdbx_env_set_mapsize(MDBX_env *env, size_t size) { } int __cold mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) { - if (unlikely(dbs > MAX_DBI)) - return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(!env)) + if (unlikely(dbs > MDBX_MAX_DBI)) return MDBX_EINVAL; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; - if (unlikely(env->me_map)) return MDBX_EPERM; @@ -9270,15 +12500,25 @@ int __cold mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) { return MDBX_SUCCESS; } -int __cold mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) { - if (unlikely(readers < 1 || readers > MDBX_READERS_LIMIT)) - return MDBX_EINVAL; +int __cold mdbx_env_get_maxdbs(MDBX_env *env, MDBX_dbi *dbs) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(!env)) + if (unlikely(!dbs)) return MDBX_EINVAL; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + *dbs = env->me_maxdbs; + return MDBX_SUCCESS; +} + +int __cold mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(readers < 1 || readers > MDBX_READERS_LIMIT)) + return MDBX_EINVAL; if (unlikely(env->me_map)) return MDBX_EPERM; @@ -9288,11 +12528,12 @@ int __cold mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) { } int __cold mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers) { - if (!env || !readers) - return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if (unlikely(!readers)) + return MDBX_EINVAL; *readers = env->me_maxreaders; return MDBX_SUCCESS; @@ -9306,7 +12547,8 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { int err = mdbx_read_header(env, &meta, &filesize_before, lck_rc); if (unlikely(err != MDBX_SUCCESS)) { if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA || - (env->me_flags & MDBX_RDONLY) != 0) + (env->me_flags & MDBX_RDONLY) != 0 || + /* recovery mode */ env->me_stuck_meta >= 0) return err; mdbx_debug("%s", "create new database"); @@ -9353,7 +12595,8 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { const size_t used_aligned2os_bytes = ceil_powerof2(used_bytes, env->me_os_psize); if ((env->me_flags & MDBX_RDONLY) /* readonly */ - || lck_rc != MDBX_RESULT_TRUE /* not exclusive */) { + || lck_rc != MDBX_RESULT_TRUE /* not exclusive */ + || /* recovery mode */ env->me_stuck_meta >= 0) { /* use present params from db */ const size_t pagesize = meta.mm_psize; err = mdbx_env_set_geometry( @@ -9361,7 +12604,7 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { meta.mm_geo.upper * pagesize, meta.mm_geo.grow * pagesize, meta.mm_geo.shrink * pagesize, meta.mm_psize); if (unlikely(err != MDBX_SUCCESS)) { - mdbx_error("%s: err %d", "could not apply preconfigured db-geometry", + mdbx_error("%s: err %d", "could not apply preconfigured geometry from db", err); return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err; } @@ -9440,10 +12683,10 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now), filesize_before, bytes2pgno(env, (size_t)filesize_before)); } else { - mdbx_notice("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO - "p, have %" PRIu64 "b/%" PRIaPGNO "p)", - env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now), - filesize_before, bytes2pgno(env, (size_t)filesize_before)); + mdbx_warning("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO + "p, have %" PRIu64 "b/%" PRIaPGNO "p)", + env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now), + filesize_before, bytes2pgno(env, (size_t)filesize_before)); if (filesize_before < used_bytes) { mdbx_error("last-page beyond end-of-file (last %" PRIaPGNO ", have %" PRIaPGNO ")", @@ -9507,12 +12750,17 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { #endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ const unsigned meta_clash_mask = mdbx_meta_eq_mask(env); - if (meta_clash_mask) { - mdbx_error("meta-pages are clashed: mask 0x%d", meta_clash_mask); - return MDBX_CORRUPTED; + if (unlikely(meta_clash_mask)) { + if (/* not recovery mode */ env->me_stuck_meta < 0) { + mdbx_error("meta-pages are clashed: mask 0x%d", meta_clash_mask); + return MDBX_CORRUPTED; + } else { + mdbx_warning("ignore meta-pages clashing (mask 0x%d) in recovery mode", + meta_clash_mask); + } } - while (1) { + while (likely(/* not recovery mode */ env->me_stuck_meta < 0)) { MDBX_meta *const head = mdbx_meta_head(env); const txnid_t head_txnid = mdbx_meta_txnid_fluid(env, head); MDBX_meta *const steady = mdbx_meta_steady(env); @@ -9522,13 +12770,6 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) { mdbx_assert(env, META_IS_STEADY(&meta) && !META_IS_STEADY(head)); - if (env->me_flags & MDBX_RDONLY) { - mdbx_error("rollback needed: (from head %" PRIaTXN - " to steady %" PRIaTXN "), but unable in read-only mode", - head_txnid, steady_txnid); - return MDBX_WANNA_RECOVERY /* LY: could not recovery/rollback */; - } - if (meta_bootid_match(head)) { MDBX_meta clone = *head; uint64_t filesize = env->me_dbgeo.now; @@ -9537,18 +12778,29 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { bytes2pgno(env, (uint8_t *)data_page(head) - env->me_map), nullptr, env->me_psize); if (err == MDBX_SUCCESS) { - mdbx_notice("opening after an unclean shutdown, " - "but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH, " - "rollback NOT needed", - bootid.x, bootid.y); + mdbx_warning( + "opening after an unclean shutdown, but boot-id(%016" PRIx64 + "-%016" PRIx64 + ") is MATCH: rollback NOT needed, steady-sync NEEDED%s", + bootid.x, bootid.y, + (env->me_flags & MDBX_RDONLY) ? ", but unable in read-only mode" + : ""); + if (env->me_flags & MDBX_RDONLY) + return MDBX_WANNA_RECOVERY /* LY: could not recovery/sync */; meta = clone; *env->me_unsynced_pages = meta.mm_geo.next; break; } - mdbx_notice("opening after an unclean shutdown, " - "but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH, " - "but last meta not valid, rollback needed", - bootid.x, bootid.y); + mdbx_warning("opening after an unclean shutdown, " + "but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH, " + "but last meta not valid, rollback needed", + bootid.x, bootid.y); + } + if (env->me_flags & MDBX_RDONLY) { + mdbx_error("rollback needed: (from head %" PRIaTXN + " to steady %" PRIaTXN "), but unable in read-only mode", + head_txnid, steady_txnid); + return MDBX_WANNA_RECOVERY /* LY: could not recovery/rollback */; } const MDBX_meta *const meta0 = METAPAGE(env, 0); @@ -9567,8 +12819,8 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { } /* LY: rollback weak checkpoint */ - mdbx_trace("rollback: from %" PRIaTXN ", to %" PRIaTXN " as %" PRIaTXN, - head_txnid, steady_txnid, undo_txnid); + mdbx_notice("rollback: from %" PRIaTXN ", to %" PRIaTXN " as %" PRIaTXN, + head_txnid, steady_txnid, undo_txnid); mdbx_ensure(env, head_txnid == mdbx_meta_txnid_stable(env, head)); if (env->me_flags & MDBX_WRITEMAP) { @@ -9604,7 +12856,7 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { } if (!env->me_lck) { - /* LY: without-lck (read-only) mode, so it is imposible that other + /* LY: without-lck (read-only) mode, so it is impossible that other * process made weak checkpoint. */ mdbx_error("%s", "without-lck, unable recovery/rollback"); return MDBX_WANNA_RECOVERY; @@ -9625,41 +12877,57 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { env->me_dxb_mmap.current); return MDBX_PROBLEM; } - if (env->me_dxb_mmap.current != env->me_dbgeo.now && - (env->me_flags & MDBX_RDONLY) == 0) { + if (env->me_dxb_mmap.current != env->me_dbgeo.now) { meta.mm_geo.now = bytes2pgno(env, env->me_dxb_mmap.current); - mdbx_verbose("update meta-geo to filesize %" PRIuPTR " bytes, %" PRIaPGNO - " pages", - env->me_dxb_mmap.current, meta.mm_geo.now); + mdbx_notice("need update meta-geo to filesize %" PRIuPTR + " bytes, %" PRIaPGNO " pages", + env->me_dxb_mmap.current, meta.mm_geo.now); } if (memcmp(&meta.mm_geo, &head->mm_geo, sizeof(meta.mm_geo))) { - const txnid_t txnid = mdbx_meta_txnid_stable(env, head); - const txnid_t next_txnid = safe64_txnid_next(txnid); - mdbx_verbose("updating meta.geo: " - "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN "), " - "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN ")", - head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper, - head->mm_geo.shrink, head->mm_geo.grow, txnid, - meta.mm_geo.lower, meta.mm_geo.now, meta.mm_geo.upper, - meta.mm_geo.shrink, meta.mm_geo.grow, next_txnid); - - mdbx_ensure(env, mdbx_meta_eq(env, &meta, head)); - mdbx_meta_set_txnid(env, &meta, next_txnid); - err = mdbx_sync_locked(env, env->me_flags | MDBX_SHRINK_ALLOWED, &meta); - if (err) { - mdbx_error("error %d, while updating meta.geo: " - "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN "), " - "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN ")", - err, head->mm_geo.lower, head->mm_geo.now, - head->mm_geo.upper, head->mm_geo.shrink, head->mm_geo.grow, - txnid, meta.mm_geo.lower, meta.mm_geo.now, meta.mm_geo.upper, - meta.mm_geo.shrink, meta.mm_geo.grow, next_txnid); - return err; + if ((env->me_flags & MDBX_RDONLY) != 0 || + /* recovery mode */ env->me_stuck_meta >= 0) { + mdbx_warning("skipped update meta.geo in %s mode: from l%" PRIaPGNO + "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u, to l%" PRIaPGNO + "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u", + (env->me_stuck_meta < 0) ? "read-only" : "recovery", + head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper, + head->mm_geo.shrink, head->mm_geo.grow, meta.mm_geo.lower, + meta.mm_geo.now, meta.mm_geo.upper, meta.mm_geo.shrink, + meta.mm_geo.grow); + } else { + const txnid_t txnid = mdbx_meta_txnid_stable(env, head); + const txnid_t next_txnid = safe64_txnid_next(txnid); + if (unlikely(txnid > MAX_TXNID)) { + mdbx_error("%s", "txnid overflow!"); + return MDBX_TXN_FULL; + } + mdbx_notice("updating meta.geo: " + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN ")", + head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper, + head->mm_geo.shrink, head->mm_geo.grow, txnid, + meta.mm_geo.lower, meta.mm_geo.now, meta.mm_geo.upper, + meta.mm_geo.shrink, meta.mm_geo.grow, next_txnid); + + mdbx_ensure(env, mdbx_meta_eq(env, &meta, head)); + mdbx_meta_set_txnid(env, &meta, next_txnid); + err = mdbx_sync_locked(env, env->me_flags | MDBX_SHRINK_ALLOWED, &meta); + if (err) { + mdbx_error("error %d, while updating meta.geo: " + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN ")", + err, head->mm_geo.lower, head->mm_geo.now, + head->mm_geo.upper, head->mm_geo.shrink, head->mm_geo.grow, + txnid, meta.mm_geo.lower, meta.mm_geo.now, + meta.mm_geo.upper, meta.mm_geo.shrink, meta.mm_geo.grow, + next_txnid); + return err; + } } } } @@ -9667,8 +12935,10 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { *env->me_discarded_tail = bytes2pgno(env, used_aligned2os_bytes); if (used_aligned2os_bytes < env->me_dxb_mmap.current) { #if defined(MADV_REMOVE) - if (lck_rc && (env->me_flags & MDBX_WRITEMAP) != 0) { - mdbx_notice("open-MADV_%s %u..%u", "REMOVE", *env->me_discarded_tail, + if (lck_rc && (env->me_flags & MDBX_WRITEMAP) != 0 && + /* not recovery mode */ env->me_stuck_meta < 0) { + mdbx_notice("open-MADV_%s %u..%u", "REMOVE (deallocate file space)", + *env->me_discarded_tail, bytes2pgno(env, env->me_dxb_mmap.current)); err = madvise(env->me_map + used_aligned2os_bytes, @@ -9718,7 +12988,7 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { /* Open and/or initialize the lock region for the environment. */ static int __cold mdbx_setup_lck(MDBX_env *env, char *lck_pathname, - mode_t mode) { + mdbx_mode_t mode) { mdbx_assert(env, env->me_lazy_fd != INVALID_HANDLE_VALUE); mdbx_assert(env, env->me_lfd == INVALID_HANDLE_VALUE); @@ -9835,7 +13105,8 @@ static int __cold mdbx_setup_lck(MDBX_env *env, char *lck_pathname, } env->me_maxreaders = (unsigned)maxreaders; - err = mdbx_mmap((env->me_flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, &env->me_lck_mmap, (size_t)size, (size_t)size, + err = mdbx_mmap((env->me_flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, + &env->me_lck_mmap, (size_t)size, (size_t)size, lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE : MMAP_OPTION_SEMAPHORE); if (unlikely(err != MDBX_SUCCESS)) @@ -9857,7 +13128,7 @@ static int __cold mdbx_setup_lck(MDBX_env *env, char *lck_pathname, struct MDBX_lockinfo *const lck = env->me_lck; if (lck_seize_rc == MDBX_RESULT_TRUE) { - /* LY: exlcusive mode, check and reset lck content */ + /* LY: exclusive mode, check and reset lck content */ memset(lck, 0, (size_t)size); mdbx_jitter4testing(false); lck->mti_magic_and_version = MDBX_LOCK_MAGIC; @@ -9867,7 +13138,7 @@ static int __cold mdbx_setup_lck(MDBX_env *env, char *lck_pathname, mdbx_error("initial-%s for lck-file failed", "msync"); goto bailout; } - err = mdbx_filesync(env->me_lck_mmap.fd, MDBX_SYNC_SIZE); + err = mdbx_fsync(env->me_lck_mmap.fd, MDBX_SYNC_SIZE); if (unlikely(err != MDBX_SUCCESS)) { mdbx_error("initial-%s for lck-file failed", "fsync"); goto bailout; @@ -10028,46 +13299,109 @@ __cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) { : MDBX_RESULT_TRUE; } -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC | MDBX_NOMEMINIT | \ - MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS -#error "Opps, some flags overlapped or wrong" -#endif - -#if (MDBX_ACCEDE | MDBX_CREATE) != ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & \ - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)) -#error "Opps, some flags overlapped or wrong" -#endif - -/* Merge flags and avoid false MDBX_UTTERLY_NOSYNC */ -static uint32_t merge_flags(const uint32_t a, const uint32_t b) { +/* Merge sync flags */ +static uint32_t merge_sync_flags(const uint32_t a, const uint32_t b) { uint32_t r = a | b; + + /* avoid false MDBX_UTTERLY_NOSYNC */ if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC)) - r -= (r & MDBX_WRITEMAP) ? MDBX_UTTERLY_NOSYNC ^ MDBX_MAPASYNC - : MDBX_UTTERLY_NOSYNC ^ MDBX_SAFE_NOSYNC; + r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC; + + /* convert MDBX_DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */ + if ((r & (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC)) == + (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC) && + !F_ISSET(r, MDBX_UTTERLY_NOSYNC)) + r = (r - MDBX_DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC; + + /* force MDBX_NOMETASYNC if MDBX_SAFE_NOSYNC enabled */ + if (r & MDBX_SAFE_NOSYNC) + r |= MDBX_NOMETASYNC; + assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC))); return r; } -int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, - mode_t mode) { - if (unlikely(!env || !pathname)) +__cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target_meta) { + if (unlikely(target_meta >= NUM_METAS)) return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if ((env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_EXCLUSIVE) + return MDBX_EPERM; + + MDBX_page *page = + (env->me_flags & MDBX_WRITEMAP) + ? pgno2page(env, target_meta) + : memcpy(env->me_pbuf, pgno2page(env, target_meta), env->me_psize); + page->mp_pgno = target_meta; + page->mp_flags = P_META; + + MDBX_meta *meta = page_meta(page); + meta->mm_magic_and_version = MDBX_DATA_MAGIC; + meta->mm_psize = env->me_psize; + txnid_t txnid = mdbx_meta_txnid_stable(env, meta); + const txnid_t txnid0 = mdbx_meta_txnid_stable(env, METAPAGE(env, 0)); + if (target_meta != 0 && txnid <= txnid0) + txnid = safe64_txnid_next(txnid0); + const txnid_t txnid1 = mdbx_meta_txnid_stable(env, METAPAGE(env, 1)); + if (target_meta != 1 && txnid <= txnid1) + txnid = safe64_txnid_next(txnid1); + const txnid_t txnid2 = mdbx_meta_txnid_stable(env, METAPAGE(env, 2)); + if (target_meta != 2 && txnid <= txnid2) + txnid = safe64_txnid_next(txnid2); + + if (!META_IS_STEADY(meta) || mdbx_recent_committed_txnid(env) != txnid) { + if (unlikely(txnid > MAX_TXNID)) { + mdbx_error("%s", "txnid overflow!"); + return MDBX_TXN_FULL; + } + mdbx_meta_set_txnid(env, meta, txnid); + meta->mm_datasync_sign = mdbx_meta_sign(meta); + } + + if (env->me_flags & MDBX_WRITEMAP) { + mdbx_flush_incoherent_cpu_writeback(); + rc = mdbx_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, target_meta), + MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + } else { + const mdbx_filehandle_t fd = (env->me_dsync_fd != INVALID_HANDLE_VALUE) + ? env->me_dsync_fd + : env->me_lazy_fd; + rc = mdbx_pwrite(fd, page, env->me_psize, pgno2bytes(env, target_meta)); + if (rc == MDBX_SUCCESS && fd == env->me_lazy_fd) + rc = mdbx_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + } + + return rc; +} + +__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, + unsigned target_meta, bool writeable) { + if (unlikely(target_meta >= NUM_METAS)) + return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + env->me_stuck_meta = (int8_t)target_meta; + return mdbx_env_open( + env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY, + 0); +} + +__cold int mdbx_env_open(MDBX_env *env, const char *pathname, + MDBX_env_flags_t flags, mdbx_mode_t mode) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!pathname)) + return MDBX_EINVAL; if (flags & ~ENV_USABLE_FLAGS) return MDBX_EINVAL; @@ -10087,12 +13421,12 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, /* pickup previously mdbx_env_set_flags(), * but avoid MDBX_UTTERLY_NOSYNC by disjunction */ - flags = merge_flags(flags, env->me_flags); + flags = merge_sync_flags(flags, env->me_flags); #if defined(_WIN32) || defined(_WIN64) const DWORD dwAttrib = GetFileAttributesW(pathnameW); if (dwAttrib == INVALID_FILE_ATTRIBUTES) { - int rc = GetLastError(); + rc = GetLastError(); if (rc != MDBX_ENOFILE) return rc; if (mode == 0 || (flags & MDBX_RDONLY) != 0) @@ -10114,7 +13448,7 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, #else struct stat st; if (stat(pathname, &st)) { - int rc = errno; + rc = errno; if (rc != MDBX_ENOFILE) return rc; if (mode == 0 || (flags & MDBX_RDONLY) != 0) @@ -10122,7 +13456,7 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, return rc; /* auto-create directory if requested */ - const mode_t dir_mode = + const mdbx_mode_t dir_mode = (/* inherit read/write permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | /* always add read/write/search for owner */ S_IRWXU | @@ -10162,13 +13496,13 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, sprintf(dxb_pathname, "%s" MDBX_DATANAME, pathname); } - int rc = MDBX_SUCCESS; + rc = MDBX_SUCCESS; if (flags & MDBX_RDONLY) { /* LY: silently ignore irrelevant flags when * we're only getting read access */ - flags &= - ~(MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | - MDBX_COALESCE | MDBX_LIFORECLAIM | MDBX_NOMEMINIT | MDBX_ACCEDE); + flags &= ~(MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC | + MDBX_NOMETASYNC | MDBX_COALESCE | MDBX_LIFORECLAIM | + MDBX_NOMEMINIT | MDBX_ACCEDE); } else { #if MDBX_MMAP_INCOHERENT_FILE_WRITE /* Temporary `workaround` for OpenBSD kernel's flaw. @@ -10235,14 +13569,13 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, rc = errno; goto bailout; } - mode = (/* inherit read permissions for group and others */ st.st_mode & - (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | - /* always add read/write/search for owner */ S_IRUSR | S_IWUSR | - ((st.st_mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP - : 0) | - ((st.st_mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH - : 0); + mode = st.st_mode; } + mode = (/* inherit read permissions for group and others */ mode & + (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | + /* always add read/write/search for owner */ S_IRUSR | S_IWUSR | + ((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) | + ((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0); #endif /* !Windows */ const int lck_rc = mdbx_setup_lck(env, lck_pathname, mode); if (MDBX_IS_ERROR(lck_rc)) { @@ -10250,9 +13583,11 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, goto bailout; } - const unsigned rigorous_flags = MDBX_SAFE_NOSYNC | MDBX_MAPASYNC; - const unsigned mode_flags = rigorous_flags | MDBX_NOMETASYNC | - MDBX_LIFORECLAIM | MDBX_COALESCE | MDBX_NORDAHEAD; + const MDBX_env_flags_t rigorous_flags = + MDBX_SAFE_NOSYNC | MDBX_DEPRECATED_MAPASYNC; + const MDBX_env_flags_t mode_flags = rigorous_flags | MDBX_NOMETASYNC | + MDBX_LIFORECLAIM | MDBX_COALESCE | + MDBX_NORDAHEAD; if (env->me_lck && lck_rc != MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) { @@ -10286,6 +13621,14 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, goto bailout; } + if (unlikely(/* recovery mode */ env->me_stuck_meta >= 0) && + (lck_rc != /* exclusive */ MDBX_RESULT_TRUE || + (flags & MDBX_EXCLUSIVE) == 0)) { + mdbx_error("%s", "recovery requires exclusive mode"); + rc = MDBX_BUSY; + goto bailout; + } + mdbx_debug("opened dbenv %p", (void *)env); if (env->me_lck) { if (lck_rc == MDBX_RESULT_TRUE) { @@ -10296,7 +13639,7 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, if (rc != MDBX_SUCCESS) goto bailout; } else { - rc = mdbx_reader_check0(env, false, NULL); + rc = mdbx_cleanup_dead_readers(env, false, NULL); if (MDBX_IS_ERROR(rc)) goto bailout; } @@ -10317,7 +13660,7 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, sizeof(unsigned) + 1); rc = mdbx_memalign_alloc( env->me_os_psize, - env->me_psize * (1 /* page buffer */ + 1 /* page killer bufer */), + env->me_psize * (1 /* page buffer */ + 1 /* page killer buffer */), &env->me_pbuf); if (rc == MDBX_SUCCESS) { memset(env->me_pbuf, -1, env->me_psize * 2); @@ -10362,7 +13705,8 @@ int __cold mdbx_env_open(MDBX_env *env, const char *pathname, unsigned flags, bailout: if (rc != MDBX_SUCCESS) { rc = mdbx_env_close0(env) ? MDBX_PANIC : rc; - env->me_flags = saved_me_flags | MDBX_FATAL_ERROR; + env->me_flags = + saved_me_flags | ((rc != MDBX_PANIC) ? 0 : MDBX_FATAL_ERROR); } else { #if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) mdbx_txn_valgrind(env, nullptr); @@ -10374,6 +13718,7 @@ bailout: /* Destroy resources from mdbx_env_open(), clear our readers & DBIs */ static int __cold mdbx_env_close0(MDBX_env *env) { + env->me_stuck_meta = -1; if (!(env->me_flags & MDBX_ENV_ACTIVE)) { mdbx_ensure(env, env->me_lcklist_next == nullptr); return MDBX_SUCCESS; @@ -10441,7 +13786,7 @@ static int __cold mdbx_env_close0(MDBX_env *env) { return rc; } -int __cold mdbx_env_close_ex(MDBX_env *env, int dont_sync) { +int __cold mdbx_env_close_ex(MDBX_env *env, bool dont_sync) { MDBX_page *dp; int rc = MDBX_SUCCESS; @@ -10451,14 +13796,14 @@ int __cold mdbx_env_close_ex(MDBX_env *env, int dont_sync) { if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) return MDBX_EBADSIGN; -#if MDBX_TXN_CHECKPID || !(defined(_WIN32) || defined(_WIN64)) - /* Check the PID even if MDBX_TXN_CHECKPID=0 on non-Windows +#if MDBX_ENV_CHECKPID || !(defined(_WIN32) || defined(_WIN64)) + /* Check the PID even if MDBX_ENV_CHECKPID=0 on non-Windows * platforms (i.e. where fork() is available). * This is required to legitimize a call after fork() * from a child process, that should be allowed to free resources. */ if (unlikely(env->me_pid != mdbx_getpid())) env->me_flags |= MDBX_FATAL_ERROR; -#endif /* MDBX_TXN_CHECKPID */ +#endif /* MDBX_ENV_CHECKPID */ if ((env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0 && env->me_txn0) { if (env->me_txn0->mt_owner && env->me_txn0->mt_owner != mdbx_thread_self()) @@ -10609,6 +13954,12 @@ static int __hot cmp_lenfast(const MDBX_val *a, const MDBX_val *b) { return likely(diff) ? diff : memcmp(a->iov_base, b->iov_base, a->iov_len); } +static bool unsure_equal(MDBX_cmp_func cmp, const MDBX_val *a, + const MDBX_val *b) { + return cmp == cmp_lenfast || cmp == cmp_lexical || cmp == cmp_reverse || + cmp == cmp_int_unaligned || cmp_lenfast(a, b) == 0; +} + /* Search for key within a page, using binary search. * Returns the smallest entry larger or equal to the key. * If exactp is non-null, stores whether the found entry was an exact match @@ -10762,22 +14113,25 @@ static int mdbx_cursor_push(MDBX_cursor *mc, MDBX_page *mp) { * * Returns 0 on success, non-zero on failure. */ __hot static int mdbx_page_get(MDBX_cursor *mc, pgno_t pgno, MDBX_page **ret, - int *lvl) { + int *lvl, const txnid_t pp_txnid) { MDBX_txn *txn = mc->mc_txn; if (unlikely(pgno >= txn->mt_next_pgno)) { - mdbx_debug("page %" PRIaPGNO " not found", pgno); - goto corrupted; + mdbx_error("page #%" PRIaPGNO " beyond next-pgno", pgno); + notfound: + *ret = nullptr; + txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_NOTFOUND; } MDBX_env *const env = txn->mt_env; - MDBX_page *p = NULL; - int level; + MDBX_page *p = nullptr; mdbx_assert(env, ((txn->mt_flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); - const uint16_t illegal_bits = (txn->mt_flags & MDBX_RDONLY) + mdbx_assert(env, pp_txnid >= MIN_TXNID && pp_txnid <= txn->mt_txnid); + const uint16_t illegal_bits = (txn->mt_flags & MDBX_TXN_RDONLY) ? P_LOOSE | P_SUBP | P_META | P_DIRTY : P_LOOSE | P_SUBP | P_META; - const uint64_t txnid = txn->mt_txnid; - if (unlikely((txn->mt_flags & (MDBX_RDONLY | MDBX_WRITEMAP)) == 0)) { + int level; + if (unlikely((txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0)) { level = 1; do { /* Spilled pages were dirtied in this txn and flushed @@ -10798,18 +14152,25 @@ spilled: p = pgno2page(env, pgno); dirty: + *(lvl ? lvl : &level) = level; + *ret = p; + if (unlikely(p->mp_pgno != pgno)) { - mdbx_error("mismatch pgno %" PRIaPGNO " (actual) != %" PRIaPGNO - " (expected)", - p->mp_pgno, pgno); + bad_page( + p, "mismatch actual pgno (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n", + p->mp_pgno, pgno); + goto notfound; + } + + if (unlikely(p->mp_flags & illegal_bits)) { + bad_page(p, "invalid page's flags (%u)\n", p->mp_flags); goto corrupted; } - if (unlikely((p->mp_flags & illegal_bits) != 0 || - p->mp_txnid > ((p->mp_flags & P_DIRTY) ? UINT64_MAX : txnid))) { - mdbx_error("invalid page's flags (0x%x) or txnid %" PRIaTXN - " > (actual) %" PRIaTXN " (expected)", - p->mp_flags, p->mp_txnid, mc->mc_txn->mt_txnid); + if (unlikely(p->mp_txnid > + ((p->mp_flags & P_DIRTY) ? UINT64_MAX : pp_txnid))) { + bad_page(p, "page mod-txnid (%" PRIaTXN ") > parent (%" PRIaTXN ")\n", + p->mp_txnid, pp_txnid); goto corrupted; } @@ -10817,23 +14178,18 @@ dirty: ((p->mp_lower | p->mp_upper) & 1) != 0 || PAGEHDRSZ + p->mp_upper > env->me_psize) && !IS_OVERFLOW(p))) { - mdbx_error("invalid page lower(%u)/upper(%u), pg-limit %u", p->mp_lower, - p->mp_upper, page_space(env)); + bad_page(p, "invalid page lower(%u)/upper(%u) with limit (%u)\n", + p->mp_lower, p->mp_upper, page_space(env)); goto corrupted; } - if (mdbx_audit_enabled()) { - int err = mdbx_page_check(mc, p, C_UPDATING); - if (unlikely(err != MDBX_SUCCESS)) - return err; - } + if (mdbx_audit_enabled()) + return mdbx_page_check(mc, p, C_UPDATING); - *(lvl ? lvl : &level) = level; - *ret = p; return MDBX_SUCCESS; corrupted: - txn->mt_flags |= MDBX_TXN_ERROR; + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; return MDBX_CORRUPTED; } @@ -10888,7 +14244,8 @@ __hot static int mdbx_page_search_root(MDBX_cursor *mc, const MDBX_val *key, mdbx_cassert(mc, i < (int)page_numkeys(mp)); node = page_node(mp, i); - if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp, NULL)) != 0)) + if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp, NULL, + pp_txnid4chk(mp, mc->mc_txn))) != 0)) return rc; mc->mc_ki[mc->mc_top] = (indx_t)i; @@ -10904,10 +14261,9 @@ __hot static int mdbx_page_search_root(MDBX_cursor *mc, const MDBX_val *key, } if (unlikely(!IS_LEAF(mp))) { - mdbx_debug("internal error, index points to a page with 0x%02x flags!?", - mp->mp_flags); mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - return MDBX_CORRUPTED; + return bad_page(mp, "index points to a page with 0x%02x flags\n", + mp->mp_flags); } mdbx_debug("found leaf page %" PRIaPGNO " for key [%s]", mp->mp_pgno, @@ -10938,8 +14294,11 @@ static int mdbx_setup_dbx(MDBX_dbx *const dbx, const MDBX_db *const db, if ((db->md_flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->md_xsize) { if (unlikely(db->md_xsize < dbx->md_vlen_min || - db->md_xsize > dbx->md_vlen_max)) + db->md_xsize > dbx->md_vlen_max)) { + mdbx_error("db.md_xsize (%u) <> min/max value-length (%zu/%zu)", + db->md_xsize, dbx->md_vlen_min, dbx->md_vlen_max); return MDBX_CORRUPTED; + } dbx->md_vlen_min = dbx->md_vlen_max = db->md_xsize; } return MDBX_SUCCESS; @@ -10965,11 +14324,14 @@ static int mdbx_fetch_sdb(MDBX_txn *txn, MDBX_dbi dbi) { return MDBX_BAD_DBI; if (unlikely((node_flags(node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) return MDBX_INCOMPATIBLE; /* not a named DB */ - rc = mdbx_node_read(&couple.outer, node, &data); + + const txnid_t pp_txnid = + pp_txnid4chk(couple.outer.mc_pg[couple.outer.mc_top], txn); + rc = mdbx_node_read(&couple.outer, node, &data, pp_txnid); if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (unlikely(data.iov_len < sizeof(MDBX_db))) + if (unlikely(data.iov_len != sizeof(MDBX_db))) return MDBX_INCOMPATIBLE; /* not a named DB */ uint16_t md_flags = UNALIGNED_PEEK_16(data.iov_base, MDBX_db, md_flags); @@ -10980,6 +14342,12 @@ static int mdbx_fetch_sdb(MDBX_txn *txn, MDBX_dbi dbi) { return MDBX_INCOMPATIBLE; memcpy(db, data.iov_base, sizeof(MDBX_db)); + mdbx_tassert(txn, txn->mt_txnid >= pp_txnid); + if (unlikely(db->md_mod_txnid > pp_txnid)) { + mdbx_error("db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", + db->md_mod_txnid, pp_txnid); + return MDBX_CORRUPTED; + } rc = mdbx_setup_dbx(dbx, db, txn->mt_env->me_psize); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -10999,7 +14367,8 @@ __hot static int mdbx_page_search_lowest(MDBX_cursor *mc) { MDBX_node *node = page_node(mp, 0); int rc; - if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp, NULL)) != 0)) + if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp, NULL, + pp_txnid4chk(mp, mc->mc_txn))) != 0)) return rc; mc->mc_ki[mc->mc_top] = 0; @@ -11048,9 +14417,15 @@ __hot static int mdbx_page_search(MDBX_cursor *mc, const MDBX_val *key, } mdbx_cassert(mc, root >= NUM_METAS); - if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) - if (unlikely((rc = mdbx_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0)) + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) { + if (unlikely((rc = mdbx_page_get( + mc, root, &mc->mc_pg[0], NULL, + (/* maybe zero in legacy DB */ mc->mc_db->md_mod_txnid && + !(*mc->mc_dbistate & DBI_DIRTY)) + ? mc->mc_db->md_mod_txnid + : mc->mc_txn->mt_txnid)) != 0)) return rc; + } mc->mc_snum = 1; mc->mc_top = 0; @@ -11077,13 +14452,14 @@ __hot static int mdbx_page_search(MDBX_cursor *mc, const MDBX_val *key, * * Returns 0 on success, non-zero on failure. */ static __always_inline int mdbx_node_read(MDBX_cursor *mc, MDBX_node *node, - MDBX_val *data) { + MDBX_val *data, + const txnid_t pp_txnid) { data->iov_len = node_ds(node); data->iov_base = node_data(node); if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) { /* Read overflow data. */ MDBX_page *omp; /* overflow page */ - int rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL); + int rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL, pp_txnid); if (unlikely((rc != MDBX_SUCCESS))) { mdbx_debug("read overflow page %" PRIaPGNO " failed", node_largedata_pgno(node)); @@ -11106,7 +14482,7 @@ int mdbx_get(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) { return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; MDBX_cursor_couple cx; rc = mdbx_cursor_init(&cx.outer, txn, dbi); @@ -11117,8 +14493,8 @@ int mdbx_get(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) { return mdbx_cursor_set(&cx.outer, (MDBX_val *)key, data, MDBX_SET, &exact); } -int mdbx_get_nearest(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data) { +int mdbx_get_equal_or_great(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data) { DKBUF; mdbx_debug("===> get db %u key [%s]", dbi, DKEY(key)); @@ -11130,7 +14506,7 @@ int mdbx_get_nearest(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED)) return MDBX_BAD_TXN; @@ -11170,7 +14546,7 @@ int mdbx_get_ex(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; MDBX_cursor_couple cx; rc = mdbx_cursor_init(&cx.outer, txn, dbi); @@ -11246,8 +14622,9 @@ static int mdbx_cursor_sibling(MDBX_cursor *mc, int move_right) { } mdbx_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); - indx = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (unlikely((rc = mdbx_page_get(mc, node_pgno(indx), &mp, NULL)) != 0)) { + indx = page_node(mp = mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (unlikely((rc = mdbx_page_get(mc, node_pgno(indx), &mp, NULL, + pp_txnid4chk(mp, mc->mc_txn))) != 0)) { /* mc will be inconsistent if caller does mc_snum++ as above */ mc->mc_flags &= ~(C_INITIALIZED | C_EOF); return rc; @@ -11325,6 +14702,9 @@ skip: " with %u keys, key index %u", mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]); + if (unlikely(!IS_LEAF(mp))) + return MDBX_CORRUPTED; + if (IS_LEAF2(mp)) { if (likely(key)) { key->iov_len = mc->mc_db->md_xsize; @@ -11333,16 +14713,16 @@ skip: return MDBX_SUCCESS; } - mdbx_cassert(mc, IS_LEAF(mp)); node = page_node(mp, mc->mc_ki[mc->mc_top]); - if (F_ISSET(node_flags(node), F_DUPDATA)) { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mp); if (unlikely(rc != MDBX_SUCCESS)) return rc; } if (data) { - if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + if (unlikely((rc = mdbx_node_read(mc, node, data, + pp_txnid4chk(mp, mc->mc_txn))) != + MDBX_SUCCESS)) return rc; if (F_ISSET(node_flags(node), F_DUPDATA)) { @@ -11429,12 +14809,14 @@ static int mdbx_cursor_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, node = page_node(mp, mc->mc_ki[mc->mc_top]); if (F_ISSET(node_flags(node), F_DUPDATA)) { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mp); if (unlikely(rc != MDBX_SUCCESS)) return rc; } if (data) { - if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + if (unlikely((rc = mdbx_node_read(mc, node, data, + pp_txnid4chk(mp, mc->mc_txn))) != + MDBX_SUCCESS)) return rc; if (F_ISSET(node_flags(node), F_DUPDATA)) { @@ -11618,7 +15000,7 @@ set1: } if (F_ISSET(node_flags(node), F_DUPDATA)) { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mp); if (unlikely(rc != MDBX_SUCCESS)) return rc; } @@ -11661,7 +15043,10 @@ set1: } } MDBX_val olddata; - if (unlikely((rc = mdbx_node_read(mc, node, &olddata)) != MDBX_SUCCESS)) + if (unlikely((rc = mdbx_node_read( + mc, node, &olddata, + pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) != + MDBX_SUCCESS)) return rc; rc = mc->mc_dbx->md_dcmp(&aligned_data, &olddata); if (rc) { @@ -11673,7 +15058,10 @@ set1: } else { if (mc->mc_xcursor) mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); - if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + if (unlikely((rc = mdbx_node_read( + mc, node, data, + pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) != + MDBX_SUCCESS)) return rc; } } @@ -11698,7 +15086,9 @@ static int mdbx_cursor_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { if (unlikely(rc != MDBX_SUCCESS)) return rc; } - mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + if (unlikely(!IS_LEAF(mc->mc_pg[mc->mc_top]))) + return MDBX_CORRUPTED; mc->mc_flags |= C_INITIALIZED; mc->mc_flags &= ~C_EOF; @@ -11713,14 +15103,17 @@ static int mdbx_cursor_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], 0); if (likely(data)) { if (F_ISSET(node_flags(node), F_DUPDATA)) { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]); if (unlikely(rc != MDBX_SUCCESS)) return rc; rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); if (unlikely(rc)) return rc; } else { - if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + if (unlikely((rc = mdbx_node_read( + mc, node, data, + pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) != + MDBX_SUCCESS)) return rc; } } @@ -11741,9 +15134,11 @@ static int mdbx_cursor_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { if (unlikely(rc != MDBX_SUCCESS)) return rc; } - mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); } + if (unlikely(!IS_LEAF(mc->mc_pg[mc->mc_top]))) + return MDBX_CORRUPTED; + mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]) - 1; mc->mc_flags |= C_INITIALIZED | C_EOF; @@ -11757,14 +15152,17 @@ static int mdbx_cursor_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (likely(data)) { if (F_ISSET(node_flags(node), F_DUPDATA)) { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]); if (unlikely(rc != MDBX_SUCCESS)) return rc; rc = mdbx_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); if (unlikely(rc)) return rc; } else { - if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + if (unlikely((rc = mdbx_node_read( + mc, node, data, + pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) != + MDBX_SUCCESS)) return rc; } } @@ -11778,7 +15176,7 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, if (unlikely(mc == NULL)) return MDBX_EINVAL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); @@ -11810,7 +15208,7 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, if (data) { if (F_ISSET(node_flags(node), F_DUPDATA)) { if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))) { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mp); if (unlikely(rc != MDBX_SUCCESS)) return rc; rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); @@ -11820,7 +15218,7 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, rc = mdbx_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_GET_CURRENT); } else { - rc = mdbx_node_read(mc, node, data); + rc = mdbx_node_read(mc, node, data, pp_txnid4chk(mp, mc->mc_txn)); } if (unlikely(rc)) return rc; @@ -11921,7 +15319,8 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(node_flags(node), F_DUPDATA)) { get_key_optional(node, key); - rc = mdbx_node_read(mc, node, data); + rc = mdbx_node_read(mc, node, data, + pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn)); break; } } @@ -11981,16 +15380,13 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, MDBX_page *sub_root = NULL; MDBX_val xdata, *rdata, dkey, olddata; MDBX_db nested_dupdb; - unsigned mcount = 0, dcount = 0, nospill; - size_t nsize; int rc2; - unsigned nflags; DKBUF; if (unlikely(mc == NULL || key == NULL || data == NULL)) return MDBX_EINVAL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED); @@ -12000,13 +15396,24 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, env = mc->mc_txn->mt_env; /* Check this first so counter will always be zero on any early failures. */ - if (flags & MDBX_MULTIPLE) { + size_t mcount = 0, dcount = 0; + if (unlikely(flags & MDBX_MULTIPLE)) { + if (unlikely(flags & MDBX_RESERVE)) + return MDBX_EINVAL; if (unlikely(!F_ISSET(mc->mc_db->md_flags, MDBX_DUPFIXED))) return MDBX_INCOMPATIBLE; - if (unlikely(data[1].iov_len >= INT_MAX)) - return MDBX_EINVAL; - dcount = (unsigned)data[1].iov_len; - data[1].iov_len = 0; + dcount = data[1].iov_len; + if (unlikely(dcount < 2 || data->iov_len == 0)) + return MDBX_BAD_VALSIZE; + if (unlikely(mc->mc_db->md_xsize != data->iov_len) && mc->mc_db->md_xsize) + return MDBX_BAD_VALSIZE; + if (unlikely(dcount > + MAX_MAPSIZE / 2 / (BRANCH_NODEMAX(MAX_PAGESIZE) - NODESIZE))) { + /* checking for multiplication overflow */ + if (unlikely(dcount > MAX_MAPSIZE / 2 / data->iov_len)) + return MDBX_TOO_LARGE; + } + data[1].iov_len = 0 /* reset done item counter */; } if (flags & MDBX_RESERVE) { @@ -12016,11 +15423,12 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, data->iov_base = nullptr; } - nospill = flags & MDBX_NOSPILL; - flags &= ~MDBX_NOSPILL; + const unsigned nospill = flags & MDBX_NOSPILL; + flags -= nospill; - if (unlikely(mc->mc_txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) - return (mc->mc_txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + if (unlikely(mc->mc_txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) + return (mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS + : MDBX_BAD_TXN; uint64_t aligned_keybytes, aligned_databytes; MDBX_val aligned_key, aligned_data; @@ -12066,6 +15474,8 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, return MDBX_BAD_VALSIZE; case 4: if (unlikely(3 & (uintptr_t)data->iov_base)) { + if (unlikely(flags & MDBX_MULTIPLE)) + return MDBX_BAD_VALSIZE; /* copy instead of return error to avoid break compatibility */ aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base, aligned_data.iov_len = 4); @@ -12074,6 +15484,8 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, break; case 8: if (unlikely(7 & (uintptr_t)data->iov_base)) { + if (unlikely(flags & MDBX_MULTIPLE)) + return MDBX_BAD_VALSIZE; /* copy instead of return error to avoid break compatibility */ aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base, aligned_data.iov_len = 8); @@ -12091,6 +15503,8 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, int dupdata_flag = 0; if ((flags & MDBX_CURRENT) != 0 && (mc->mc_flags & C_SUB) == 0) { + if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE))) + return MDBX_EINVAL; /* Опция MDBX_CURRENT означает, что запрошено обновление текущей записи, * на которой сейчас стоит курсор. Проверяем что переданный ключ совпадает * со значением в текущей позиции курсора. @@ -12103,6 +15517,9 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, if (mc->mc_dbx->md_cmp(key, ¤t_key) != 0) return MDBX_EKEYMISMATCH; + if (unlikely((flags & MDBX_MULTIPLE))) + goto drop_current; + if (F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)) { MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (F_ISSET(node_flags(node), F_DUPDATA)) { @@ -12110,24 +15527,31 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, mc->mc_xcursor != NULL && (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)); /* Если за ключом более одного значения, либо если размер данных - * отличается, то вместо inplace обновления требуется удаление и + * отличается, то вместо обновления требуется удаление и * последующая вставка. */ if (mc->mc_xcursor->mx_db.md_entries > 1 || current_data.iov_len != data->iov_len) { - rc = mdbx_cursor_del(mc, 0); - if (rc != MDBX_SUCCESS) + drop_current: + rc = mdbx_cursor_del(mc, flags & MDBX_ALLDUPS); + if (unlikely(rc != MDBX_SUCCESS)) return rc; flags -= MDBX_CURRENT; + goto skip_check_samedata; } } else if (unlikely(node_size(key, data) > /* See note inside leaf_size() */ env->me_branch_nodemax)) { rc = mdbx_cursor_del(mc, 0); - if (rc != MDBX_SUCCESS) + if (unlikely(rc != MDBX_SUCCESS)) return rc; flags -= MDBX_CURRENT; + goto skip_check_samedata; } } + if (!(flags & MDBX_RESERVE) && + unlikely(cmp_lenfast(¤t_data, data) == 0)) + return MDBX_SUCCESS /* the same data, nothing to update */; + skip_check_samedata:; } if (mc->mc_db->md_root == P_INVALID) { @@ -12137,38 +15561,65 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, mc->mc_flags &= ~C_INITIALIZED; rc = MDBX_NO_ROOT; } else if ((flags & MDBX_CURRENT) == 0) { - int exact = 0; - if ((flags & MDBX_APPEND) != 0 && mc->mc_db->md_entries > 0) { + int exact = false; + if ((flags & MDBX_APPEND) && mc->mc_db->md_entries > 0) { rc = mdbx_cursor_last(mc, &dkey, &olddata); - if (rc == 0) { + if (likely(rc == MDBX_SUCCESS)) { rc = mc->mc_dbx->md_cmp(key, &dkey); - if (rc > 0) { + if (likely(rc > 0)) { + mc->mc_ki[mc->mc_top]++; /* step forward for appending */ rc = MDBX_NOTFOUND; - mc->mc_ki[mc->mc_top]++; - } else if (unlikely(rc < 0 || (flags & MDBX_APPENDDUP) == 0)) { - /* new key is <= last key */ - rc = MDBX_EKEYMISMATCH; + } else { + if (unlikely(rc != 0 || !(flags & MDBX_APPENDDUP))) + /* new-key < last-key + * or new-key == last-key without MDBX_APPENDDUP */ + return MDBX_EKEYMISMATCH; + exact = true; } } } else { rc = mdbx_cursor_set(mc, (MDBX_val *)key, &olddata, MDBX_SET, &exact); } - if ((flags & MDBX_NOOVERWRITE) && - (rc == MDBX_SUCCESS || rc == MDBX_EKEYMISMATCH)) { - mdbx_debug("duplicate key [%s]", DKEY(key)); - *data = olddata; - return MDBX_KEYEXIST; - } if (likely(rc == MDBX_SUCCESS)) { if (exact) { - if (mc->mc_flags & C_SUB) { - mdbx_assert(env, data->iov_len == 0); - return (flags & MDBX_NODUPDATA) ? MDBX_KEYEXIST : MDBX_SUCCESS; + if (unlikely(flags & MDBX_NOOVERWRITE)) { + mdbx_debug("duplicate key [%s]", DKEY(key)); + *data = olddata; + return MDBX_KEYEXIST; + } + if (unlikely(mc->mc_flags & C_SUB)) { + /* nested subtree of DUPSORT-database with the same key, + * nothing to update */ + mdbx_assert(env, data->iov_len == 0 && olddata.iov_len == 0); + return MDBX_SUCCESS; + } + if (unlikely(flags & MDBX_ALLDUPS) && mc->mc_xcursor && + (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = mdbx_cursor_del(mc, MDBX_ALLDUPS); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + flags -= MDBX_ALLDUPS; + rc = MDBX_NOTFOUND; + exact = false; + } else /* checking for early exit without dirtying pages */ + if (!(flags & (MDBX_RESERVE | MDBX_MULTIPLE)) && + unlikely(mc->mc_dbx->md_dcmp(data, &olddata) == 0)) { + if (!mc->mc_xcursor) + /* the same data, nothing to update */ + return MDBX_SUCCESS; + if (flags & MDBX_NODUPDATA) + return MDBX_KEYEXIST; + if (flags & MDBX_APPENDDUP) + return MDBX_EKEYMISMATCH; + if (likely(unsure_equal(mc->mc_dbx->md_dcmp, data, &olddata))) + /* data is match exactly byte-to-byte, nothing to update */ + return MDBX_SUCCESS; + else { + /* The data has differences, but the user-provided comparator + * considers them equal. So continue update since called without. + * Continue to update since was called without MDBX_NODUPDATA. */ + } } - if (!(flags & MDBX_RESERVE) && - unlikely(mc->mc_dbx->md_dcmp(data, &olddata) == 0)) - return ((flags & MDBX_NODUPDATA) && mc->mc_xcursor) ? MDBX_KEYEXIST - : MDBX_SUCCESS; } } else if (unlikely(rc != MDBX_NOTFOUND)) return rc; @@ -12178,7 +15629,7 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, /* Cursor is positioned, check for room in the dirty list */ if (!nospill) { - if (flags & MDBX_MULTIPLE) { + if (unlikely(flags & MDBX_MULTIPLE)) { rdata = &xdata; xdata.iov_len = data->iov_len * dcount; } else { @@ -12208,12 +15659,12 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, if (mc->mc_db->md_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) { assert(data->iov_len >= mc->mc_dbx->md_vlen_min && data->iov_len <= mc->mc_dbx->md_vlen_max); - mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = data->iov_len; assert(mc->mc_xcursor != NULL); mc->mc_db->md_xsize = mc->mc_xcursor->mx_db.md_xsize = - (unsigned)data->iov_len; - mc->mc_xcursor->mx_dbx.md_klen_min = mc->mc_xcursor->mx_dbx.md_klen_max = - data->iov_len; + (unsigned)(mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = + mc->mc_xcursor->mx_dbx.md_klen_min = + mc->mc_xcursor->mx_dbx.md_klen_max = + data->iov_len); } *mc->mc_dbistate |= DBI_DIRTY; if ((mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_DUPFIXED)) == MDBX_DUPFIXED) @@ -12230,7 +15681,7 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, insert_key = insert_data = (rc != MDBX_SUCCESS); uint16_t fp_flags = P_LEAF | P_DIRTY; MDBX_page *fp = env->me_pbuf; - fp->mp_txnid = INVALID_TXNID; + fp->mp_txnid = pp_txnid2chk(mc->mc_txn); if (insert_key) { /* The key does not exist */ mdbx_debug("inserting key at index %i", mc->mc_ki[mc->mc_top]); @@ -12301,7 +15752,9 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, const pgno_t pg = node_largedata_pgno(node); MDBX_page *omp; - if (unlikely((rc2 = mdbx_page_get(mc, pg, &omp, &level)) != 0)) + if (unlikely((rc2 = mdbx_page_get( + mc, pg, &omp, &level, + pp_txnid4chk(mc->mc_pg[mc->mc_top], mc->mc_txn))) != 0)) return rc2; ovpages = omp->mp_pages; @@ -12386,8 +15839,25 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, /* Was a single item before, must convert now */ if (!F_ISSET(node_flags(node), F_DUPDATA)) { - /* Just overwrite the current item */ - if (flags & MDBX_CURRENT) { + /* does data match? */ + const int cmp = mc->mc_dbx->md_dcmp(data, &olddata); + if ((flags & MDBX_APPENDDUP) && unlikely(cmp <= 0)) + return MDBX_EKEYMISMATCH; + if (cmp == 0) { + if (flags & MDBX_NODUPDATA) + return MDBX_KEYEXIST; + if (likely(unsure_equal(mc->mc_dbx->md_dcmp, data, &olddata))) { + /* data is match exactly byte-to-byte, nothing to update */ + if (unlikely(flags & MDBX_MULTIPLE)) { + rc = MDBX_SUCCESS; + goto continue_multiple; + } + return MDBX_SUCCESS; + } else { + /* The data has differences, but the user-provided comparator + * considers them equal. So continue update since called without. + * Continue to update since was called without MDBX_NODUPDATA. */ + } mdbx_cassert( mc, node_size(key, data) <= @@ -12395,11 +15865,8 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, goto current; } - /* does data match? */ - if (!mc->mc_dbx->md_dcmp(data, &olddata)) { - if (unlikely(flags & (MDBX_NODUPDATA | MDBX_APPENDDUP))) - return MDBX_KEYEXIST; - /* overwrite it */ + /* Just overwrite the current item */ + if (flags & MDBX_CURRENT) { mdbx_cassert( mc, node_size(key, data) <= @@ -12486,7 +15953,8 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, offset = env->me_psize - (unsigned)olddata.iov_len; flags |= F_DUPDATA | F_SUBDATA; nested_dupdb.md_root = mp->mp_pgno; - nested_dupdb.md_seq = nested_dupdb.md_mod_txnid = 0; + nested_dupdb.md_seq = 0; + nested_dupdb.md_mod_txnid = pp_txnid2chk(mc->mc_txn); sub_root = mp; } if (mp != fp) { @@ -12560,10 +16028,10 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, rdata = data; -new_sub: - nflags = flags & NODE_ADD_FLAGS; - nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->iov_len - : leaf_size(env, key, rdata); +new_sub:; + unsigned nflags = flags & NODE_ADD_FLAGS; + size_t nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->iov_len + : leaf_size(env, key, rdata); if (page_room(mc->mc_pg[mc->mc_top]) < nsize) { if ((flags & (F_DUPDATA | F_SUBDATA)) == F_DUPDATA) nflags &= ~MDBX_APPEND; /* sub-page may need room to grow */ @@ -12612,16 +16080,18 @@ new_sub: xdata.iov_len = 0; xdata.iov_base = nullptr; MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (flags & MDBX_CURRENT) { - xflags = (flags & MDBX_NODUPDATA) - ? MDBX_CURRENT | MDBX_NOOVERWRITE | MDBX_NOSPILL - : MDBX_CURRENT | MDBX_NOSPILL; - } else { - rc2 = mdbx_xcursor_init1(mc, node); +#define SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE 1 + STATIC_ASSERT( + (MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) == + MDBX_NOOVERWRITE); + xflags = MDBX_CURRENT | MDBX_NOSPILL | + ((flags & MDBX_NODUPDATA) >> + SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE); + if ((flags & MDBX_CURRENT) == 0) { + xflags -= MDBX_CURRENT; + rc2 = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]); if (unlikely(rc2 != MDBX_SUCCESS)) return rc2; - xflags = (flags & MDBX_NODUPDATA) ? MDBX_NOOVERWRITE | MDBX_NOSPILL - : MDBX_NOSPILL; } if (sub_root) mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; @@ -12666,6 +16136,7 @@ new_sub: rc = mdbx_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); if (flags & F_SUBDATA) { void *db = node_data(node); + mc->mc_xcursor->mx_db.md_mod_txnid = pp_txnid2chk(mc->mc_txn); memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db)); } insert_data = (ecount != (size_t)mc->mc_xcursor->mx_db.md_entries); @@ -12683,6 +16154,7 @@ new_sub: } if (flags & MDBX_MULTIPLE) { if (!rc) { + continue_multiple: mcount++; /* let caller know how many succeeded, if any */ data[1].iov_len = mcount; @@ -12707,11 +16179,11 @@ fail: return rc; } -int mdbx_cursor_del(MDBX_cursor *mc, unsigned flags) { +int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) { if (unlikely(!mc)) return MDBX_EINVAL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED); @@ -12719,7 +16191,7 @@ int mdbx_cursor_del(MDBX_cursor *mc, unsigned flags) { return rc; if (unlikely(!(mc->mc_flags & C_INITIALIZED))) - return MDBX_EINVAL; + return MDBX_ENODATA; if (unlikely(mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top]))) return MDBX_NOTFOUND; @@ -12738,7 +16210,7 @@ int mdbx_cursor_del(MDBX_cursor *mc, unsigned flags) { MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); if (F_ISSET(node_flags(node), F_DUPDATA)) { - if (flags & MDBX_NODUPDATA) { + if (flags & (MDBX_ALLDUPS | /* for compatibility */ MDBX_NODUPDATA)) { /* mdbx_cursor_del0() will subtract the final entry */ mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; @@ -12754,6 +16226,7 @@ int mdbx_cursor_del(MDBX_cursor *mc, unsigned flags) { if (node_flags(node) & F_SUBDATA) { /* update subDB info */ void *db = node_data(node); + mc->mc_xcursor->mx_db.md_mod_txnid = pp_txnid2chk(mc->mc_txn); memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db)); } else { MDBX_cursor *m2; @@ -12798,17 +16271,15 @@ int mdbx_cursor_del(MDBX_cursor *mc, unsigned flags) { } } /* MDBX passes F_SUBDATA in 'flags' to delete a DB record */ - else if (unlikely((node_flags(node) ^ flags) & F_SUBDATA)) { - rc = MDBX_INCOMPATIBLE; - goto fail; - } + else if (unlikely((node_flags(node) ^ flags) & F_SUBDATA)) + return MDBX_INCOMPATIBLE; /* add overflow pages to free list */ if (F_ISSET(node_flags(node), F_BIGDATA)) { MDBX_page *omp; - if (unlikely( - (rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL)) || - (rc = mdbx_page_retire(mc, omp)))) + if (unlikely((rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL, + pp_txnid4chk(mp, mc->mc_txn))) || + (rc = mdbx_page_retire(mc, omp)))) goto fail; } @@ -12956,7 +16427,7 @@ static int __must_check_result mdbx_node_add_leaf(MDBX_cursor *mc, mdbx_cassert(mc, PAGETYPE(mp) == P_LEAF); MDBX_page *largepage = NULL; - size_t leaf_bytes = 0; + size_t leaf_bytes; if (unlikely(flags & F_BIGDATA)) { /* Data already on overflow page. */ STATIC_ASSERT(sizeof(pgno_t) % 2 == 0); @@ -12965,7 +16436,10 @@ static int __must_check_result mdbx_node_add_leaf(MDBX_cursor *mc, /* See note inside leaf_size() */ mc->mc_txn->mt_env->me_branch_nodemax)) { /* Put data on overflow page. */ - mdbx_cassert(mc, !F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)); + mdbx_ensure(mc->mc_txn->mt_env, + !F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)); + if (unlikely(flags & (F_DUPDATA | F_SUBDATA))) + return MDBX_PROBLEM; const pgno_t ovpages = number_of_ovpages(mc->mc_txn->mt_env, data->iov_len); int rc = mdbx_page_new(mc, P_OVERFLOW, ovpages, &largepage); if (unlikely(rc != MDBX_SUCCESS)) @@ -13156,8 +16630,11 @@ static void mdbx_node_shrink(MDBX_page *mp, unsigned indx) { * [in] mc The main cursor whose sorted-dups cursor is to be initialized. */ static int mdbx_xcursor_init0(MDBX_cursor *mc) { MDBX_xcursor *mx = mc->mc_xcursor; - if (unlikely(mx == nullptr)) + if (unlikely(mx == nullptr)) { + mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)", + mc->mc_dbi); return MDBX_CORRUPTED; + } mx->mx_cursor.mc_xcursor = NULL; mx->mx_cursor.mc_txn = mc->mc_txn; @@ -13182,22 +16659,43 @@ static int mdbx_xcursor_init0(MDBX_cursor *mc) { * [in] mc The main cursor whose sorted-dups cursor is to be initialized. * [in] node The data containing the MDBX_db record for the sorted-dup database. */ -static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node) { +static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node, + const MDBX_page *mp) { MDBX_xcursor *mx = mc->mc_xcursor; - if (unlikely(mx == nullptr)) + if (unlikely(mx == nullptr)) { + mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)", + mc->mc_dbi); return MDBX_CORRUPTED; + } - if (node_flags(node) & F_SUBDATA) { - if (unlikely(node_ds(node) != sizeof(MDBX_db))) + const uint8_t flags = node_flags(node); + switch (flags) { + default: + mdbx_error("invalid node flags %u", flags); + return MDBX_CORRUPTED; + case F_DUPDATA | F_SUBDATA: + if (unlikely(node_ds(node) != sizeof(MDBX_db))) { + mdbx_error("invalid nested-db record size %zu", node_ds(node)); return MDBX_CORRUPTED; + } memcpy(&mx->mx_db, node_data(node), sizeof(MDBX_db)); + const txnid_t pp_txnid = IS_DIRTY(mp) ? mc->mc_txn->mt_txnid : mp->mp_txnid; + if (unlikely(mx->mx_db.md_mod_txnid > pp_txnid)) { + mdbx_error("nested-db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN + ")", + mx->mx_db.md_mod_txnid, pp_txnid); + return MDBX_CORRUPTED; + } mx->mx_cursor.mc_pg[0] = 0; mx->mx_cursor.mc_snum = 0; mx->mx_cursor.mc_top = 0; mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_COPYING | C_SKIPORD)); - } else { - if (unlikely(node_ds(node) <= PAGEHDRSZ)) + break; + case F_DUPDATA: + if (unlikely(node_ds(node) <= PAGEHDRSZ)) { + mdbx_error("invalid nested-page size %zu", node_ds(node)); return MDBX_CORRUPTED; + } MDBX_page *fp = node_data(node); mx->mx_db.md_depth = 1; mx->mx_db.md_branch_pages = 0; @@ -13205,6 +16703,7 @@ static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node) { mx->mx_db.md_overflow_pages = 0; mx->mx_db.md_entries = page_numkeys(fp); mx->mx_db.md_root = fp->mp_pgno; + mx->mx_db.md_mod_txnid = mp->mp_txnid; mx->mx_cursor.mc_snum = 1; mx->mx_cursor.mc_top = 0; mx->mx_cursor.mc_flags = @@ -13214,16 +16713,27 @@ static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node) { mx->mx_db.md_flags = flags_db2sub(mc->mc_db->md_flags); mx->mx_db.md_xsize = (mc->mc_db->md_flags & MDBX_DUPFIXED) ? fp->mp_leaf2_ksize : 0; + break; } if (unlikely(mx->mx_db.md_xsize != mc->mc_db->md_xsize)) { - if (unlikely(mc->mc_db->md_xsize != 0)) + if (unlikely(mc->mc_db->md_xsize != 0)) { + mdbx_error("cursor mismatched nested-db md_xsize %u", + mc->mc_db->md_xsize); return MDBX_CORRUPTED; - if (unlikely((mc->mc_db->md_flags & MDBX_DUPFIXED) == 0)) + } + if (unlikely((mc->mc_db->md_flags & MDBX_DUPFIXED) == 0)) { + mdbx_error("mismatched nested-db md_flags %u", mc->mc_db->md_flags); return MDBX_CORRUPTED; + } if (unlikely(mx->mx_db.md_xsize < mc->mc_dbx->md_vlen_min || - mx->mx_db.md_xsize > mc->mc_dbx->md_vlen_max)) + mx->mx_db.md_xsize > mc->mc_dbx->md_vlen_max)) { + mdbx_error("mismatched nested-db.md_xsize (%u) <> min/max value-length " + "(%zu/%zu)", + mx->mx_db.md_xsize, mc->mc_dbx->md_vlen_min, + mc->mc_dbx->md_vlen_max); return MDBX_CORRUPTED; + } mc->mc_db->md_xsize = mx->mx_db.md_xsize; mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = mx->mx_db.md_xsize; } @@ -13244,10 +16754,13 @@ static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node) { * [in] src_mx The xcursor of an up-to-date cursor. * [in] new_dupdata True if converting from a non-F_DUPDATA item. */ static int mdbx_xcursor_init2(MDBX_cursor *mc, MDBX_xcursor *src_mx, - int new_dupdata) { + bool new_dupdata) { MDBX_xcursor *mx = mc->mc_xcursor; - if (unlikely(mx == nullptr)) + if (unlikely(mx == nullptr)) { + mdbx_error("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)", + mc->mc_dbi); return MDBX_CORRUPTED; + } if (new_dupdata) { mx->mx_cursor.mc_snum = 1; @@ -13273,7 +16786,7 @@ static __inline int mdbx_couple_init(MDBX_cursor_couple *couple, const MDBX_dbi dbi, MDBX_txn *const txn, MDBX_db *const db, MDBX_dbx *const dbx, uint8_t *const dbstate) { - couple->outer.mc_signature = MDBX_MC_SIGNATURE; + couple->outer.mc_signature = MDBX_MC_LIVE; couple->outer.mc_next = NULL; couple->outer.mc_backup = NULL; couple->outer.mc_dbi = dbi; @@ -13298,7 +16811,7 @@ static __inline int mdbx_couple_init(MDBX_cursor_couple *couple, } if (couple->outer.mc_db->md_flags & MDBX_DUPSORT) { - couple->inner.mx_cursor.mc_signature = MDBX_MC_SIGNATURE; + couple->inner.mx_cursor.mc_signature = MDBX_MC_LIVE; couple->outer.mc_xcursor = &couple->inner; rc = mdbx_xcursor_init0(&couple->outer); if (unlikely(rc != MDBX_SUCCESS)) @@ -13317,76 +16830,111 @@ static int mdbx_cursor_init(MDBX_cursor *mc, MDBX_txn *txn, MDBX_dbi dbi) { &txn->mt_dbistate[dbi]); } -int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) { - if (unlikely(!ret)) +MDBX_cursor *mdbx_cursor_create(void *context) { + MDBX_cursor_couple *couple = mdbx_calloc(1, sizeof(MDBX_cursor_couple)); + if (unlikely(!couple)) + return nullptr; + + couple->outer.mc_signature = MDBX_MC_READY4CLOSE; + couple->outer.mc_dbi = UINT_MAX; + couple->mc_userctx = context; + return &couple->outer; +} + +int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) { + if (unlikely(!mc)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE && + mc->mc_signature != MDBX_MC_LIVE)) + return MDBX_EINVAL; + + MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer); + couple->mc_userctx = ctx; + return MDBX_SUCCESS; +} + +void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) { + if (unlikely(!mc)) + return nullptr; + + if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE && + mc->mc_signature != MDBX_MC_LIVE)) + return nullptr; + + MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer); + return couple->mc_userctx; +} + +int mdbx_cursor_bind(MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) { + if (unlikely(!mc)) return MDBX_EINVAL; - *ret = NULL; + + if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE)) { + if (unlikely(mc->mc_signature != MDBX_MC_LIVE || mc->mc_backup)) + return MDBX_EINVAL; + if (unlikely(!mc->mc_txn || mc->mc_txn->mt_signature != MDBX_MT_SIGNATURE)) + return MDBX_PROBLEM; + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDBX_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) + prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + mc->mc_signature = MDBX_MC_READY4CLOSE; + mc->mc_flags = 0; + mc->mc_dbi = UINT_MAX; + } + + assert(!mc->mc_backup && !mc->mc_flags); + if (unlikely(mc->mc_backup || mc->mc_flags)) + return MDBX_PROBLEM; int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_VALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; - if (unlikely(dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDBX_RDONLY))) + if (unlikely(dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDBX_TXN_RDONLY))) return MDBX_EACCESS; - const size_t size = (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) - ? sizeof(MDBX_cursor_couple) - : sizeof(MDBX_cursor); + rc = mdbx_cursor_init(mc, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - MDBX_cursor *mc; - if (likely((mc = mdbx_malloc(size)) != NULL)) { - rc = mdbx_cursor_init(mc, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - mdbx_free(mc); - return rc; - } - if (txn->mt_cursors) { - mc->mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = mc; - mc->mc_flags |= C_UNTRACK; - } - } else { - return MDBX_ENOMEM; + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; } - *ret = mc; return MDBX_SUCCESS; } -int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *mc) { - if (unlikely(!mc)) +int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) { + if (unlikely(!ret)) return MDBX_EINVAL; + *ret = NULL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE && - mc->mc_signature != MDBX_MC_READY4CLOSE)) - return MDBX_EINVAL; + MDBX_cursor *const mc = mdbx_cursor_create(nullptr); + if (unlikely(!mc)) + return MDBX_ENOMEM; - int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) + int rc = mdbx_cursor_bind(txn, mc, dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_cursor_close(mc); return rc; - - if (unlikely(!mdbx_txn_dbi_exists(txn, mc->mc_dbi, DBI_VALID))) - return MDBX_EINVAL; - - if (unlikely(mc->mc_backup)) - return MDBX_EINVAL; - - if (unlikely((mc->mc_flags & C_UNTRACK) || txn->mt_cursors)) { - MDBX_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; - while (*prev && *prev != mc) - prev = &(*prev)->mc_next; - if (*prev == mc) - *prev = mc->mc_next; - mc->mc_signature = MDBX_MC_READY4CLOSE; } - if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED)) - return MDBX_BAD_TXN; + *ret = mc; + return MDBX_SUCCESS; +} - return mdbx_cursor_init(mc, txn, mc->mc_dbi); +int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *mc) { + return likely(mc) ? mdbx_cursor_bind(txn, mc, mc->mc_dbi) : MDBX_EINVAL; } /* Return the count of duplicate data items for the current key */ @@ -13394,7 +16942,7 @@ int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) { if (unlikely(mc == NULL)) return MDBX_EINVAL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); @@ -13431,7 +16979,7 @@ int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) { void mdbx_cursor_close(MDBX_cursor *mc) { if (mc) { - mdbx_ensure(NULL, mc->mc_signature == MDBX_MC_SIGNATURE || + mdbx_ensure(NULL, mc->mc_signature == MDBX_MC_LIVE || mc->mc_signature == MDBX_MC_READY4CLOSE); if (!mc->mc_backup) { /* Remove from txn, if tracked. @@ -13448,14 +16996,14 @@ void mdbx_cursor_close(MDBX_cursor *mc) { mdbx_free(mc); } else { /* cursor closed before nested txn ends */ - mdbx_cassert(mc, mc->mc_signature == MDBX_MC_SIGNATURE); + mdbx_cassert(mc, mc->mc_signature == MDBX_MC_LIVE); mc->mc_signature = MDBX_MC_WAIT4EOT; } } } MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *mc) { - if (unlikely(!mc || mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE)) return NULL; MDBX_txn *txn = mc->mc_txn; if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) @@ -13466,7 +17014,7 @@ MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *mc) { } MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) { - if (unlikely(!mc || mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE)) return UINT_MAX; return mc->mc_dbi; } @@ -14118,40 +17666,45 @@ static void mdbx_cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst) { * [in] mc Cursor pointing to the page where rebalancing should begin. * Returns 0 on success, non-zero on failure. */ static int mdbx_rebalance(MDBX_cursor *mc) { - int rc; - mdbx_cassert(mc, mc->mc_snum > 0); mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth || IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); const int pagetype = PAGETYPE(mc->mc_pg[mc->mc_top]); - const unsigned minkeys = (P_BRANCH == 1) ? (pagetype & P_BRANCH) + 1 - : (pagetype & P_BRANCH) ? 2 : 1; + STATIC_ASSERT(P_BRANCH == 1); + const unsigned minkeys = (pagetype & P_BRANCH) + 1; /* The threshold of minimum page fill factor, in form of a negative binary - * exponent, i.e. 2 means 1/(2**3) == 1/4 == 25%. Pages emptier than this - * are candidates for merging. */ + * exponent, i.e. X = 2 means 1/(2**X) == 1/(2**2) == 1/4 == 25%. + * Pages emptier than this are candidates for merging. */ const unsigned threshold_fill_exp2 = 2; /* The threshold of minimum page fill factor, as a number of free bytes on a * page. Pages emptier than this are candidates for merging. */ - const unsigned spaceleft_threshold = + const unsigned room_threshold = page_space(mc->mc_txn->mt_env) - (page_space(mc->mc_txn->mt_env) >> threshold_fill_exp2); + const MDBX_page *const tp = mc->mc_pg[mc->mc_top]; mdbx_debug("rebalancing %s page %" PRIaPGNO " (has %u keys, %.1f%% full)", - (pagetype & P_LEAF) ? "leaf" : "branch", - mc->mc_pg[mc->mc_top]->mp_pgno, - page_numkeys(mc->mc_pg[mc->mc_top]), - page_fill(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top])); - - if (page_fill_enough(mc->mc_pg[mc->mc_top], spaceleft_threshold, minkeys)) { - mdbx_debug("no need to rebalance page %" PRIaPGNO ", above fill threshold", - mc->mc_pg[mc->mc_top]->mp_pgno); + (pagetype & P_LEAF) ? "leaf" : "branch", tp->mp_pgno, + page_numkeys(tp), page_fill(mc->mc_txn->mt_env, tp)); + + if (unlikely(page_numkeys(tp) < minkeys)) { + mdbx_debug("page %" PRIaPGNO " must be merged due keys < %u threshold", + tp->mp_pgno, minkeys); + } else if (unlikely(page_room(tp) > room_threshold)) { + mdbx_debug("page %" PRIaPGNO " should be merged due room %u > %u threshold", + tp->mp_pgno, page_room(tp), room_threshold); + } else { + mdbx_debug("no need to rebalance page %" PRIaPGNO + ", room %u < %u threshold", + tp->mp_pgno, page_room(tp), room_threshold); mdbx_cassert(mc, mc->mc_db->md_entries > 0); return MDBX_SUCCESS; } + int rc; if (mc->mc_snum < 2) { MDBX_page *const mp = mc->mc_pg[0]; const unsigned nkeys = page_numkeys(mp); @@ -14193,7 +17746,8 @@ static int mdbx_rebalance(MDBX_cursor *mc) { } else if (IS_BRANCH(mp) && nkeys == 1) { mdbx_debug("%s", "collapsing root page!"); mc->mc_db->md_root = node_pgno(page_node(mp, 0)); - rc = mdbx_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL); + rc = mdbx_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL, + pp_txnid4chk(mp, mc->mc_txn)); if (unlikely(rc != MDBX_SUCCESS)) return rc; mc->mc_db->md_depth--; @@ -14255,7 +17809,7 @@ static int mdbx_rebalance(MDBX_cursor *mc) { if (mn.mc_ki[pre_top] > 0) { rc = mdbx_page_get( &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] - 1)), - &left, NULL); + &left, NULL, pp_txnid4chk(mn.mc_pg[pre_top], mc->mc_txn)); if (unlikely(rc != MDBX_SUCCESS)) return rc; mdbx_cassert(mc, PAGETYPE(left) == PAGETYPE(mc->mc_pg[mc->mc_top])); @@ -14263,7 +17817,7 @@ static int mdbx_rebalance(MDBX_cursor *mc) { if (mn.mc_ki[pre_top] + 1u < page_numkeys(mn.mc_pg[pre_top])) { rc = mdbx_page_get( &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] + 1)), - &right, NULL); + &right, NULL, pp_txnid4chk(mn.mc_pg[pre_top], mc->mc_txn)); if (unlikely(rc != MDBX_SUCCESS)) return rc; mdbx_cassert(mc, PAGETYPE(right) == PAGETYPE(mc->mc_pg[mc->mc_top])); @@ -14272,7 +17826,7 @@ static int mdbx_rebalance(MDBX_cursor *mc) { const indx_t ki_top = mc->mc_ki[mc->mc_top]; const indx_t ki_pre_top = mn.mc_ki[pre_top]; const indx_t nkeys = (indx_t)page_numkeys(mn.mc_pg[mn.mc_top]); - if (left && page_room(left) > spaceleft_threshold && + if (left && page_room(left) > room_threshold && (!right || page_room(right) < page_room(left))) { /* try merge with left */ mdbx_cassert(mc, page_numkeys(left) >= minkeys); @@ -14291,7 +17845,7 @@ static int mdbx_rebalance(MDBX_cursor *mc) { return rc; } } - if (right && page_room(right) > spaceleft_threshold) { + if (right && page_room(right) > room_threshold) { /* try merge with right */ mdbx_cassert(mc, page_numkeys(right) >= minkeys); mn.mc_pg[mn.mc_top] = right; @@ -14382,22 +17936,21 @@ static __cold int mdbx_page_check(MDBX_cursor *const mc, MDBX_env *const env = mc->mc_txn->mt_env; const unsigned nkeys = page_numkeys(mp); char *const end_of_page = (char *)mp + env->me_psize; - mdbx_assert(env, mp->mp_pgno >= MIN_PAGENO && mp->mp_pgno <= MAX_PAGENO); if (unlikely(mp->mp_pgno < MIN_PAGENO || mp->mp_pgno > MAX_PAGENO)) - return MDBX_CORRUPTED; + return bad_page(mp, "invalid pgno (%u)\n", mp->mp_pgno); if (IS_OVERFLOW(mp)) { - mdbx_assert(env, mp->mp_pages >= 1 && mp->mp_pages < MAX_PAGENO / 2); if (unlikely(mp->mp_pages < 1 && mp->mp_pages >= MAX_PAGENO / 2)) - return MDBX_CORRUPTED; - mdbx_assert(env, mp->mp_pgno <= MAX_PAGENO - mp->mp_pages); - if (unlikely(mp->mp_pgno > MAX_PAGENO - mp->mp_pages)) - return MDBX_CORRUPTED; + return bad_page(mp, "invalid overflow n-pages (%u)\n", mp->mp_pages); + if (unlikely(mp->mp_pgno > mc->mc_txn->mt_next_pgno - mp->mp_pages)) + return bad_page(mp, "overflow page beyond (%u) next-pgno\n", + mp->mp_pgno + mp->mp_pages); return MDBX_SUCCESS; } + + int rc = MDBX_SUCCESS; if ((options & C_UPDATING) == 0 || !IS_DIRTY(mp)) { - mdbx_assert(env, nkeys >= 2 || !IS_BRANCH(mp)); if (unlikely(nkeys < 2 && IS_BRANCH(mp))) - return MDBX_CORRUPTED; + rc = bad_page(mp, "branch-page nkey (%u) < 2\n", nkeys); } MDBX_val here, prev = {0, 0}; @@ -14405,58 +17958,60 @@ static __cold int mdbx_page_check(MDBX_cursor *const mc, if (IS_LEAF2(mp)) { const size_t ksize = mp->mp_leaf2_ksize; char *const key = page_leaf2key(mp, i, ksize); - mdbx_assert(env, key + ksize <= end_of_page); - if (unlikely(end_of_page < key + ksize)) - return MDBX_CORRUPTED; + if (unlikely(end_of_page < key + ksize)) { + rc = bad_page(mp, "leaf2-key beyond (%zu) page-end\n", + key + ksize - end_of_page); + continue; + } if ((options & C_COPYING) == 0) { if (unlikely(ksize != mc->mc_dbx->md_klen_min)) { - mdbx_assert(env, ksize >= mc->mc_dbx->md_klen_min); - mdbx_assert(env, ksize <= mc->mc_dbx->md_klen_max); if (unlikely(ksize < mc->mc_dbx->md_klen_min || ksize > mc->mc_dbx->md_klen_max)) - return MDBX_CORRUPTED; - mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = ksize; + rc = bad_page( + mp, "leaf2-key size (%zu) <> min/max key-length (%zu/%zu)\n", + ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max); + else + mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = ksize; } if ((options & C_SKIPORD) == 0) { here.iov_len = ksize; here.iov_base = key; - if (prev.iov_base) { - mdbx_assert(env, mc->mc_dbx->md_cmp(&here, &prev) > 0); - if (unlikely(mc->mc_dbx->md_cmp(&here, &prev) <= 0)) - return MDBX_CORRUPTED; - } + if (prev.iov_base && unlikely(mc->mc_dbx->md_cmp(&here, &prev) <= 0)) + rc = bad_page(mp, "leaf2-key #%u wrong order\n", i); prev = here; } } } else { const MDBX_node *const node = page_node(mp, i); const char *node_end = (char *)node + NODESIZE; - mdbx_assert(env, node_end <= end_of_page); - if (unlikely(node_end > end_of_page)) - return MDBX_CORRUPTED; + if (unlikely(node_end > end_of_page)) { + rc = bad_page(mp, "node (%zu) beyond page-end\n", + node_end - end_of_page); + continue; + } if (IS_LEAF(mp) || i > 0) { size_t ksize = node_ks(node); char *key = node_key(node); - mdbx_assert(env, key + ksize <= end_of_page); - if (unlikely(end_of_page < key + ksize)) - return MDBX_CORRUPTED; + if (unlikely(end_of_page < key + ksize)) { + rc = bad_page(mp, "node-key (%zu) beyond page-end\n", + key + ksize - end_of_page); + continue; + } if ((options & C_COPYING) == 0) { - mdbx_assert(env, ksize >= mc->mc_dbx->md_klen_min); - mdbx_assert(env, ksize <= mc->mc_dbx->md_klen_max); if (unlikely(ksize < mc->mc_dbx->md_klen_min || ksize > mc->mc_dbx->md_klen_max)) - return MDBX_CORRUPTED; + rc = bad_page( + mp, "node-key size (%zu) <> min/max key-length (%zu/%zu)\n", + ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max); if ((options & C_SKIPORD) == 0) { here.iov_base = key; here.iov_len = ksize; - if (prev.iov_base) { - mdbx_assert(env, mc->mc_dbx->md_cmp(&here, &prev) > 0); - if (unlikely(mc->mc_dbx->md_cmp(&here, &prev) <= 0)) - return MDBX_CORRUPTED; - } + if (prev.iov_base && + unlikely(mc->mc_dbx->md_cmp(&here, &prev) <= 0)) + rc = bad_page(mp, "node-key #%u wrong order\n", i); prev = here; } } @@ -14464,70 +18019,94 @@ static __cold int mdbx_page_check(MDBX_cursor *const mc, if (IS_BRANCH(mp)) { if ((options & C_RETIRING) == 0) { const pgno_t ref = node_pgno(node); - mdbx_assert(env, ref >= MIN_PAGENO); - mdbx_assert(env, ref < mc->mc_txn->mt_next_pgno); if (unlikely(ref < MIN_PAGENO || ref >= mc->mc_txn->mt_next_pgno)) - return MDBX_CORRUPTED; + rc = bad_page(mp, "branch-node wrong pgno (%u)\n", ref); } continue; } - if (node_flags(node) == F_BIGDATA /* data on large-page */) { + + switch (node_flags(node)) { + default: + rc = bad_page(mp, "invalid node flags (%u)\n", node_flags(node)); + break; + case F_BIGDATA /* data on large-page */: + case 0 /* usual */: + case F_SUBDATA /* sub-db */: + case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: + case F_DUPDATA /* short sub-page */: + break; + } + + if (node_flags(node) & F_BIGDATA) { const size_t dsize = node_ds(node); if ((options & C_COPYING) == 0) { - mdbx_assert(env, dsize > mc->mc_dbx->md_vlen_min); - mdbx_assert(env, dsize <= mc->mc_dbx->md_vlen_max); if (unlikely(dsize <= mc->mc_dbx->md_vlen_min || dsize > mc->mc_dbx->md_vlen_max)) - return MDBX_CORRUPTED; + rc = bad_page( + mp, + "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n", + dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max); } if ((options & C_RETIRING) == 0) { MDBX_page *lp; - int err = mdbx_page_get(mc, node_largedata_pgno(node), &lp, NULL); + int err = mdbx_page_get(mc, node_largedata_pgno(node), &lp, NULL, + pp_txnid4chk(mp, mc->mc_txn)); if (unlikely(err != MDBX_SUCCESS)) return err; - mdbx_assert(env, IS_OVERFLOW(lp)); - mdbx_assert(env, number_of_ovpages(env, dsize) == lp->mp_pages); - if (unlikely(!IS_OVERFLOW(lp) || - number_of_ovpages(env, dsize) != lp->mp_pages)) - return MDBX_CORRUPTED; + if (unlikely(!IS_OVERFLOW(lp))) { + rc = bad_page(mp, "big-node refs to non-overflow page (%u)\n", + lp->mp_pgno); + continue; + } + if (unlikely(number_of_ovpages(env, dsize) != lp->mp_pages)) + rc = + bad_page(mp, "big-node size (%zu) mismatch n-pages size (%u)\n", + dsize, lp->mp_pages); } continue; } const size_t dsize = node_ds(node); const char *const data = node_data(node); - mdbx_assert(env, data + dsize <= end_of_page); - if (unlikely(end_of_page < data + dsize)) - return MDBX_CORRUPTED; + if (unlikely(end_of_page < data + dsize)) { + rc = bad_page(mp, + "node-data(%u of %u, %zu bytes) beyond (%zu) page-end\n", + i, nkeys, dsize, data + dsize - end_of_page); + continue; + } switch (node_flags(node)) { default: - mdbx_assert(env, false); - return MDBX_CORRUPTED; + /* wrong, but already handled */ + continue; case 0 /* usual */: if ((options & C_COPYING) == 0) { - mdbx_assert(env, dsize >= mc->mc_dbx->md_vlen_min); - mdbx_assert(env, dsize <= mc->mc_dbx->md_vlen_max); if (unlikely(dsize < mc->mc_dbx->md_vlen_min || - dsize > mc->mc_dbx->md_vlen_max)) - return MDBX_CORRUPTED; + dsize > mc->mc_dbx->md_vlen_max)) { + rc = bad_page( + mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n", + dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max); + continue; + } } break; case F_SUBDATA /* sub-db */: - mdbx_assert(env, dsize >= sizeof(MDBX_db)); - if (unlikely(dsize < sizeof(MDBX_db))) - return MDBX_CORRUPTED; + if (unlikely(dsize != sizeof(MDBX_db))) { + rc = bad_page(mp, "invalid sub-db record size (%zu)\n", dsize); + continue; + } break; case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: - mdbx_assert(env, dsize == sizeof(MDBX_db)); - if (unlikely(dsize != sizeof(MDBX_db))) - return MDBX_CORRUPTED; + if (unlikely(dsize != sizeof(MDBX_db))) { + rc = bad_page(mp, "invalid nested-db record size (%zu)\n", dsize); + continue; + } break; case F_DUPDATA /* short sub-page */: - mdbx_assert(env, dsize > PAGEHDRSZ); - if (unlikely(dsize <= PAGEHDRSZ)) - return MDBX_CORRUPTED; - else { + if (unlikely(dsize <= PAGEHDRSZ)) { + rc = bad_page(mp, "invalid nested-page record size (%zu)\n", dsize); + continue; + } else { const MDBX_page *const sp = (MDBX_page *)data; const char *const end_of_subpage = data + dsize; const int nsubkeys = page_numkeys(sp); @@ -14536,90 +18115,85 @@ static __cold int mdbx_page_check(MDBX_cursor *const mc, case P_LEAF | P_LEAF2 | P_SUBP: break; default: - mdbx_assert(env, false); - return MDBX_CORRUPTED; + rc = bad_page(mp, "invalid nested-page flags (%u)\n", sp->mp_flags); + continue; } MDBX_val sub_here, sub_prev = {0, 0}; for (int j = 0; j < nsubkeys; j++) { - mdbx_assert(env, IS_LEAF(sp)); - if (unlikely(!IS_LEAF(sp))) - return MDBX_CORRUPTED; if (IS_LEAF2(sp)) { /* LEAF2 pages have no mp_ptrs[] or node headers */ size_t sub_ksize = sp->mp_leaf2_ksize; char *sub_key = page_leaf2key(sp, j, sub_ksize); - mdbx_assert(env, sub_key + sub_ksize <= end_of_subpage); - if (unlikely(end_of_subpage < sub_key + sub_ksize)) - return MDBX_CORRUPTED; + if (unlikely(end_of_subpage < sub_key + sub_ksize)) { + rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n", + sub_key + sub_ksize - end_of_subpage); + continue; + } if ((options & C_COPYING) == 0) { if (unlikely(sub_ksize != mc->mc_dbx->md_vlen_min)) { - mdbx_assert(env, sub_ksize >= mc->mc_dbx->md_vlen_min); - mdbx_assert(env, sub_ksize <= mc->mc_dbx->md_vlen_max); if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min || - sub_ksize > mc->mc_dbx->md_vlen_max)) - return MDBX_CORRUPTED; + sub_ksize > mc->mc_dbx->md_vlen_max)) { + rc = bad_page(mp, + "nested-leaf2-key size (%zu) <> min/max " + "value-length (%zu/%zu)\n", + sub_ksize, mc->mc_dbx->md_vlen_min, + mc->mc_dbx->md_vlen_max); + continue; + } mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = sub_ksize; } if ((options & C_SKIPORD) == 0) { sub_here.iov_len = sub_ksize; sub_here.iov_base = sub_key; - if (sub_prev.iov_base) { - mdbx_assert(env, - mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) < 0); - if (unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= - 0)) - return MDBX_CORRUPTED; - } + if (sub_prev.iov_base && + unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0)) + rc = bad_page(mp, "nested-leaf2-key #%u wrong order\n", j); sub_prev = sub_here; } } } else { const MDBX_node *const sub_node = page_node(sp, j); const char *sub_node_end = (char *)sub_node + NODESIZE; - mdbx_assert(env, sub_node_end <= end_of_subpage); - if (unlikely(sub_node_end > end_of_subpage)) - return MDBX_CORRUPTED; - mdbx_assert(env, node_flags(sub_node) == 0); + if (unlikely(sub_node_end > end_of_subpage)) { + rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n", + end_of_subpage - sub_node_end); + continue; + } if (unlikely(node_flags(sub_node) != 0)) - return MDBX_CORRUPTED; + rc = bad_page(mp, "nested-node invalid flags (%u)\n", + node_flags(sub_node)); size_t sub_ksize = node_ks(sub_node); char *sub_key = node_key(sub_node); size_t sub_dsize = node_ds(sub_node); - char *sub_data = node_data(sub_node); + /* char *sub_data = node_data(sub_node); */ if ((options & C_COPYING) == 0) { - mdbx_assert(env, sub_ksize >= mc->mc_dbx->md_vlen_min); - mdbx_assert(env, sub_ksize <= mc->mc_dbx->md_vlen_max); if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min || sub_ksize > mc->mc_dbx->md_vlen_max)) - return MDBX_CORRUPTED; + rc = bad_page(mp, + "nested-node-key size (%zu) <> min/max " + "value-length (%zu/%zu)\n", + sub_ksize, mc->mc_dbx->md_vlen_min, + mc->mc_dbx->md_vlen_max); if ((options & C_SKIPORD) == 0) { sub_here.iov_len = sub_ksize; sub_here.iov_base = sub_key; - if (sub_prev.iov_base) { - mdbx_assert(env, - mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) < 0); - if (unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= - 0)) - return MDBX_CORRUPTED; - } + if (sub_prev.iov_base && + unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0)) + rc = bad_page(mp, "nested-node-key #%u wrong order\n", j); sub_prev = sub_here; } } - mdbx_assert(env, sub_dsize == 0); if (unlikely(sub_dsize != 0)) - return MDBX_CORRUPTED; - - mdbx_assert(env, sub_key + sub_ksize <= end_of_subpage); + rc = bad_page(mp, "nested-node non-empty data size (%zu)\n", + sub_dsize); if (unlikely(end_of_subpage < sub_key + sub_ksize)) - return MDBX_CORRUPTED; - mdbx_assert(env, sub_data + sub_dsize <= end_of_subpage); - if (unlikely(end_of_subpage < sub_data + sub_dsize)) - return MDBX_CORRUPTED; + rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n", + sub_key + sub_ksize - end_of_subpage); } } } @@ -14627,7 +18201,7 @@ static __cold int mdbx_page_check(MDBX_cursor *const mc, } } } - return MDBX_SUCCESS; + return rc; } static __cold int mdbx_cursor_check(MDBX_cursor *mc, unsigned options) { @@ -14680,7 +18254,8 @@ static __cold int mdbx_cursor_check(MDBX_cursor *mc, unsigned options) { return MDBX_CURSOR_FULL; pgno_t pgno = node_pgno(node); MDBX_page *np; - int rc = mdbx_page_get(mc, pgno, &np, NULL); + int rc = + mdbx_page_get(mc, pgno, &np, NULL, pp_txnid4chk(mp, mc->mc_txn)); mdbx_cassert(mc, rc == MDBX_SUCCESS); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -14788,7 +18363,7 @@ static int mdbx_cursor_del0(MDBX_cursor *mc) { if (!(node_flags(node) & F_SUBDATA)) m3->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); } else { - rc = mdbx_xcursor_init1(m3, node); + rc = mdbx_xcursor_init1(m3, node, m3->mc_pg[m3->mc_top]); if (unlikely(rc != MDBX_SUCCESS)) break; m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; @@ -14818,7 +18393,7 @@ static int mdbx_cursor_del0(MDBX_cursor *mc) { if (!(node_flags(node) & F_SUBDATA)) mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); } else { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mc->mc_pg[mc->mc_top]); if (likely(rc != MDBX_SUCCESS)) mc->mc_xcursor->mx_cursor.mc_flags |= C_DEL; } @@ -14845,10 +18420,10 @@ int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; - if (unlikely(txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) - return (txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) + return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; return mdbx_del0(txn, dbi, key, data, 0); } @@ -14874,7 +18449,7 @@ static int mdbx_del0(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, data = &rdata; } else { op = MDBX_SET; - flags |= MDBX_NODUPDATA; + flags |= MDBX_ALLDUPS; } rc = mdbx_cursor_set(&cx.outer, (MDBX_val *)key, (MDBX_val *)data, op, &exact); @@ -15204,7 +18779,7 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *newkey, rc = mdbx_node_add_leaf2(mc, 0, newkey); } break; default: - rc = MDBX_CORRUPTED; + rc = bad_page(rp, "wrong page-type %u\n", PAGETYPE(rp)); } if (rc) goto done; @@ -15261,7 +18836,7 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *newkey, rc = mdbx_node_add_leaf2(mc, n, &rkey); } break; */ default: - rc = MDBX_CORRUPTED; + rc = bad_page(rp, "wrong page-type %u\n", PAGETYPE(rp)); } if (rc) goto done; @@ -15387,14 +18962,15 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; - if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE | - MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT))) + if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | + MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND | + MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE))) return MDBX_EINVAL; - if (unlikely(txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) - return (txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) + return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; MDBX_cursor_couple cx; rc = mdbx_cursor_init(&cx.outer, txn, dbi); @@ -15407,7 +18983,8 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, if (flags & MDBX_CURRENT) { rc = mdbx_cursor_get(&cx.outer, (MDBX_val *)key, NULL, MDBX_SET); if (likely(rc == MDBX_SUCCESS) && - (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT)) { + (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) && + (flags & MDBX_ALLDUPS) == 0) { /* LY: allows update (explicit overwrite) only for unique keys */ MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top], cx.outer.mc_ki[cx.outer.mc_top]); @@ -15431,7 +19008,7 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, #ifndef MDBX_WBUF #define MDBX_WBUF ((size_t)1024 * 1024) #endif -#define MDBX_EOF 0x10 /* mdbx_env_copyfd1() is done reading */ +#define MDBX_EOF 0x10 /* mdbx_env_copythr() is done reading */ /* State needed for a double-buffering compacting copy. */ typedef struct mdbx_copy { @@ -15539,7 +19116,8 @@ static int __cold mdbx_env_cwalk(mdbx_copy *my, pgno_t *pg, int flags) { couple.outer.mc_flags = couple.inner.mx_cursor.mc_flags = C_COPYING | C_SKIPORD; - rc = mdbx_page_get(&couple.outer, *pg, &couple.outer.mc_pg[0], NULL); + rc = mdbx_page_get(&couple.outer, *pg, &couple.outer.mc_pg[0], NULL, + my->mc_txn->mt_txnid); if (unlikely(rc != MDBX_SUCCESS)) return rc; rc = mdbx_page_search_root(&couple.outer, NULL, MDBX_PS_FIRST); @@ -15584,7 +19162,8 @@ static int __cold mdbx_env_cwalk(mdbx_copy *my, pgno_t *pg, int flags) { const pgno_t pgno = node_largedata_pgno(node); poke_pgno(node_data(node), my->mc_next_pgno); - rc = mdbx_page_get(&couple.outer, pgno, &omp, NULL); + rc = mdbx_page_get(&couple.outer, pgno, &omp, NULL, + pp_txnid4chk(mp, my->mc_txn)); if (unlikely(rc != MDBX_SUCCESS)) goto done; if (my->mc_wlen[toggle] >= MDBX_WBUF) { @@ -15607,7 +19186,7 @@ static int __cold mdbx_env_cwalk(mdbx_copy *my, pgno_t *pg, int flags) { toggle = my->mc_toggle; } } else if (node_flags(node) & F_SUBDATA) { - if (node_ds(node) < sizeof(MDBX_db)) { + if (node_ds(node) != sizeof(MDBX_db)) { rc = MDBX_CORRUPTED; goto done; } @@ -15638,7 +19217,7 @@ static int __cold mdbx_env_cwalk(mdbx_copy *my, pgno_t *pg, int flags) { rc = mdbx_page_get( &couple.outer, node_pgno(page_node(mp, couple.outer.mc_ki[couple.outer.mc_top])), - &mp, NULL); + &mp, NULL, pp_txnid4chk(mp, my->mc_txn)); if (unlikely(rc != MDBX_SUCCESS)) goto done; couple.outer.mc_top++; @@ -15728,10 +19307,10 @@ static int __cold mdbx_env_compact(MDBX_env *env, MDBX_txn *read_txn, MDBX_meta *const meta = mdbx_init_metas(env, buffer); mdbx_meta_set_txnid(env, meta, read_txn->mt_txnid); - if (flags & MDBX_CP_FORCE_RESIZEABLE) + if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE) make_sizeable(meta); - /* copy canary sequenses if present */ + /* copy canary sequences if present */ if (read_txn->mt_canary.v) { meta->mm_canary = read_txn->mt_canary; meta->mm_canary.v = mdbx_meta_txnid_stable(env, meta); @@ -15823,9 +19402,9 @@ static int __cold mdbx_env_compact(MDBX_env *env, MDBX_txn *read_txn, return MDBX_CORRUPTED; /* page leak or corrupt DB */ } if (root < new_root) { - mdbx_notice("post-compactification root %" PRIaPGNO - " LT expected %" PRIaPGNO " (page leak(s) in source DB)", - root, new_root); + mdbx_warning("post-compactification root %" PRIaPGNO + " LT expected %" PRIaPGNO " (page leak(s) in source DB)", + root, new_root); /* fixup meta */ meta->mm_dbs[MAIN_DBI].md_root = root; meta->mm_geo.next = root + 1; @@ -15845,7 +19424,7 @@ static int __cold mdbx_env_compact(MDBX_env *env, MDBX_txn *read_txn, for (size_t offset = used_size; offset < whole_size;) { const size_t chunk = (MDBX_WBUF < whole_size - offset) ? MDBX_WBUF : whole_size - offset; - /* copy to avoit EFAULT in case swapped-out */ + /* copy to avoid EFAULT in case swapped-out */ int rc = mdbx_write(fd, data_buffer, chunk); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -15869,7 +19448,7 @@ static int __cold mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn, if (unlikely(rc != MDBX_SUCCESS)) return rc; - rc = mdbx_txn_renew0(read_txn, MDBX_RDONLY); + rc = mdbx_txn_renew0(read_txn, MDBX_TXN_RDONLY); if (unlikely(rc != MDBX_SUCCESS)) { mdbx_txn_unlock(env); return rc; @@ -15880,11 +19459,11 @@ static int __cold mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn, /* Make a snapshot of meta-pages, * but writing ones after the data was flushed */ memcpy(buffer, env->me_map, meta_bytes); - MDBX_meta *const headcopy = /* LY: get pointer to the spanshot copy */ + MDBX_meta *const headcopy = /* LY: get pointer to the snapshot copy */ (MDBX_meta *)(buffer + ((uint8_t *)mdbx_meta_head(env) - env->me_map)); mdbx_txn_unlock(env); - if (flags & MDBX_CP_FORCE_RESIZEABLE) + if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE) make_sizeable(headcopy); /* Update signature to steady */ headcopy->mm_datasync_sign = mdbx_meta_sign(headcopy); @@ -15900,38 +19479,44 @@ static int __cold mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn, uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, env->me_os_psize); for (size_t offset = meta_bytes; rc == MDBX_SUCCESS && offset < used_size;) { - if (dest_is_pipe) { -#if (defined(__linux__) || defined(__gnu_linux__)) && \ - (!defined(__ANDROID_API__) || __ANDROID_API__ >= 21) && \ - !defined(MDBX_SAFE4QEMU) +#if MDBX_USE_SENDFILE + static bool sendfile_unavailable; + if (dest_is_pipe && likely(!sendfile_unavailable)) { off_t in_offset = offset; - const intptr_t written = + const ssize_t written = sendfile(fd, env->me_lazy_fd, &in_offset, used_size - offset); - if (unlikely(written <= 0)) { - rc = written ? errno : MDBX_ENODATA; - break; + if (likely(written > 0)) { + offset = in_offset; + continue; } - offset = in_offset; - continue; -#endif - } else { -#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) && !defined(MDBX_SAFE4QEMU) + rc = MDBX_ENODATA; + if (written == 0 || ignore_enosys(rc = errno) != MDBX_RESULT_TRUE) + break; + sendfile_unavailable = true; + } +#endif /* MDBX_USE_SENDFILE */ + +#if MDBX_USE_COPYFILERANGE + static bool copyfilerange_unavailable; + if (!dest_is_pipe && likely(!copyfilerange_unavailable)) { off_t in_offset = offset, out_offset = offset; ssize_t bytes_copied = copy_file_range( env->me_lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0); - if (unlikely(bytes_copied <= 0)) { - rc = bytes_copied ? errno : MDBX_ENODATA; - break; + if (likely(bytes_copied > 0)) { + offset = in_offset; + continue; } - offset = in_offset; - continue; -#endif + rc = MDBX_ENODATA; + if (bytes_copied == 0 || ignore_enosys(rc = errno) != MDBX_RESULT_TRUE) + break; + copyfilerange_unavailable = true; } +#endif /* MDBX_USE_COPYFILERANGE */ /* fallback to portable */ const size_t chunk = (MDBX_WBUF < used_size - offset) ? MDBX_WBUF : used_size - offset; - /* copy to avoit EFAULT in case swapped-out */ + /* copy to avoid EFAULT in case swapped-out */ memcpy(data_buffer, env->me_map + offset, chunk); rc = mdbx_write(fd, data_buffer, chunk); offset += chunk; @@ -15947,7 +19532,7 @@ static int __cold mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn, rc == MDBX_SUCCESS && offset < whole_size;) { const size_t chunk = (MDBX_WBUF < whole_size - offset) ? MDBX_WBUF : whole_size - offset; - /* copy to avoit EFAULT in case swapped-out */ + /* copy to avoid EFAULT in case swapped-out */ rc = mdbx_write(fd, data_buffer, chunk); offset += chunk; } @@ -15959,18 +19544,16 @@ static int __cold mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn, int __cold mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, unsigned flags) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; const int dest_is_pipe = mdbx_is_pipe(fd); if (MDBX_IS_ERROR(dest_is_pipe)) return dest_is_pipe; if (!dest_is_pipe) { - int rc = mdbx_fseek(fd, 0); + rc = mdbx_fseek(fd, 0); if (unlikely(rc != MDBX_SUCCESS)) return rc; } @@ -15981,14 +19564,14 @@ int __cold mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, env->me_os_psize); uint8_t *buffer = NULL; - int rc = mdbx_memalign_alloc(env->me_os_psize, buffer_size, (void **)&buffer); + rc = mdbx_memalign_alloc(env->me_os_psize, buffer_size, (void **)&buffer); if (unlikely(rc != MDBX_SUCCESS)) return rc; MDBX_txn *read_txn = NULL; /* Do the lock/unlock of the reader mutex before starting the * write txn. Otherwise other read txns could block writers. */ - rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &read_txn); + rc = mdbx_txn_begin(env, NULL, MDBX_TXN_RDONLY, &read_txn); if (unlikely(rc != MDBX_SUCCESS)) { mdbx_memalign_free(buffer); return rc; @@ -16010,38 +19593,66 @@ int __cold mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, if (!dest_is_pipe) { if (likely(rc == MDBX_SUCCESS)) - rc = mdbx_filesync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE); + rc = mdbx_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE); /* Write actual meta */ if (likely(rc == MDBX_SUCCESS)) rc = mdbx_pwrite(fd, buffer, pgno2bytes(env, NUM_METAS), 0); if (likely(rc == MDBX_SUCCESS)) - rc = mdbx_filesync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + rc = mdbx_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); } mdbx_memalign_free(buffer); return rc; } -int __cold mdbx_env_copy(MDBX_env *env, const char *dest_path, unsigned flags) { - if (unlikely(!env || !dest_path)) - return MDBX_EINVAL; +int __cold mdbx_env_copy(MDBX_env *env, const char *dest_path, + MDBX_copy_flags_t flags) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if (unlikely(!dest_path)) + return MDBX_EINVAL; /* The destination path must exist, but the destination file must not. * We don't want the OS to cache the writes, since the source data is * already in the OS cache. */ mdbx_filehandle_t newfd; - int rc = mdbx_openfile(MDBX_OPEN_COPY, env, dest_path, &newfd, + rc = mdbx_openfile(MDBX_OPEN_COPY, env, dest_path, &newfd, #if defined(_WIN32) || defined(_WIN64) - (mode_t)-1 + (mdbx_mode_t)-1 #else - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP #endif ); + + if (rc == MDBX_SUCCESS) { +#if defined(_WIN32) || defined(_WIN64) + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(newfd, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, 0, INT32_MAX, &ov)) + rc = GetLastError(); +#else + struct flock lock_op; + memset(&lock_op, 0, sizeof(lock_op)); + lock_op.l_type = F_WRLCK; + lock_op.l_whence = SEEK_SET; + lock_op.l_start = 0; + lock_op.l_len = + (sizeof(lock_op.l_len) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff; + if (fcntl(newfd, F_SETLK, &lock_op) +#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \ + (!defined(__ANDROID_API__) || __ANDROID_API__ >= 24) + || flock(newfd, LOCK_EX | LOCK_NB) +#endif /* Linux */ + ) + rc = errno; +#endif /* Windows / POSIX */ + } + if (rc == MDBX_SUCCESS) rc = mdbx_env_copy2fd(env, newfd, flags); @@ -16058,12 +19669,11 @@ int __cold mdbx_env_copy(MDBX_env *env, const char *dest_path, unsigned flags) { /******************************************************************************/ -int __cold mdbx_env_set_flags(MDBX_env *env, unsigned flags, int onoff) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; +int __cold mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, + bool onoff) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; if (unlikely(flags & ~ENV_CHANGEABLE_FLAGS)) return MDBX_EPERM; @@ -16074,12 +19684,12 @@ int __cold mdbx_env_set_flags(MDBX_env *env, unsigned flags, int onoff) { if (unlikely(env->me_txn0->mt_owner == mdbx_thread_self())) return MDBX_BUSY; - int rc = mdbx_txn_lock(env, false); + rc = mdbx_txn_lock(env, false); if (unlikely(rc)) return rc; if (onoff) - env->me_flags = merge_flags(env->me_flags, flags); + env->me_flags = merge_sync_flags(env->me_flags, flags); else env->me_flags &= ~flags; @@ -16088,22 +19698,21 @@ int __cold mdbx_env_set_flags(MDBX_env *env, unsigned flags, int onoff) { } int __cold mdbx_env_get_flags(const MDBX_env *env, unsigned *arg) { - if (unlikely(!env || !arg)) - return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if (unlikely(!arg)) + return MDBX_EINVAL; *arg = env->me_flags & ENV_USABLE_FLAGS; return MDBX_SUCCESS; } int __cold mdbx_env_set_userctx(MDBX_env *env, void *ctx) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; env->me_userctx = ctx; return MDBX_SUCCESS; @@ -16114,11 +19723,9 @@ void *__cold mdbx_env_get_userctx(const MDBX_env *env) { } int __cold mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; #if MDBX_DEBUG env->me_assert_func = func; @@ -16130,22 +19737,24 @@ int __cold mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) { } int __cold mdbx_env_get_path(const MDBX_env *env, const char **arg) { - if (unlikely(!env || !arg)) - return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if (unlikely(!arg)) + return MDBX_EINVAL; *arg = env->me_path; return MDBX_SUCCESS; } int __cold mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) { - if (unlikely(!env || !arg)) - return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if (unlikely(!arg)) + return MDBX_EINVAL; *arg = env->me_lazy_fd; return MDBX_SUCCESS; @@ -16184,8 +19793,9 @@ int __cold mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, return err; } if (env) { - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int err = check_env(env); + if (unlikely(err != MDBX_SUCCESS)) + return err; if (txn && unlikely(txn->mt_env != env)) return MDBX_EINVAL; } @@ -16220,7 +19830,7 @@ int __cold mdbx_dbi_dupsort_depthmask(MDBX_txn *txn, MDBX_dbi dbi, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_VALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; MDBX_cursor_couple cx; rc = mdbx_cursor_init(&cx.outer, txn, dbi); @@ -16252,6 +19862,7 @@ int __cold mdbx_dbi_dupsort_depthmask(MDBX_txn *txn, MDBX_dbi dbi, *mask |= 1 << unaligned_peek_u16(1, &db->md_depth); break; default: + mdbx_error("wrong node-flags %u", flags); return MDBX_CORRUPTED; } rc = mdbx_cursor_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP); @@ -16275,8 +19886,9 @@ int __cold mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, return err; } if (env) { - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int err = check_env(env); + if (unlikely(err != MDBX_SUCCESS)) + return err; if (txn && unlikely(txn->mt_env != env)) return MDBX_EINVAL; } else { @@ -16304,12 +19916,12 @@ int __cold mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, arg->mi_meta2_txnid = mdbx_meta_txnid_fluid(env, meta2); arg->mi_meta2_sign = meta2->mm_datasync_sign; if (likely(bytes > size_before_bootid)) { - arg->mi_bootid.meta0.l = meta0->mm_bootid.x; - arg->mi_bootid.meta1.l = meta0->mm_bootid.x; - arg->mi_bootid.meta2.l = meta0->mm_bootid.x; - arg->mi_bootid.meta0.h = meta0->mm_bootid.y; - arg->mi_bootid.meta1.h = meta0->mm_bootid.y; - arg->mi_bootid.meta2.h = meta0->mm_bootid.y; + arg->mi_bootid.meta0.x = meta0->mm_bootid.x; + arg->mi_bootid.meta1.x = meta0->mm_bootid.x; + arg->mi_bootid.meta2.x = meta0->mm_bootid.x; + arg->mi_bootid.meta0.y = meta0->mm_bootid.y; + arg->mi_bootid.meta1.y = meta0->mm_bootid.y; + arg->mi_bootid.meta2.y = meta0->mm_bootid.y; } const MDBX_meta *txn_meta = recent_meta; @@ -16319,7 +19931,7 @@ int __cold mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, arg->mi_last_pgno = txn->mt_next_pgno - 1; arg->mi_geo.current = pgno2bytes(env, txn->mt_end_pgno); - const txnid_t wanna_meta_txnid = (txn->mt_flags & MDBX_RDONLY) + const txnid_t wanna_meta_txnid = (txn->mt_flags & MDBX_TXN_RDONLY) ? txn->mt_txnid : txn->mt_txnid - MDBX_TXNID_STEP; txn_meta = (arg->mi_meta0_txnid == wanna_meta_txnid) ? meta0 : txn_meta; @@ -16364,8 +19976,8 @@ int __cold mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, arg->mi_autosync_threshold = pgno2bytes(env, *env->me_autosync_threshold); arg->mi_autosync_period_seconds16dot16 = mdbx_osal_monotime_to_16dot16(*env->me_autosync_period); - arg->mi_bootid.current.l = bootid.x; - arg->mi_bootid.current.h = bootid.y; + arg->mi_bootid.current.x = bootid.x; + arg->mi_bootid.current.y = bootid.y; arg->mi_mode = lck ? lck->mti_envmode : env->me_flags; } @@ -16421,7 +20033,7 @@ static int mdbx_dbi_bind(MDBX_txn *txn, const MDBX_dbi dbi, unsigned user_flags, * seems that is case #1 above */ user_flags = txn->mt_dbs[dbi].md_flags; } else if ((user_flags & MDBX_CREATE) && txn->mt_dbs[dbi].md_entries == 0) { - if (txn->mt_flags & MDBX_RDONLY) + if (txn->mt_flags & MDBX_TXN_RDONLY) return /* FIXME: return extended info */ MDBX_EACCESS; /* make sure flags changes get committed */ txn->mt_dbs[dbi].md_flags = user_flags & DB_PERSISTENT_FLAGS; @@ -16452,9 +20064,9 @@ static int mdbx_dbi_bind(MDBX_txn *txn, const MDBX_dbi dbi, unsigned user_flags, return MDBX_SUCCESS; } -int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, - MDBX_cmp_func *datacmp) { +static int dbi_open(MDBX_txn *txn, const char *table_name, unsigned user_flags, + MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp) { int rc = MDBX_EINVAL; if (unlikely(!dbi)) return rc; @@ -16559,13 +20171,13 @@ int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags, rc = MDBX_INCOMPATIBLE; goto early_bailout; } - if (unlikely(data.iov_len < sizeof(MDBX_db))) { + if (unlikely(data.iov_len != sizeof(MDBX_db))) { rc = MDBX_CORRUPTED; goto early_bailout; } } - if (rc != MDBX_SUCCESS && unlikely(txn->mt_flags & MDBX_RDONLY)) { + if (rc != MDBX_SUCCESS && unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) { rc = MDBX_EACCESS; goto early_bailout; } @@ -16671,9 +20283,15 @@ int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags, return rc; } -int mdbx_dbi_open(MDBX_txn *txn, const char *table_name, unsigned table_flags, - MDBX_dbi *dbi) { - return mdbx_dbi_open_ex(txn, table_name, table_flags, dbi, nullptr, nullptr); +int mdbx_dbi_open(MDBX_txn *txn, const char *table_name, + MDBX_db_flags_t table_flags, MDBX_dbi *dbi) { + return dbi_open(txn, table_name, table_flags, dbi, nullptr, nullptr); +} + +int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, + MDBX_db_flags_t table_flags, MDBX_dbi *dbi, + MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { + return dbi_open(txn, table_name, table_flags, dbi, keycmp, datacmp); } int __cold mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, @@ -16686,7 +20304,7 @@ int __cold mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_VALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) @@ -16723,16 +20341,14 @@ static int mdbx_dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) { } int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; if (unlikely(dbi < CORE_DBS || dbi >= env->me_maxdbs)) - return MDBX_EINVAL; + return MDBX_BAD_DBI; - int rc = mdbx_fastmutex_acquire(&env->me_dbi_lock); + rc = mdbx_fastmutex_acquire(&env->me_dbi_lock); if (likely(rc == MDBX_SUCCESS)) { rc = mdbx_dbi_close_locked(env, dbi); mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); @@ -16750,7 +20366,7 @@ int mdbx_dbi_flags_ex(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_VALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; *flags = txn->mt_dbs[dbi].md_flags & DB_PERSISTENT_FLAGS; *state = @@ -16797,7 +20413,8 @@ static int mdbx_drop0(MDBX_cursor *mc, int subs) { MDBX_node *node = page_node(mp, i); if (node_flags(node) & F_BIGDATA) { MDBX_page *omp; - rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL); + rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL, + pp_txnid4chk(mp, mc->mc_txn)); if (unlikely(rc)) goto done; mdbx_cassert(mc, IS_OVERFLOW(omp)); @@ -16807,7 +20424,7 @@ static int mdbx_drop0(MDBX_cursor *mc, int subs) { if (!mc->mc_db->md_overflow_pages && !subs) break; } else if (subs && (node_flags(node) & F_SUBDATA)) { - rc = mdbx_xcursor_init1(mc, node); + rc = mdbx_xcursor_init1(mc, node, mp); if (unlikely(rc != MDBX_SUCCESS)) goto done; rc = mdbx_drop0(&mc->mc_xcursor->mx_cursor, 0); @@ -16856,16 +20473,13 @@ static int mdbx_drop0(MDBX_cursor *mc, int subs) { return rc; } -int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, int del) { +int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) { int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (unlikely(1 < (unsigned)del)) - return MDBX_EINVAL; - if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; if (unlikely(TXN_DBI_CHANGED(txn, dbi))) return MDBX_BAD_DBI; @@ -16876,7 +20490,7 @@ int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, int del) { return rc; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) { - rc = MDBX_EINVAL; + rc = MDBX_BAD_DBI; goto bailout; } @@ -16934,7 +20548,7 @@ int mdbx_set_compare(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) { return rc; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; txn->mt_dbxs[dbi].md_cmp = cmp; return MDBX_SUCCESS; @@ -16946,7 +20560,7 @@ int mdbx_set_dupsort(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) { return rc; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; txn->mt_dbxs[dbi].md_dcmp = cmp; return MDBX_SUCCESS; @@ -16954,13 +20568,14 @@ int mdbx_set_dupsort(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) { int __cold mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, void *ctx) { - if (unlikely(!env || !func)) - return MDBX_EINVAL; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if (unlikely(!func)) + return MDBX_EINVAL; - int rc = MDBX_RESULT_TRUE; + rc = MDBX_RESULT_TRUE; int serial = 0; if (likely(env->me_lck)) { const unsigned snap_nreaders = env->me_lck->mti_numreaders; @@ -17018,7 +20633,7 @@ int __cold mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, /* Insert pid into list if not already present. * return -1 if already present. */ -static int __cold mdbx_pid_insert(uint32_t *ids, uint32_t pid) { +static bool __cold mdbx_pid_insert(uint32_t *ids, uint32_t pid) { /* binary search of pid in list */ unsigned base = 0; unsigned cursor = 1; @@ -17037,7 +20652,7 @@ static int __cold mdbx_pid_insert(uint32_t *ids, uint32_t pid) { n -= pivot + 1; } else { /* found, so it's a duplicate */ - return -1; + return false; } } @@ -17048,35 +20663,27 @@ static int __cold mdbx_pid_insert(uint32_t *ids, uint32_t pid) { for (n = ids[0]; n > cursor; n--) ids[n] = ids[n - 1]; ids[n] = pid; - return 0; + return true; } int __cold mdbx_reader_check(MDBX_env *env, int *dead) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; - if (dead) *dead = 0; - return mdbx_reader_check0(env, false, dead); + return mdbx_cleanup_dead_readers(env, false, dead); } /* Return: * MDBX_RESULT_TRUE - done and mutex recovered * MDBX_SUCCESS - done * Otherwise errcode. */ -int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) { - mdbx_assert(env, rdt_locked >= 0); - -#if MDBX_TXN_CHECKPID - if (unlikely(env->me_pid != mdbx_getpid())) { - env->me_flags |= MDBX_FATAL_ERROR; - return MDBX_PANIC; - } -#endif /* MDBX_TXN_CHECKPID */ +MDBX_INTERNAL_FUNC int __cold mdbx_cleanup_dead_readers(MDBX_env *env, + int rdt_locked, + int *dead) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + mdbx_assert(env, rdt_locked >= 0); MDBX_lockinfo *const lck = env->me_lck; if (unlikely(lck == NULL)) { /* exclusive mode */ @@ -17096,15 +20703,14 @@ int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) { return MDBX_ENOMEM; pids[0] = 0; - - int rc = MDBX_SUCCESS, count = 0; + int count = 0; for (unsigned i = 0; i < snap_nreaders; i++) { const uint32_t pid = lck->mti_readers[i].mr_pid; if (pid == 0) continue /* skip empty */; if (pid == env->me_pid) continue /* skip self */; - if (mdbx_pid_insert(pids, pid) != 0) + if (!mdbx_pid_insert(pids, pid)) continue /* such pid already processed */; int err = mdbx_rpid_check(env, pid); @@ -17151,6 +20757,7 @@ int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) { mdbx_debug("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, lck->mti_readers[j].mr_txnid.inconsistent); lck->mti_readers[j].mr_pid = 0; + mdbx_compiler_barrier(); lck->mti_readers_refresh_flag = true; count++; } @@ -17171,12 +20778,8 @@ int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) { int __cold mdbx_setup_debug(int loglevel, int flags, MDBX_debug_func *logger) { const int rc = mdbx_runtime_flags | (mdbx_loglevel << 16); -#if !MDBX_DEBUG - (void)loglevel; -#else if (loglevel != MDBX_LOG_DONTCHANGE) mdbx_loglevel = (uint8_t)loglevel; -#endif if (flags != MDBX_DBG_DONTCHANGE) { flags &= @@ -17192,8 +20795,9 @@ int __cold mdbx_setup_debug(int loglevel, int flags, MDBX_debug_func *logger) { return rc; } -static txnid_t __cold mdbx_oomkick(MDBX_env *env, const txnid_t laggard) { - mdbx_debug("%s", "DB size maxed out"); +static txnid_t __cold mdbx_kick_longlived_readers(MDBX_env *env, + const txnid_t laggard) { + mdbx_debug("DB size maxed out by reading #%" PRIaTXN, laggard); int retry; for (retry = 0; retry < INT_MAX; ++retry) { @@ -17201,10 +20805,10 @@ static txnid_t __cold mdbx_oomkick(MDBX_env *env, const txnid_t laggard) { mdbx_assert(env, oldest < env->me_txn0->mt_txnid); mdbx_assert(env, oldest >= laggard); mdbx_assert(env, oldest >= *env->me_oldest); - if (oldest == laggard || unlikely(env->me_lck == NULL /* exclusive mode */)) + if (oldest == laggard || unlikely(!env->me_lck /* without-LCK mode */)) return oldest; - if (MDBX_IS_ERROR(mdbx_reader_check0(env, false, NULL))) + if (MDBX_IS_ERROR(mdbx_cleanup_dead_readers(env, false, NULL))) break; MDBX_reader *asleep = nullptr; @@ -17233,20 +20837,20 @@ static txnid_t __cold mdbx_oomkick(MDBX_env *env, const txnid_t laggard) { } if (laggard < oldest || !asleep) { - if (retry && env->me_oom_func) { - /* LY: notify end of oom-loop */ + if (retry && env->me_hsr_callback) { + /* LY: notify end of hsr-loop */ const txnid_t gap = oldest - laggard; - env->me_oom_func(env, 0, 0, laggard, - (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, 0, - -retry); + env->me_hsr_callback(env, env->me_txn, 0, 0, laggard, + (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, 0, + -retry); } - mdbx_notice("oom-kick: update oldest %" PRIaTXN " -> %" PRIaTXN, + mdbx_notice("hsr-kick: update oldest %" PRIaTXN " -> %" PRIaTXN, *env->me_oldest, oldest); mdbx_assert(env, *env->me_oldest <= oldest); return *env->me_oldest = oldest; } - if (!env->me_oom_func) + if (!env->me_hsr_callback) break; uint32_t pid = asleep->mr_pid; @@ -17262,9 +20866,9 @@ static txnid_t __cold mdbx_oomkick(MDBX_env *env, const txnid_t laggard) { (oldest_retired > head_retired) ? pgno2bytes(env, (pgno_t)(oldest_retired - head_retired)) : 0; - int rc = env->me_oom_func(env, pid, (mdbx_tid_t)tid, laggard, - (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, - space, retry); + int rc = env->me_hsr_callback( + env, env->me_txn, pid, (mdbx_tid_t)tid, laggard, + (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry); if (rc < 0) break; @@ -17276,26 +20880,25 @@ static txnid_t __cold mdbx_oomkick(MDBX_env *env, const txnid_t laggard) { asleep->mr_tid = 0; asleep->mr_pid = 0; } + mdbx_compiler_barrier(); lck->mti_readers_refresh_flag = true; mdbx_flush_incoherent_cpu_writeback(); } } - if (retry && env->me_oom_func) { - /* LY: notify end of oom-loop */ - env->me_oom_func(env, 0, 0, laggard, 0, 0, -retry); + if (retry && env->me_hsr_callback) { + /* LY: notify end of hsr-loop */ + env->me_hsr_callback(env, env->me_txn, 0, 0, laggard, 0, 0, -retry); } return mdbx_find_oldest(env->me_txn); } int __cold mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR))) + if (unlikely(env->me_flags & MDBX_RDONLY)) return MDBX_EACCESS; if (unlikely(!env->me_map)) @@ -17303,21 +20906,19 @@ int __cold mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) { *env->me_autosync_threshold = bytes2pgno(env, threshold + env->me_psize - 1); if (threshold) { - int err = mdbx_env_sync_poll(env); - if (unlikely(MDBX_IS_ERROR(err))) - return err; + rc = mdbx_env_sync_poll(env); + if (unlikely(MDBX_IS_ERROR(rc))) + return rc; } return MDBX_SUCCESS; } int __cold mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (unlikely(env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR))) + if (unlikely(env->me_flags & MDBX_RDONLY)) return MDBX_EACCESS; if (unlikely(!env->me_map)) @@ -17325,27 +20926,25 @@ int __cold mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) { *env->me_autosync_period = mdbx_osal_16dot16_to_monotime(seconds_16dot16); if (seconds_16dot16) { - int err = mdbx_env_sync_poll(env); - if (unlikely(MDBX_IS_ERROR(err))) - return err; + rc = mdbx_env_sync_poll(env); + if (unlikely(MDBX_IS_ERROR(rc))) + return rc; } return MDBX_SUCCESS; } -int __cold mdbx_env_set_oomfunc(MDBX_env *env, MDBX_oom_func *oomfunc) { - if (unlikely(!env)) - return MDBX_EINVAL; - - if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; +int __cold mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr) { + int rc = check_env(env); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - env->me_oom_func = oomfunc; + env->me_hsr_callback = hsr; return MDBX_SUCCESS; } -MDBX_oom_func *__cold mdbx_env_get_oomfunc(const MDBX_env *env) { +MDBX_hsr_func *__cold mdbx_env_get_hsr(const MDBX_env *env) { return likely(env && env->me_signature == MDBX_ME_SIGNATURE) - ? env->me_oom_func + ? env->me_hsr_callback : NULL; } @@ -17360,7 +20959,7 @@ int mdbx_txn_straggler(const MDBX_txn *txn, int *percent) return (rc > 0) ? -rc : rc; MDBX_env *env = txn->mt_env; - if (unlikely((txn->mt_flags & MDBX_RDONLY) == 0)) { + if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) { if (percent) *percent = (int)((txn->mt_next_pgno * UINT64_C(100) + txn->mt_end_pgno / 2) / @@ -17393,46 +20992,65 @@ typedef struct mdbx_walk_ctx { static int __cold mdbx_walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const db, const char *name, int deep); -/* Depth-first tree traversal. */ -static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, pgno_t pgno, - const char *name, int deep) { - assert(pgno != P_INVALID); - MDBX_page *mp; - int rc = mdbx_page_get(ctx->mw_cursor, pgno, &mp, NULL); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - rc = mdbx_page_check(ctx->mw_cursor, mp, 0); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +static MDBX_page_type_t walk_page_type(const MDBX_page *mp) { + if (mp) + switch (mp->mp_flags) { + case P_BRANCH: + return MDBX_page_branch; + case P_LEAF: + return MDBX_page_leaf; + case P_LEAF | P_LEAF2: + return MDBX_page_dupfixed_leaf; + case P_OVERFLOW: + return MDBX_page_large; + case P_META: + return MDBX_page_meta; + } + return MDBX_page_broken; +} - const int nkeys = page_numkeys(mp); - size_t header_size = IS_LEAF2(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->mp_lower; - size_t unused_size = page_room(mp); +/* Depth-first tree traversal. */ +static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, const pgno_t pgno, + const char *name, int deep, + txnid_t parent_txnid) { + assert(pgno != P_INVALID); + MDBX_page *mp = nullptr; + int rc, err = mdbx_page_get(ctx->mw_cursor, pgno, &mp, NULL, parent_txnid); + if (err == MDBX_SUCCESS) + err = mdbx_page_check(ctx->mw_cursor, mp, 0); + + MDBX_page_type_t type = walk_page_type(mp); + const int nentries = (mp && !IS_OVERFLOW(mp)) ? page_numkeys(mp) : 1; + unsigned npages = (mp && IS_OVERFLOW(mp)) ? mp->mp_pages : 1; + size_t pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages); + size_t header_size = (mp && !IS_LEAF2(mp) && !IS_OVERFLOW(mp)) + ? PAGEHDRSZ + mp->mp_lower + : PAGEHDRSZ; size_t payload_size = 0; + size_t unused_size = + (mp && !IS_OVERFLOW(mp) ? page_room(mp) : pagesize - header_size) - + payload_size; size_t align_bytes = 0; - MDBX_page_type_t type; - /* LY: Don't use mask here, e.g bitwise - * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP). - * Pages should not me marked dirty/loose or otherwise. */ - switch (mp->mp_flags) { - case P_BRANCH: - type = MDBX_page_branch; - if (unlikely(nkeys < 2)) - return MDBX_CORRUPTED; - break; - case P_LEAF: - type = MDBX_page_leaf; - break; - case P_LEAF | P_LEAF2: - type = MDBX_page_dupfixed_leaf; - break; - default: - return MDBX_CORRUPTED; + if (err == MDBX_SUCCESS) { + /* LY: Don't use mask here, e.g bitwise + * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP). + * Pages should not me marked dirty/loose or otherwise. */ + switch (mp->mp_flags) { + default: + err = MDBX_CORRUPTED; + break; + case P_BRANCH: + if (unlikely(nentries < 2)) + err = MDBX_CORRUPTED; + case P_LEAF: + case P_LEAF | P_LEAF2: + break; + } } - for (int i = 0; i < nkeys; + for (int i = 0; err == MDBX_SUCCESS && i < nentries; align_bytes += ((payload_size + align_bytes) & 1), i++) { if (type == MDBX_page_dupfixed_leaf) { /* LEAF2 pages have no mp_ptrs[] or node headers */ @@ -17450,56 +21068,59 @@ static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, pgno_t pgno, assert(type == MDBX_page_leaf); switch (node_flags(node)) { - case 0 /* usual node */: { + case 0 /* usual node */: payload_size += node_ds(node); - } break; + break; case F_BIGDATA /* long data on the large/overflow page */: { payload_size += sizeof(pgno_t); - const pgno_t large_pgno = node_largedata_pgno(node); - MDBX_page *op; - rc = mdbx_page_get(ctx->mw_cursor, large_pgno, &op, NULL); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - rc = mdbx_page_check(ctx->mw_cursor, op, 0); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + const size_t over_payload = node_ds(node); + const size_t over_header = PAGEHDRSZ; + npages = 1; - /* LY: Don't use mask here, e.g bitwise - * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP). - * Pages should not me marked dirty/loose or otherwise. */ - if (unlikely(P_OVERFLOW != op->mp_flags)) - return MDBX_CORRUPTED; + MDBX_page *op; + err = mdbx_page_get(ctx->mw_cursor, large_pgno, &op, NULL, + pp_txnid4chk(mp, ctx->mw_txn)); + if (err == MDBX_SUCCESS) + err = mdbx_page_check(ctx->mw_cursor, op, 0); + if (err == MDBX_SUCCESS) { + /* LY: Don't use mask here, e.g bitwise + * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP). + * Pages should not me marked dirty/loose or otherwise. */ + if (P_OVERFLOW != op->mp_flags) + err = bad_page(mp, "wrong page type %d for large data", op->mp_flags); + else + npages = op->mp_pages; + } - const size_t over_header = PAGEHDRSZ; - const size_t over_payload = node_ds(node); - const size_t over_unused = - pgno2bytes(ctx->mw_cursor->mc_txn->mt_env, op->mp_pages) - - over_payload - over_header; - - rc = ctx->mw_visitor( - large_pgno, op->mp_pages, ctx->mw_user, deep, name, - pgno2bytes(ctx->mw_cursor->mc_txn->mt_env, op->mp_pages), - MDBX_page_large, 1, over_payload, over_header, over_unused); + pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages); + const size_t over_unused = pagesize - over_payload - over_header; + rc = ctx->mw_visitor(large_pgno, npages, ctx->mw_user, deep, name, + pagesize, MDBX_page_large, err, 1, over_payload, + over_header, over_unused); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; } break; case F_SUBDATA /* sub-db */: { const size_t namelen = node_ks(node); - if (unlikely(namelen == 0 || node_ds(node) < sizeof(MDBX_db))) - return MDBX_CORRUPTED; payload_size += node_ds(node); + if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db))) + err = MDBX_CORRUPTED; } break; - case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: { - if (unlikely(node_ds(node) != sizeof(MDBX_db))) - return MDBX_CORRUPTED; + case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: payload_size += sizeof(MDBX_db); - } break; + if (unlikely(node_ds(node) != sizeof(MDBX_db))) + err = MDBX_CORRUPTED; + break; case F_DUPDATA /* short sub-page */: { - if (unlikely(node_ds(node) <= PAGEHDRSZ)) - return MDBX_CORRUPTED; + if (unlikely(node_ds(node) <= PAGEHDRSZ)) { + err = MDBX_CORRUPTED; + break; + } MDBX_page *sp = node_data(node); const int nsubkeys = page_numkeys(sp); @@ -17518,10 +21139,11 @@ static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, pgno_t pgno, subtype = MDBX_subpage_dupfixed_leaf; break; default: - return MDBX_CORRUPTED; + subtype = MDBX_subpage_broken; + err = MDBX_CORRUPTED; } - for (int j = 0; j < nsubkeys; + for (int j = 0; err == MDBX_SUCCESS && j < nsubkeys; subalign_bytes += ((subpayload_size + subalign_bytes) & 1), j++) { if (subtype == MDBX_subpage_dupfixed_leaf) { @@ -17532,13 +21154,15 @@ static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, pgno_t pgno, MDBX_node *subnode = page_node(sp, j); subpayload_size += NODESIZE + node_ks(subnode) + node_ds(subnode); if (unlikely(node_flags(subnode) != 0)) - return MDBX_CORRUPTED; + err = MDBX_CORRUPTED; } } rc = ctx->mw_visitor(pgno, 0, ctx->mw_user, deep + 1, name, node_ds(node), - subtype, nsubkeys, subpayload_size, subheader_size, - subunused_size + subalign_bytes); + subtype, err, nsubkeys, subpayload_size, + subheader_size, subunused_size + subalign_bytes); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; header_size += subheader_size; unused_size += subunused_size; payload_size += subpayload_size; @@ -17546,31 +21170,28 @@ static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, pgno_t pgno, } break; default: - return MDBX_CORRUPTED; + err = MDBX_CORRUPTED; } - - if (unlikely(rc != MDBX_SUCCESS)) - return rc; } - rc = ctx->mw_visitor(mp->mp_pgno, 1, ctx->mw_user, deep, name, - ctx->mw_cursor->mc_txn->mt_env->me_psize, type, nkeys, + rc = ctx->mw_visitor(pgno, 1, ctx->mw_user, deep, name, + ctx->mw_txn->mt_env->me_psize, type, err, nentries, payload_size, header_size, unused_size + align_bytes); - if (unlikely(rc != MDBX_SUCCESS)) return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; - for (int i = 0; i < nkeys; i++) { + for (int i = 0; err == MDBX_SUCCESS && i < nentries; i++) { if (type == MDBX_page_dupfixed_leaf) continue; MDBX_node *node = page_node(mp, i); if (type == MDBX_page_branch) { - rc = mdbx_walk_tree(ctx, node_pgno(node), name, deep + 1); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc != MDBX_RESULT_TRUE) - return rc; - break; + err = mdbx_walk_tree(ctx, node_pgno(node), name, deep + 1, + pp_txnid4chk(mp, ctx->mw_txn)); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_RESULT_TRUE) + break; + return err; } continue; } @@ -17583,8 +21204,10 @@ static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, pgno_t pgno, case F_SUBDATA /* sub-db */: { const size_t namelen = node_ks(node); - if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db))) - return MDBX_CORRUPTED; + if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db))) { + err = MDBX_CORRUPTED; + break; + } char namebuf_onstask[64]; char *const sub_name = (namelen < sizeof(namebuf_onstask)) @@ -17594,36 +21217,33 @@ static int __cold mdbx_walk_tree(mdbx_walk_ctx_t *ctx, pgno_t pgno, memcpy(sub_name, node_key(node), namelen); sub_name[namelen] = 0; memcpy(&db, node_data(node), sizeof(db)); - rc = mdbx_walk_sdb(ctx, &db, sub_name, deep + 1); + err = mdbx_walk_sdb(ctx, &db, sub_name, deep + 1); if (sub_name != namebuf_onstask) mdbx_free(sub_name); } else { - rc = MDBX_ENOMEM; + err = MDBX_ENOMEM; } } break; case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: - if (unlikely(node_ds(node) != sizeof(MDBX_db))) - return MDBX_CORRUPTED; - - if (unlikely(ctx->mw_cursor->mc_xcursor == NULL)) - return MDBX_CORRUPTED; - - memcpy(&db, node_data(node), sizeof(db)); - assert(ctx->mw_cursor->mc_xcursor == - &container_of(ctx->mw_cursor, MDBX_cursor_couple, outer)->inner); - ctx->mw_cursor = &ctx->mw_cursor->mc_xcursor->mx_cursor; - rc = mdbx_walk_tree(ctx, db.md_root, name, deep + 1); - MDBX_xcursor *inner_xcursor = - container_of(ctx->mw_cursor, MDBX_xcursor, mx_cursor); - MDBX_cursor_couple *couple = - container_of(inner_xcursor, MDBX_cursor_couple, inner); - ctx->mw_cursor = &couple->outer; + if (unlikely(node_ds(node) != sizeof(MDBX_db) || + ctx->mw_cursor->mc_xcursor == NULL)) + err = MDBX_CORRUPTED; + else { + memcpy(&db, node_data(node), sizeof(db)); + assert(ctx->mw_cursor->mc_xcursor == + &container_of(ctx->mw_cursor, MDBX_cursor_couple, outer)->inner); + ctx->mw_cursor = &ctx->mw_cursor->mc_xcursor->mx_cursor; + err = mdbx_walk_tree(ctx, db.md_root, name, deep + 1, + pp_txnid4chk(mp, ctx->mw_txn)); + MDBX_xcursor *inner_xcursor = + container_of(ctx->mw_cursor, MDBX_xcursor, mx_cursor); + MDBX_cursor_couple *couple = + container_of(inner_xcursor, MDBX_cursor_couple, inner); + ctx->mw_cursor = &couple->outer; + } break; } - - if (unlikely(rc != MDBX_SUCCESS)) - return rc; } return MDBX_SUCCESS; @@ -17647,13 +21267,13 @@ static int __cold mdbx_walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const db, } couple.outer.mc_next = ctx->mw_cursor; ctx->mw_cursor = &couple.outer; - rc = mdbx_walk_tree(ctx, db->md_root, name, deep); + rc = mdbx_walk_tree(ctx, db->md_root, name, deep, ctx->mw_txn->mt_txnid); ctx->mw_cursor = couple.outer.mc_next; return rc; } int __cold mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, - void *user, int dont_check_keys_ordering) { + void *user, bool dont_check_keys_ordering) { int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -17663,24 +21283,21 @@ int __cold mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, ctx.mw_txn = txn; ctx.mw_user = user; ctx.mw_visitor = visitor; - ctx.mw_dont_check_keys_ordering = dont_check_keys_ordering != 0; + ctx.mw_dont_check_keys_ordering = dont_check_keys_ordering; rc = visitor(0, NUM_METAS, user, 0, MDBX_PGWALK_META, - pgno2bytes(txn->mt_env, NUM_METAS), MDBX_page_meta, NUM_METAS, - sizeof(MDBX_meta) * NUM_METAS, PAGEHDRSZ * NUM_METAS, + pgno2bytes(txn->mt_env, NUM_METAS), MDBX_page_meta, MDBX_SUCCESS, + NUM_METAS, sizeof(MDBX_meta) * NUM_METAS, PAGEHDRSZ * NUM_METAS, (txn->mt_env->me_psize - sizeof(MDBX_meta) - PAGEHDRSZ) * NUM_METAS); if (!MDBX_IS_ERROR(rc)) rc = mdbx_walk_sdb(&ctx, &txn->mt_dbs[FREE_DBI], MDBX_PGWALK_GC, 0); if (!MDBX_IS_ERROR(rc)) rc = mdbx_walk_sdb(&ctx, &txn->mt_dbs[MAIN_DBI], MDBX_PGWALK_MAIN, 0); - if (!MDBX_IS_ERROR(rc)) - rc = visitor(P_INVALID, 0, user, INT_MIN, NULL, 0, MDBX_page_void, 0, 0, 0, - 0); return rc; } -int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary) { +int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) { int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -17699,7 +21316,7 @@ int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary) { return MDBX_SUCCESS; } -int mdbx_canary_get(const MDBX_txn *txn, mdbx_canary *canary) { +int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) { int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -17715,7 +21332,7 @@ int mdbx_cursor_on_first(const MDBX_cursor *mc) { if (unlikely(mc == NULL)) return MDBX_EINVAL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; if (!(mc->mc_flags & C_INITIALIZED)) @@ -17733,7 +21350,7 @@ int mdbx_cursor_on_last(const MDBX_cursor *mc) { if (unlikely(mc == NULL)) return MDBX_EINVAL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; if (!(mc->mc_flags & C_INITIALIZED)) @@ -17752,7 +21369,7 @@ int mdbx_cursor_eof(const MDBX_cursor *mc) { if (unlikely(mc == NULL)) return MDBX_EINVAL; - if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; if ((mc->mc_flags & C_INITIALIZED) == 0) @@ -17784,8 +21401,8 @@ __hot static int cursor_diff(const MDBX_cursor *const __restrict x, r->level = 0; r->root_nkeys = 0; - if (unlikely(y->mc_signature != MDBX_MC_SIGNATURE || - x->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(y->mc_signature != MDBX_MC_LIVE || + x->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; int rc = check_txn(x->mc_txn, MDBX_TXN_BLOCKED); @@ -17930,7 +21547,7 @@ int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, return rc; if (unlikely(dr.diff == 0) && - F_ISSET(first->mc_db->md_flags & first->mc_db->md_flags, + F_ISSET(first->mc_db->md_flags & last->mc_db->md_flags, MDBX_DUPSORT | C_INITIALIZED)) { first = &first->mc_xcursor->mx_cursor; last = &last->mc_xcursor->mx_cursor; @@ -17951,7 +21568,7 @@ int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE)) return MDBX_EINVAL; - if (unlikely(cursor->mc_signature != MDBX_MC_SIGNATURE)) + if (unlikely(cursor->mc_signature != MDBX_MC_LIVE)) return MDBX_EBADSIGN; int rc = check_txn(cursor->mc_txn, MDBX_TXN_BLOCKED); @@ -18019,7 +21636,7 @@ int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *begin_key, return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; MDBX_cursor_couple begin; /* LY: first, initialize cursor to refresh a DB in case it have DB_STALE */ @@ -18176,8 +21793,11 @@ int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *begin_key, * - внешняя аллокация курсоров, в том числе на стеке (без malloc). * - получения статуса страницы по адресу (знать о P_DIRTY). */ -int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *new_data, MDBX_val *old_data, unsigned flags) { + +int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, + MDBX_val *new_data, MDBX_val *old_data, + MDBX_put_flags_t flags, MDBX_preserve_func preserver, + void *preserver_context) { int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -18188,14 +21808,16 @@ int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, if (unlikely(old_data->iov_base == NULL && old_data->iov_len)) return MDBX_EINVAL; - if (unlikely(new_data == NULL && !(flags & MDBX_CURRENT))) + if (unlikely(new_data == NULL && + (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT)) return MDBX_EINVAL; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; - if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE | - MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT))) + if (unlikely(flags & + ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | + MDBX_RESERVE | MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT))) return MDBX_EINVAL; MDBX_cursor_couple cx; @@ -18219,13 +21841,6 @@ int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, rc = mdbx_cursor_get(&cx.outer, &present_key, old_data, MDBX_GET_BOTH); if (rc != MDBX_SUCCESS) goto bailout; - - if (new_data) { - /* обновление конкретного дубликата */ - if (cx.outer.mc_dbx->md_dcmp(old_data, new_data) == 0) - /* если данные совпадают, то ничего делать не надо */ - goto bailout; - } } else { /* в old_data буфер для сохранения предыдущего значения */ if (unlikely(new_data && old_data->iov_base == new_data->iov_base)) @@ -18255,57 +21870,60 @@ int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, goto bailout; } } - /* если данные совпадают, то ничего делать не надо */ - if (new_data && - cx.outer.mc_dbx->md_dcmp(&present_data, new_data) == 0) { - *old_data = *new_data; - goto bailout; - } /* В оригинальной LMDB флажок MDBX_CURRENT здесь приведет * к замене данных без учета MDBX_DUPSORT сортировки, * но здесь это в любом случае допустимо, так как мы * проверили что для ключа есть только одно значение. */ - } else if ((flags & MDBX_NODUPDATA) && - cx.outer.mc_dbx->md_dcmp(&present_data, new_data) == 0) { - /* если данные совпадают и установлен MDBX_NODUPDATA */ - rc = MDBX_KEYEXIST; - goto bailout; - } - } else { - /* если данные совпадают, то ничего делать не надо */ - if (new_data && - cx.outer.mc_dbx->md_dcmp(&present_data, new_data) == 0) { - *old_data = *new_data; - goto bailout; } - flags |= MDBX_CURRENT; } if (IS_DIRTY(page)) { - if (unlikely(old_data->iov_len < present_data.iov_len)) { - old_data->iov_base = NULL; - old_data->iov_len = present_data.iov_len; - rc = MDBX_RESULT_TRUE; + if (new_data && cmp_lenfast(&present_data, new_data) == 0) { + /* если данные совпадают, то ничего делать не надо */ + *old_data = *new_data; goto bailout; } - memcpy(old_data->iov_base, present_data.iov_base, present_data.iov_len); - old_data->iov_len = present_data.iov_len; + rc = preserver ? preserver(preserver_context, old_data, + present_data.iov_base, present_data.iov_len) + : MDBX_SUCCESS; + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; } else { *old_data = present_data; } + flags |= MDBX_CURRENT; } } if (likely(new_data)) rc = mdbx_cursor_put(&cx.outer, key, new_data, flags); else - rc = mdbx_cursor_del(&cx.outer, 0); + rc = mdbx_cursor_del(&cx.outer, flags & MDBX_ALLDUPS); bailout: txn->mt_cursors[dbi] = cx.outer.mc_next; return rc; } +static int default_value_preserver(void *context, MDBX_val *target, + const void *src, size_t bytes) { + (void)context; + if (unlikely(target->iov_len < bytes)) { + target->iov_base = nullptr; + target->iov_len = bytes; + return MDBX_RESULT_TRUE; + } + memcpy(target->iov_base, src, target->iov_len = bytes); + return MDBX_SUCCESS; +} + +int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, + MDBX_val *new_data, MDBX_val *old_data, + MDBX_put_flags_t flags) { + return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags, + default_value_preserver, nullptr); +} + /* Функция сообщает находится ли указанный адрес в "грязной" странице у * заданной пишущей транзакции. В конечном счете это позволяет избавиться от * лишнего копирования данных из НЕ-грязных страниц. @@ -18333,21 +21951,21 @@ int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (txn->mt_flags & MDBX_RDONLY) - return MDBX_RESULT_FALSE; - const MDBX_env *env = txn->mt_env; const ptrdiff_t offset = (uint8_t *)ptr - env->me_map; if (offset >= 0) { const pgno_t pgno = bytes2pgno(env, offset); if (likely(pgno < txn->mt_next_pgno)) { + if (txn->mt_flags & MDBX_TXN_RDONLY) + return MDBX_RESULT_FALSE; + const MDBX_page *page = pgno2page(env, pgno); if (unlikely(page->mp_pgno != pgno)) { /* The ptr pointed into middle of a large page, * not to the beginning of a data. */ return MDBX_EINVAL; } - if (unlikely(page->mp_flags & (P_DIRTY | P_LOOSE | P_KEEP))) + if (unlikely(page->mp_flags & (P_DIRTY | P_LOOSE | P_KEEP | P_META))) return MDBX_RESULT_TRUE; if (likely(txn->tw.spill_pages == nullptr)) return MDBX_RESULT_FALSE; @@ -18357,7 +21975,7 @@ int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { if ((size_t)offset < env->me_dxb_mmap.limit) { /* Указатель адресует что-то в пределах mmap, но за границей * распределенных страниц. Такое может случится если mdbx_is_dirty() - * вызывает после операции, в ходе которой гразная страница попала + * вызывается после операции, в ходе которой грязная страница попала * в loose и затем была возвращена в нераспределенное пространство. */ return MDBX_RESULT_TRUE; } @@ -18367,7 +21985,7 @@ int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { * передан некорректный адрес, либо адрес в теневой странице, которая была * выделена посредством malloc(). * - * Для WRITE_MAP режима такая страница однозначно "не грязная", + * Для режима WRITE_MAP режима страница однозначно "не грязная", * а для режимов без WRITE_MAP следует просматривать списки dirty * и spilled страниц у каких-либо транзакций (в том числе дочерних). * @@ -18388,7 +22006,7 @@ int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, return rc; if (unlikely(!mdbx_txn_dbi_exists(txn, dbi, DBI_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; if (unlikely(TXN_DBI_CHANGED(txn, dbi))) return MDBX_BAD_DBI; @@ -18404,7 +22022,7 @@ int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, *result = dbs->md_seq; if (likely(increment > 0)) { - if (unlikely(txn->mt_flags & MDBX_RDONLY)) + if (unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) return MDBX_EACCESS; uint64_t new = dbs->md_seq + increment; @@ -18709,7 +22327,7 @@ static __inline int mdbx_attr_peek(MDBX_val *data, mdbx_attr_t *attrptr) { } static __inline int mdbx_attr_poke(MDBX_val *reserved, MDBX_val *data, - mdbx_attr_t attr, unsigned flags) { + mdbx_attr_t attr, MDBX_put_flags_t flags) { mdbx_attr_t *space = reserved->iov_base; if (flags & MDBX_RESERVE) { if (likely(data != NULL)) { @@ -18744,7 +22362,7 @@ int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, } int mdbx_put_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, - mdbx_attr_t attr, unsigned flags) { + mdbx_attr_t attr, MDBX_put_flags_t flags) { MDBX_val reserve; reserve.iov_base = NULL; reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t); @@ -18757,7 +22375,7 @@ int mdbx_put_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, } int mdbx_cursor_put_attr(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, - mdbx_attr_t attr, unsigned flags) { + mdbx_attr_t attr, MDBX_put_flags_t flags) { MDBX_val reserve; reserve.iov_base = NULL; reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t); @@ -18778,10 +22396,10 @@ int mdbx_set_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, return MDBX_EBADSIGN; if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) - return MDBX_EINVAL; + return MDBX_BAD_DBI; - if (unlikely(txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) - return (txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) + return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; MDBX_cursor_couple cx; MDBX_val old_data; @@ -18834,7 +22452,7 @@ __dll_export __has_attribute(__externally_visible__) __attribute__((__externally_visible__)) #endif - const mdbx_build_info mdbx_build = { + const struct MDBX_build_info mdbx_build = { #ifdef MDBX_BUILD_TIMESTAMP MDBX_BUILD_TIMESTAMP #else @@ -18944,9 +22562,6 @@ __dll_export #endif /* MDBX_BUILD_TYPE */ , "MDBX_DEBUG=" STRINGIFY(MDBX_DEBUG) -#ifdef MDBX_LOGLEVEL_BUILD - " MDBX_LOGLEVEL_BUILD=" STRINGIFY(MDBX_LOGLEVEL_BUILD) -#endif /* MDBX_LOGLEVEL_BUILD */ " MDBX_WORDBITS=" STRINGIFY(MDBX_WORDBITS) " BYTE_ORDER=" #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ @@ -18959,7 +22574,7 @@ __dll_export #if MDBX_HUGE_TRANSACTIONS " MDBX_HUGE_TRANSACTIONS=YES" #endif /* MDBX_HUGE_TRANSACTIONS */ - " MDBX_TXN_CHECKPID=" MDBX_TXN_CHECKPID_CONFIG + " MDBX_ENV_CHECKPID=" MDBX_ENV_CHECKPID_CONFIG " MDBX_TXN_CHECKOWNER=" MDBX_TXN_CHECKOWNER_CONFIG " MDBX_64BIT_ATOMIC=" MDBX_64BIT_ATOMIC_CONFIG " MDBX_64BIT_CAS=" MDBX_64BIT_CAS_CONFIG @@ -19037,7 +22652,11 @@ __dll_export #endif /* MDBX_BUILD_FLAGS */ #if !(defined(MDBX_BUILD_FLAGS_CONFIG) || defined(MDBX_BUILD_FLAGS)) "undefined (please use correct build script)" +#ifdef _MSC_VER +#pragma message("warning: Build flags undefined. Please use correct build script") +#else #warning "Build flags undefined. Please use correct build script" +#endif // _MSC_VER #endif }; @@ -19065,3 +22684,4007 @@ LIBMDBX_API __attribute__((__weak__)) const char *__asan_default_options() { /* *INDENT-ON* */ /* clang-format on */ +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +#if defined(_WIN32) || defined(_WIN64) + +#include <winioctl.h> + +static int waitstatus2errcode(DWORD result) { + switch (result) { + case WAIT_OBJECT_0: + return MDBX_SUCCESS; + case WAIT_FAILED: + return GetLastError(); + case WAIT_ABANDONED: + return ERROR_ABANDONED_WAIT_0; + case WAIT_IO_COMPLETION: + return ERROR_USER_APC; + case WAIT_TIMEOUT: + return ERROR_TIMEOUT; + default: + return ERROR_UNHANDLED_ERROR; + } +} + +/* Map a result from an NTAPI call to WIN32 error code. */ +static int ntstatus2errcode(NTSTATUS status) { + DWORD dummy; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Internal = status; + return GetOverlappedResult(NULL, &ov, &dummy, FALSE) ? MDBX_SUCCESS + : GetLastError(); +} + +/* We use native NT APIs to setup the memory map, so that we can + * let the DB file grow incrementally instead of always preallocating + * the full size. These APIs are defined in <wdm.h> and <ntifs.h> + * but those headers are meant for driver-level development and + * conflict with the regular user-level headers, so we explicitly + * declare them here. Using these APIs also means we must link to + * ntdll.dll, which is not linked by default in user code. */ + +extern NTSTATUS NTAPI NtCreateSection( + OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess, + IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, + IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection, + IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle); + +typedef struct _SECTION_BASIC_INFORMATION { + ULONG Unknown; + ULONG SectionAttributes; + LARGE_INTEGER SectionSize; +} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION; + +extern NTSTATUS NTAPI NtMapViewOfSection( + IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, + IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize, + IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize, + IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType, + IN ULONG Win32Protect); + +extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, + IN OPTIONAL PVOID BaseAddress); + +extern NTSTATUS NTAPI NtClose(HANDLE Handle); + +extern NTSTATUS NTAPI NtAllocateVirtualMemory( + IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits, + IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect); + +extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, + IN PVOID *BaseAddress, + IN OUT PSIZE_T RegionSize, + IN ULONG FreeType); + +#ifndef WOF_CURRENT_VERSION +typedef struct _WOF_EXTERNAL_INFO { + DWORD Version; + DWORD Provider; +} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO; +#endif /* WOF_CURRENT_VERSION */ + +#ifndef WIM_PROVIDER_CURRENT_VERSION +#define WIM_PROVIDER_HASH_SIZE 20 + +typedef struct _WIM_PROVIDER_EXTERNAL_INFO { + DWORD Version; + DWORD Flags; + LARGE_INTEGER DataSourceId; + BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE]; +} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO; +#endif /* WIM_PROVIDER_CURRENT_VERSION */ + +#ifndef FILE_PROVIDER_CURRENT_VERSION +typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 { + ULONG Version; + ULONG Algorithm; + ULONG Flags; +} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1; +#endif /* FILE_PROVIDER_CURRENT_VERSION */ + +#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED +#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL) +#endif +#ifndef STATUS_INVALID_DEVICE_REQUEST +#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L) +#endif + +#ifndef FILE_DEVICE_FILE_SYSTEM +#define FILE_DEVICE_FILE_SYSTEM 0x00000009 +#endif + +#ifndef FSCTL_GET_EXTERNAL_BACKING +#define FSCTL_GET_EXTERNAL_BACKING \ + CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) +#endif + +#endif /* _WIN32 || _WIN64 */ + +/*----------------------------------------------------------------------------*/ + +#if defined(__UCLIBC__) +__extern_C void __assert(const char *, const char *, unsigned int, const char *) +#ifdef __THROW + __THROW +#else + __nothrow +#endif /* __THROW */ + MDBX_NORETURN; +#define __assert_fail(assertion, file, line, function) \ + __assert(assertion, file, line, function) + +#elif _POSIX_C_SOURCE > 200212 && \ + /* workaround for avoid musl libc wrong prototype */ ( \ + defined(__GLIBC__) || defined(__GNU_LIBRARY__)) +/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */ +__extern_C void __assert_fail(const char *assertion, const char *file, + unsigned line, const char *function) +#ifdef __THROW + __THROW +#else + __nothrow +#endif /* __THROW */ + MDBX_NORETURN; + +#elif defined(__APPLE__) || defined(__MACH__) +__extern_C void __assert_rtn(const char *function, const char *file, int line, + const char *assertion) /* __nothrow */ +#ifdef __dead2 + __dead2 +#else + MDBX_NORETURN +#endif /* __dead2 */ +#ifdef __disable_tail_calls + __disable_tail_calls +#endif /* __disable_tail_calls */ + ; + +#define __assert_fail(assertion, file, line, function) \ + __assert_rtn(function, file, line, assertion) +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +__extern_C void __assert_c99(const char *assection, const char *file, int line, + const char *function) MDBX_NORETURN; +#define __assert_fail(assertion, file, line, function) \ + __assert_c99(assertion, file, line, function) +#elif defined(__OpenBSD__) +__extern_C __dead void __assert2(const char *file, int line, + const char *function, + const char *assertion) /* __nothrow */; +#define __assert_fail(assertion, file, line, function) \ + __assert2(file, line, function, assertion) +#elif defined(__NetBSD__) +__extern_C __dead void __assert13(const char *file, int line, + const char *function, + const char *assertion) /* __nothrow */; +#define __assert_fail(assertion, file, line, function) \ + __assert13(file, line, function, assertion) +#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) +__extern_C void __assert(const char *function, const char *file, int line, + const char *assertion) /* __nothrow */ +#ifdef __dead2 + __dead2 +#else + MDBX_NORETURN +#endif /* __dead2 */ +#ifdef __disable_tail_calls + __disable_tail_calls +#endif /* __disable_tail_calls */ + ; +#define __assert_fail(assertion, file, line, function) \ + __assert(function, file, line, assertion) + +#endif /* __assert_fail */ + +#if !defined(__ANDROID_API__) || MDBX_DEBUG + +void __cold mdbx_assert_fail(const MDBX_env *env, const char *msg, + const char *func, int line) { +#if MDBX_DEBUG + if (env && env->me_assert_func) { + env->me_assert_func(env, msg, func, line); + return; + } +#else + (void)env; +#endif /* MDBX_DEBUG */ + + if (mdbx_debug_logger) + mdbx_debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg); + else { +#if defined(_WIN32) || defined(_WIN64) + char *message = nullptr; + const int num = mdbx_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", + msg, func ? func : "unknown", line); + if (num < 1 || !message) + message = "<troubles with assertion-message preparation>"; + OutputDebugStringA(message); + if (IsDebuggerPresent()) + DebugBreak(); +#elif defined(__ANDROID_API__) + __android_log_assert(msg, "mdbx", "%s:%u", func, line); +#else + __assert_fail(msg, "mdbx", line, func); +#endif + } + +#if defined(_WIN32) || defined(_WIN64) + FatalExit(ERROR_UNHANDLED_ERROR); +#else + abort(); +#endif +} + +#endif /* __ANDROID_API__ || MDBX_DEBUG */ + +__cold void mdbx_panic(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + + char *message = nullptr; + const int num = mdbx_vasprintf(&message, fmt, ap); + va_end(ap); + const char *const const_message = + (num < 1 || !message) ? "<troubles with panic-message preparation>" + : message; + +#if defined(_WIN32) || defined(_WIN64) + OutputDebugStringA("\r\nMDBX-PANIC: "); + OutputDebugStringA(const_message); + if (IsDebuggerPresent()) + DebugBreak(); + FatalExit(ERROR_UNHANDLED_ERROR); +#else +#if defined(__ANDROID_API__) + __android_log_assert("panic", "mdbx", "%s", const_message); +#else + __assert_fail(const_message, "mdbx", 0, "panic"); +#endif /* __ANDROID_API__ */ + abort(); +#endif +} + +/*----------------------------------------------------------------------------*/ + +#ifndef mdbx_vasprintf +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, + va_list ap) { + va_list ones; + va_copy(ones, ap); + int needed = vsnprintf(nullptr, 0, fmt, ap); + + if (unlikely(needed < 0 || needed >= INT_MAX)) { + *strp = nullptr; + va_end(ones); + return needed; + } + + *strp = mdbx_malloc(needed + 1); + if (unlikely(*strp == nullptr)) { + va_end(ones); +#if defined(_WIN32) || defined(_WIN64) + SetLastError(MDBX_ENOMEM); +#else + errno = MDBX_ENOMEM; +#endif + return -1; + } + + int actual = vsnprintf(*strp, needed + 1, fmt, ones); + va_end(ones); + + assert(actual == needed); + if (unlikely(actual < 0)) { + mdbx_free(*strp); + *strp = nullptr; + } + return actual; +} +#endif /* mdbx_vasprintf */ + +#ifndef mdbx_asprintf +MDBX_INTERNAL_FUNC int mdbx_asprintf(char **strp, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int rc = mdbx_vasprintf(strp, fmt, ap); + va_end(ap); + return rc; +} +#endif /* mdbx_asprintf */ + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result) { + assert(is_powerof2(alignment) && alignment >= sizeof(void *)); +#if defined(_WIN32) || defined(_WIN64) + (void)alignment; + *result = VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */; +#elif defined(_ISOC11_SOURCE) + *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment)); + return *result ? MDBX_SUCCESS : errno; +#elif _POSIX_VERSION >= 200112L && \ + (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17) + *result = nullptr; + return posix_memalign(result, alignment, bytes); +#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L + *result = memalign(alignment, bytes); + return *result ? MDBX_SUCCESS : errno; +#else +#error FIXME +#endif +} +#endif /* mdbx_memalign_alloc */ + +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr) { +#if defined(_WIN32) || defined(_WIN64) + VirtualFree(ptr, 0, MEM_RELEASE); +#else + mdbx_free(ptr); +#endif +} +#endif /* mdbx_memalign_free */ + +#ifndef mdbx_strdup +char *mdbx_strdup(const char *str) { + if (!str) + return NULL; + size_t bytes = strlen(str) + 1; + char *dup = mdbx_malloc(bytes); + if (dup) + memcpy(dup, str, bytes); + return dup; +} +#endif /* mdbx_strdup */ + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair) { + int rc; + memset(condpair, 0, sizeof(mdbx_condpair_t)); +#if defined(_WIN32) || defined(_WIN64) + if ((condpair->mutex = CreateMutexW(NULL, FALSE, NULL)) == NULL) { + rc = GetLastError(); + goto bailout_mutex; + } + if ((condpair->event[0] = CreateEventW(NULL, FALSE, FALSE, NULL)) == NULL) { + rc = GetLastError(); + goto bailout_event; + } + if ((condpair->event[1] = CreateEventW(NULL, FALSE, FALSE, NULL)) != NULL) + return MDBX_SUCCESS; + + rc = GetLastError(); + (void)CloseHandle(condpair->event[0]); +bailout_event: + (void)CloseHandle(condpair->mutex); +#else + rc = pthread_mutex_init(&condpair->mutex, NULL); + if (unlikely(rc != 0)) + goto bailout_mutex; + rc = pthread_cond_init(&condpair->cond[0], NULL); + if (unlikely(rc != 0)) + goto bailout_cond; + rc = pthread_cond_init(&condpair->cond[1], NULL); + if (likely(rc == 0)) + return MDBX_SUCCESS; + + (void)pthread_cond_destroy(&condpair->cond[0]); +bailout_cond: + (void)pthread_mutex_destroy(&condpair->mutex); +#endif +bailout_mutex: + memset(condpair, 0, sizeof(mdbx_condpair_t)); + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair) { +#if defined(_WIN32) || defined(_WIN64) + int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : GetLastError(); + rc = CloseHandle(condpair->event[0]) ? rc : GetLastError(); + rc = CloseHandle(condpair->event[1]) ? rc : GetLastError(); +#else + int err, rc = pthread_mutex_destroy(&condpair->mutex); + rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc; + rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc; +#endif + memset(condpair, 0, sizeof(mdbx_condpair_t)); + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = WaitForSingleObject(condpair->mutex, INFINITE); + return waitstatus2errcode(code); +#else + return pthread_mutex_lock(&condpair->mutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair) { +#if defined(_WIN32) || defined(_WIN64) + return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : GetLastError(); +#else + return pthread_mutex_unlock(&condpair->mutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part) { +#if defined(_WIN32) || defined(_WIN64) + return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : GetLastError(); +#else + return pthread_cond_signal(&condpair->cond[part]); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, + bool part) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], + INFINITE, FALSE); + if (code == WAIT_OBJECT_0) { + code = WaitForSingleObject(condpair->mutex, INFINITE); + if (code == WAIT_OBJECT_0) + return MDBX_SUCCESS; + } + return waitstatus2errcode(code); +#else + return pthread_cond_wait(&condpair->cond[part], &condpair->mutex); +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + InitializeCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_init(fastmutex, NULL); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + DeleteCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_destroy(fastmutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + EnterCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_lock(fastmutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + LeaveCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_unlock(fastmutex); +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname) { +#if defined(_WIN32) || defined(_WIN64) + const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX); + if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX) + return ERROR_INVALID_NAME; + wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t)); + if (wlen != mbstowcs(pathnameW, pathname, wlen + 1)) + return ERROR_INVALID_NAME; + return DeleteFileW(pathnameW) ? MDBX_SUCCESS : GetLastError(); +#else + return unlink(pathname) ? errno : MDBX_SUCCESS; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits) { + *fd = INVALID_HANDLE_VALUE; + +#if defined(_WIN32) || defined(_WIN64) + const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX); + if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX) + return ERROR_INVALID_NAME; + wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t)); + if (wlen != mbstowcs(pathnameW, pathname, wlen + 1)) + return ERROR_INVALID_NAME; + + DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING; + DWORD FlagsAndAttributes = + FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; + DWORD DesiredAccess = FILE_READ_ATTRIBUTES; + DWORD ShareMode = (env->me_flags & MDBX_EXCLUSIVE) + ? 0 + : (FILE_SHARE_READ | FILE_SHARE_WRITE); + + switch (purpose) { + default: + return ERROR_INVALID_PARAMETER; + case MDBX_OPEN_LCK: + CreationDisposition = OPEN_ALWAYS; + DesiredAccess |= GENERIC_READ | GENERIC_WRITE; + FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY; + break; + case MDBX_OPEN_DXB_READ: + CreationDisposition = OPEN_EXISTING; + DesiredAccess |= GENERIC_READ; + ShareMode |= FILE_SHARE_READ; + break; + case MDBX_OPEN_DXB_LAZY: + DesiredAccess |= GENERIC_READ | GENERIC_WRITE; + break; + case MDBX_OPEN_DXB_DSYNC: + CreationDisposition = OPEN_EXISTING; + DesiredAccess |= GENERIC_WRITE; + FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH; + break; + case MDBX_OPEN_COPY: + CreationDisposition = CREATE_NEW; + ShareMode = 0; + DesiredAccess |= GENERIC_WRITE; + FlagsAndAttributes |= + (env->me_psize < env->me_os_psize) ? 0 : FILE_FLAG_NO_BUFFERING; + break; + } + + *fd = CreateFileW(pathnameW, DesiredAccess, ShareMode, NULL, + CreationDisposition, FlagsAndAttributes, NULL); + if (*fd == INVALID_HANDLE_VALUE) + return GetLastError(); + + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(*fd, &info)) { + int err = GetLastError(); + CloseHandle(*fd); + *fd = INVALID_HANDLE_VALUE; + return err; + } + const DWORD AttributesDiff = + (info.dwFileAttributes ^ FlagsAndAttributes) & + (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | + FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED); + if (AttributesDiff) + (void)SetFileAttributesW(pathnameW, info.dwFileAttributes ^ AttributesDiff); + +#else + int flags = unix_mode_bits ? O_CREAT : 0; + switch (purpose) { + default: + return EINVAL; + case MDBX_OPEN_LCK: + flags |= O_RDWR; + break; + case MDBX_OPEN_DXB_READ: + flags = O_RDONLY; + break; + case MDBX_OPEN_DXB_LAZY: + flags |= O_RDWR; + break; + case MDBX_OPEN_COPY: + flags = O_CREAT | O_WRONLY | O_EXCL; + break; + case MDBX_OPEN_DXB_DSYNC: + flags |= O_WRONLY; +#if defined(O_DSYNC) + flags |= O_DSYNC; +#elif defined(O_SYNC) + flags |= O_SYNC; +#elif defined(O_FSYNC) + flags |= O_FSYNC; +#endif + break; + } + + const bool direct_nocache_for_copy = + env->me_psize >= env->me_os_psize && purpose == MDBX_OPEN_COPY; + if (direct_nocache_for_copy) { +#if defined(O_DIRECT) + flags |= O_DIRECT; +#endif /* O_DIRECT */ +#if defined(O_NOCACHE) + flags |= O_NOCACHE; +#endif /* O_NOCACHE */ + } + +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif /* O_CLOEXEC */ + + *fd = open(pathname, flags, unix_mode_bits); +#if defined(O_DIRECT) + if (*fd < 0 && (flags & O_DIRECT) && + (errno == EINVAL || errno == EAFNOSUPPORT)) { + flags &= ~(O_DIRECT | O_EXCL); + *fd = open(pathname, flags, unix_mode_bits); + } +#endif /* O_DIRECT */ + if (*fd < 0) + return errno; + +#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC) + const int fd_flags = fcntl(*fd, F_GETFD); + if (fd_flags != -1) + (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC); +#endif /* FD_CLOEXEC && !O_CLOEXEC */ + + if (direct_nocache_for_copy) { +#if defined(F_NOCACHE) && !defined(O_NOCACHE) + (void)fcntl(*fd, F_NOCACHE, 1); +#endif /* F_NOCACHE */ + } + +#endif + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd) { +#if defined(_WIN32) || defined(_WIN64) + return CloseHandle(fd) ? MDBX_SUCCESS : GetLastError(); +#else + return (close(fd) == 0) ? MDBX_SUCCESS : errno; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, + uint64_t offset) { + if (bytes > MAX_WRITE) + return MDBX_EINVAL; +#if defined(_WIN32) || defined(_WIN64) + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + + DWORD read = 0; + if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) { + int rc = GetLastError(); + return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc; + } +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + intptr_t read = pread(fd, buf, bytes, offset); + if (read < 0) { + int rc = errno; + return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc; + } +#endif + return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA; +} + +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t bytes, uint64_t offset) { + while (true) { +#if defined(_WIN32) || defined(_WIN64) + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + + DWORD written; + if (unlikely(!WriteFile( + fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, + &written, &ov))) + return GetLastError(); + if (likely(bytes == written)) + return MDBX_SUCCESS; +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = + pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset); + if (likely(bytes == (size_t)written)) + return MDBX_SUCCESS; + if (written < 0) { + const int rc = errno; + if (rc != EINTR) + return rc; + continue; + } +#endif + bytes -= written; + offset += written; + buf = (char *)buf + written; + } +} + +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t bytes) { + while (true) { +#if defined(_WIN32) || defined(_WIN64) + DWORD written; + if (unlikely(!WriteFile( + fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, + &written, nullptr))) + return GetLastError(); + if (likely(bytes == written)) + return MDBX_SUCCESS; +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = + write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE); + if (likely(bytes == (size_t)written)) + return MDBX_SUCCESS; + if (written < 0) { + const int rc = errno; + if (rc != EINTR) + return rc; + continue; + } +#endif + bytes -= written; + buf = (char *)buf + written; + } +} + +int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, int iovcnt, + uint64_t offset, size_t expected_written) { +#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ < 24) + size_t written = 0; + for (int i = 0; i < iovcnt; ++i) { + int rc = mdbx_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + written += iov[i].iov_len; + offset += iov[i].iov_len; + } + return (expected_written == written) ? MDBX_SUCCESS + : MDBX_EIO /* ERROR_WRITE_FAULT */; +#else + int rc; + intptr_t written; + do { + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + written = pwritev(fd, iov, iovcnt, offset); + if (likely(expected_written == (size_t)written)) + return MDBX_SUCCESS; + rc = errno; + } while (rc == EINTR); + return (written < 0) ? rc : MDBX_EIO /* Use which error code? */; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + enum mdbx_syncmode_bits mode_bits) { +#if defined(_WIN32) || defined(_WIN64) + if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd)) + return GetLastError(); + return MDBX_SUCCESS; +#else + +#if defined(__APPLE__) && \ + MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY + if (mode_bits & MDBX_SYNC_IODQ) + return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno; +#endif /* MacOS */ + + /* LY: This approach is always safe and without appreciable performance + * degradation, even on a kernel with fdatasync's bug. + * + * For more info about of a corresponding fdatasync() bug + * see http://www.spinics.net/lists/linux-ext4/msg33714.html */ + while (1) { + switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) { + case MDBX_SYNC_NONE: + return MDBX_SUCCESS /* nothing to do */; +#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0 + case MDBX_SYNC_DATA: + if (fdatasync(fd) == 0) + return MDBX_SUCCESS; + break /* error */; +#if defined(__linux__) || defined(__gnu_linux__) + case MDBX_SYNC_SIZE: + if (mdbx_linux_kernel_version >= 0x03060000) + return MDBX_SUCCESS; + __fallthrough /* fall through */; +#endif /* Linux */ +#endif /* _POSIX_SYNCHRONIZED_IO > 0 */ + default: + if (fsync(fd) == 0) + return MDBX_SUCCESS; + } + + int rc = errno; + if (rc != EINTR) + return rc; + } +#endif +} + +int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length) { +#if defined(_WIN32) || defined(_WIN64) + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(fd, &info)) + return GetLastError(); + *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; +#else + struct stat st; + + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + if (fstat(fd, &st)) + return errno; + + *length = st.st_size; +#endif + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd) { +#if defined(_WIN32) || defined(_WIN64) + switch (GetFileType(fd)) { + case FILE_TYPE_DISK: + return MDBX_RESULT_FALSE; + case FILE_TYPE_CHAR: + case FILE_TYPE_PIPE: + return MDBX_RESULT_TRUE; + default: + return GetLastError(); + } +#else + struct stat info; + if (fstat(fd, &info)) + return errno; + switch (info.st_mode & S_IFMT) { + case S_IFBLK: + case S_IFREG: + return MDBX_RESULT_FALSE; + case S_IFCHR: + case S_IFIFO: + case S_IFSOCK: + return MDBX_RESULT_TRUE; + case S_IFDIR: + case S_IFLNK: + default: + return MDBX_INCOMPATIBLE; + } +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length) { +#if defined(_WIN32) || defined(_WIN64) + if (mdbx_SetFileInformationByHandle) { + FILE_END_OF_FILE_INFO EndOfFileInfo; + EndOfFileInfo.EndOfFile.QuadPart = length; + return mdbx_SetFileInformationByHandle(fd, FileEndOfFileInfo, + &EndOfFileInfo, + sizeof(FILE_END_OF_FILE_INFO)) + ? MDBX_SUCCESS + : GetLastError(); + } else { + LARGE_INTEGER li; + li.QuadPart = length; + return (SetFilePointerEx(fd, li, NULL, FILE_BEGIN) && SetEndOfFile(fd)) + ? MDBX_SUCCESS + : GetLastError(); + } +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos) { +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER li; + li.QuadPart = pos; + return SetFilePointerEx(fd, li, NULL, FILE_BEGIN) ? MDBX_SUCCESS + : GetLastError(); +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS; +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg) { +#if defined(_WIN32) || defined(_WIN64) + *thread = CreateThread(NULL, 0, start_routine, arg, 0, NULL); + return *thread ? MDBX_SUCCESS : GetLastError(); +#else + return pthread_create(thread, NULL, start_routine, arg); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = WaitForSingleObject(thread, INFINITE); + return waitstatus2errcode(code); +#else + void *unused_retval = &unused_retval; + return pthread_join(thread, &unused_retval); +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits) { + uint8_t *ptr = (uint8_t *)map->address + offset; +#if defined(_WIN32) || defined(_WIN64) + if (!FlushViewOfFile(ptr, length)) + return GetLastError(); +#else +#if defined(__linux__) || defined(__gnu_linux__) + if (mode_bits == MDBX_SYNC_NONE && mdbx_linux_kernel_version > 0x02061300) + /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly + * tracks dirty pages and flushes them to storage as necessary. */ + return MDBX_SUCCESS; +#endif /* Linux */ + if (msync(ptr, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC)) + return errno; + mode_bits &= ~MDBX_SYNC_DATA; +#endif + return mdbx_fsync(map->fd, mode_bits); +} + +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err) { +#if defined(_WIN32) || defined(_WIN64) + (void)pathname; + (void)err; + if (!mdbx_GetVolumeInformationByHandleW) + return MDBX_ENOSYS; + DWORD unused, flags; + if (!mdbx_GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, + &flags, nullptr, 0)) + return GetLastError(); + if ((flags & FILE_READ_ONLY_VOLUME) == 0) + return MDBX_EACCESS; +#else + struct statvfs info; + if (err != MDBX_ENOFILE) { + if (statvfs(pathname, &info)) + return errno; + if ((info.f_flag & ST_RDONLY) == 0) + return err; + } + if (fstatvfs(handle, &info)) + return errno; + if ((info.f_flag & ST_RDONLY) == 0) + return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err; +#endif /* !Windows */ + return MDBX_SUCCESS; +} + +static int mdbx_check_fs_local(mdbx_filehandle_t handle, int flags) { +#if defined(_WIN32) || defined(_WIN64) + if (mdbx_RunningUnderWine() && !(flags & MDBX_EXCLUSIVE)) + return ERROR_NOT_CAPABLE /* workaround for Wine */; + + if (GetFileType(handle) != FILE_TYPE_DISK) + return ERROR_FILE_OFFLINE; + + if (mdbx_GetFileInformationByHandleEx) { + FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo; + if (mdbx_GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, + &RemoteProtocolInfo, + sizeof(RemoteProtocolInfo))) { + if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && + !(flags & MDBX_RDONLY)) + return ERROR_FILE_OFFLINE; + if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && + !(flags & MDBX_EXCLUSIVE)) + return ERROR_REMOTE_STORAGE_MEDIA_ERROR; + } + } + + if (mdbx_NtFsControlFile) { + NTSTATUS rc; + struct { + WOF_EXTERNAL_INFO wof_info; + union { + WIM_PROVIDER_EXTERNAL_INFO wim_info; + FILE_PROVIDER_EXTERNAL_INFO_V1 file_info; + }; + size_t reserved_for_microsoft_madness[42]; + } GetExternalBacking_OutputBuffer; + IO_STATUS_BLOCK StatusBlock; + rc = mdbx_NtFsControlFile(handle, NULL, NULL, NULL, &StatusBlock, + FSCTL_GET_EXTERNAL_BACKING, NULL, 0, + &GetExternalBacking_OutputBuffer, + sizeof(GetExternalBacking_OutputBuffer)); + if (NT_SUCCESS(rc)) { + if (!(flags & MDBX_EXCLUSIVE)) + return ERROR_REMOTE_STORAGE_MEDIA_ERROR; + } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && + rc != STATUS_INVALID_DEVICE_REQUEST) + return ntstatus2errcode(rc); + } + + if (mdbx_GetVolumeInformationByHandleW && mdbx_GetFinalPathNameByHandleW) { + WCHAR *PathBuffer = mdbx_malloc(sizeof(WCHAR) * INT16_MAX); + if (!PathBuffer) + return MDBX_ENOMEM; + + int rc = MDBX_SUCCESS; + DWORD VolumeSerialNumber, FileSystemFlags; + if (!mdbx_GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, + &VolumeSerialNumber, NULL, + &FileSystemFlags, NULL, 0)) { + rc = GetLastError(); + goto bailout; + } + + if ((flags & MDBX_RDONLY) == 0) { + if (FileSystemFlags & + (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | + FILE_VOLUME_IS_COMPRESSED)) { + rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; + goto bailout; + } + } + + if (!mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, + FILE_NAME_NORMALIZED | + VOLUME_NAME_NT)) { + rc = GetLastError(); + goto bailout; + } + + if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) { + if (!(flags & MDBX_EXCLUSIVE)) { + rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; + goto bailout; + } + } else if (mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, + FILE_NAME_NORMALIZED | + VOLUME_NAME_DOS)) { + UINT DriveType = GetDriveTypeW(PathBuffer); + if (DriveType == DRIVE_NO_ROOT_DIR && + _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 && + _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) { + PathBuffer[7] = 0; + DriveType = GetDriveTypeW(PathBuffer + 4); + } + switch (DriveType) { + case DRIVE_CDROM: + if (flags & MDBX_RDONLY) + break; + // fall through + case DRIVE_UNKNOWN: + case DRIVE_NO_ROOT_DIR: + case DRIVE_REMOTE: + default: + if (!(flags & MDBX_EXCLUSIVE)) + rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; + // fall through + case DRIVE_REMOVABLE: + case DRIVE_FIXED: + case DRIVE_RAMDISK: + break; + } + } + bailout: + mdbx_free(PathBuffer); + return rc; + } + +#else + + struct statvfs statvfs_info; + if (fstatvfs(handle, &statvfs_info)) + return errno; +#if defined(ST_LOCAL) || defined(ST_EXPORTED) + const unsigned long st_flags = statvfs_info.f_flag; +#endif /* ST_LOCAL || ST_EXPORTED */ + +#if defined(__NetBSD__) + const unsigned type = 0; + const char *const name = statvfs_info.f_fstypename; + const size_t name_len = VFS_NAMELEN; +#elif defined(_AIX) || defined(__OS400__) + const char *const name = statvfs_info.f_basetype; + const size_t name_len = sizeof(statvfs_info.f_basetype); + struct stat st; + if (fstat(handle, &st)) + return errno; + const unsigned type = st.st_vfstype; + if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; +#elif defined(FSTYPSZ) || defined(_FSTYPSZ) + const unsigned type = 0; + const char *const name = statvfs_info.f_basetype; + const size_t name_len = sizeof(statvfs_info.f_basetype); +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ) + const unsigned type = 0; + struct stat st; + if (fstat(handle, &st)) + return errno; + const char *const name = st.st_fstype; + const size_t name_len = strlen(name); +#else + struct statfs statfs_info; + if (fstatfs(handle, &statfs_info)) + return errno; +#if defined(__OpenBSD__) + const unsigned type = 0; +#else + const unsigned type = statfs_info.f_type; +#endif +#if defined(MNT_LOCAL) || defined(MNT_EXPORTED) + const unsigned long mnt_flags = statfs_info.f_flags; +#endif /* MNT_LOCAL || MNT_EXPORTED */ +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ + defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) + const char *const name = statfs_info.f_fstypename; + const size_t name_len = sizeof(statfs_info.f_fstypename); +#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21 + const char *const name = ""; + const unsigned name_len = 0; +#else + + const char *name = ""; + unsigned name_len = 0; + + struct stat st; + if (fstat(handle, &st)) + return errno; + + char pathbuf[PATH_MAX]; + FILE *mounted = nullptr; +#if defined(__linux__) || defined(__gnu_linux__) + mounted = setmntent("/proc/mounts", "r"); +#endif /* Linux */ + if (!mounted) + mounted = setmntent("/etc/mtab", "r"); + if (mounted) { + const struct mntent *ent; +#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \ + (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19)) + struct mntent entbuf; + const bool should_copy = false; + while (nullptr != + (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf)))) +#else + const bool should_copy = true; + while (nullptr != (ent = getmntent(mounted))) +#endif + { + struct stat mnt; + if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) { + if (should_copy) { + name = + strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1); + pathbuf[name_len] = 0; + } else { + name = ent->mnt_fsname; + name_len = strlen(name); + } + break; + } + } + endmntent(mounted); + } +#endif /* !xBSD && !Android/Bionic */ +#endif + + if (name_len) { + if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || + strncasecmp("cifs", name, name_len) == 0 || + strncasecmp("ncpfs", name, name_len) == 0 || + strncasecmp("smbfs", name, name_len) == 0 || + ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && + strncasecmp("fuseblk", name, name_len) != 0)) && + !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; + if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || + strcasecmp("sshfs", name) == 0) + return MDBX_EREMOTE; + } + +#ifdef ST_LOCAL + if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; +#elif defined(MNT_LOCAL) + if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; +#endif /* ST/MNT_LOCAL */ + +#ifdef ST_EXPORTED + if ((st_flags & ST_EXPORTED) != 0 && !(flags & MDBX_RDONLY)) + return MDBX_EREMOTE; +#elif defined(MNT_EXPORTED) + if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & MDBX_RDONLY)) + return MDBX_EREMOTE; +#endif /* ST/MNT_EXPORTED */ + + switch (type) { + case 0xFF534D42 /* CIFS_MAGIC_NUMBER */: + case 0x6969 /* NFS_SUPER_MAGIC */: + case 0x564c /* NCP_SUPER_MAGIC */: + case 0x517B /* SMB_SUPER_MAGIC */: +#if defined(__digital__) || defined(__osf__) || defined(__osf) + case 0x0E /* Tru64 NFS */: +#endif +#ifdef ST_FST_NFS + case ST_FST_NFS: +#endif + if ((flags & MDBX_EXCLUSIVE) == 0) + return MDBX_EREMOTE; + case 0: + default: + break; + } +#endif /* Unix */ + + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t size, const size_t limit, + const unsigned options) { + assert(size <= limit); + map->limit = 0; + map->current = 0; + map->address = nullptr; +#if defined(_WIN32) || defined(_WIN64) + map->section = NULL; + map->filesize = 0; +#endif /* Windows */ + + int err = mdbx_check_fs_local(map->fd, flags); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) { + err = mdbx_ftruncate(map->fd, size); + if (err != MDBX_SUCCESS) + return err; +#if defined(_WIN32) || defined(_WIN64) + map->filesize = size; +#else + map->current = size; +#endif + } else { + uint64_t filesize = 0; + err = mdbx_filesize(map->fd, &filesize); + if (err != MDBX_SUCCESS) + return err; +#if defined(_WIN32) || defined(_WIN64) + map->filesize = filesize; +#else + map->current = (filesize > limit) ? limit : (size_t)filesize; +#endif + } + +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER SectionSize; + SectionSize.QuadPart = size; + err = NtCreateSection( + &map->section, + /* DesiredAccess */ + (flags & MDBX_WRITEMAP) + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | + SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, + /* ObjectAttributes */ NULL, /* MaximumSize (InitialSize) */ &SectionSize, + /* SectionPageProtection */ + (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, + /* AllocationAttributes */ SEC_RESERVE, map->fd); + if (!NT_SUCCESS(err)) + return ntstatus2errcode(err); + + SIZE_T ViewSize = + (flags & MDBX_RDONLY) ? 0 : mdbx_RunningUnderWine() ? size : limit; + err = NtMapViewOfSection( + map->section, GetCurrentProcess(), &map->address, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ NULL, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); + if (!NT_SUCCESS(err)) { + NtClose(map->section); + map->section = 0; + map->address = nullptr; + return ntstatus2errcode(err); + } + assert(map->address != MAP_FAILED); + + map->current = (size_t)SectionSize.QuadPart; + map->limit = ViewSize; + +#else + +#ifndef MAP_TRYFIXED +#define MAP_TRYFIXED 0 +#endif + +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 +#endif + +#ifndef MAP_CONCEAL +#define MAP_CONCEAL 0 +#endif + +#ifndef MAP_NOSYNC +#define MAP_NOSYNC 0 +#endif + + map->address = mmap( + NULL, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, + MAP_SHARED | MAP_FILE | + (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) | + ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC + : MAP_CONCEAL), + map->fd, 0); + + if (unlikely(map->address == MAP_FAILED)) { + map->limit = 0; + map->current = 0; + map->address = nullptr; + return errno; + } + map->limit = limit; + +#ifdef MADV_DONTFORK + if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0)) + return errno; +#endif +#ifdef MADV_NOHUGEPAGE + (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE); +#endif + +#endif + + VALGRIND_MAKE_MEM_DEFINED(map->address, map->current); + ASAN_UNPOISON_MEMORY_REGION(map->address, map->current); + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map) { + VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current); + /* Unpoisoning is required for ASAN to avoid false-positive diagnostic + * when this memory will re-used by malloc or another mmapping. + * See https://github.com/erthink/libmdbx/pull/93#issuecomment-613687203 */ + ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit); +#if defined(_WIN32) || defined(_WIN64) + if (map->section) + NtClose(map->section); + NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->address); + if (!NT_SUCCESS(rc)) + ntstatus2errcode(rc); +#else + if (unlikely(munmap(map->address, map->limit))) + return errno; +#endif + + map->limit = 0; + map->current = 0; + map->address = nullptr; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t size, + size_t limit, const bool may_move) { + assert(size <= limit); +#if defined(_WIN32) || defined(_WIN64) + assert(size != map->current || limit != map->limit || size < map->filesize); + + NTSTATUS status; + LARGE_INTEGER SectionSize; + int err, rc = MDBX_SUCCESS; + + if (!(flags & MDBX_RDONLY) && limit == map->limit && size > map->current) { + /* growth rw-section */ + if (!mdbx_NtExtendSection) + return MDBX_UNABLE_EXTEND_MAPSIZE /* workaround for Wine */; + SectionSize.QuadPart = size; + status = mdbx_NtExtendSection(map->section, &SectionSize); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + map->current = size; + if (map->filesize < size) + map->filesize = size; + return MDBX_SUCCESS; + } + + if (limit > map->limit) { + /* check ability of address space for growth before unmap */ + PVOID BaseAddress = (PBYTE)map->address + map->limit; + SIZE_T RegionSize = limit - map->limit; + status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, + &RegionSize, MEM_RESERVE, PAGE_NOACCESS); + if (status == /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) + return MDBX_UNABLE_EXTEND_MAPSIZE; + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + + status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, + MEM_RELEASE); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + } + + /* Windows unable: + * - shrink a mapped file; + * - change size of mapped view; + * - extend read-only mapping; + * Therefore we should unmap/map entire section. */ + status = NtUnmapViewOfSection(GetCurrentProcess(), map->address); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + status = NtClose(map->section); + map->section = NULL; + PVOID ReservedAddress = NULL; + SIZE_T ReservedSize = limit; + + if (!NT_SUCCESS(status)) { + bailout_ntstatus: + err = ntstatus2errcode(status); + bailout: + map->address = NULL; + map->current = map->limit = 0; + if (ReservedAddress) { + ReservedSize = 0; + status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, + &ReservedSize, MEM_RELEASE); + assert(NT_SUCCESS(status)); + (void)status; + } + return err; + } + + /* resizing of the file may take a while, + * therefore we reserve address space to avoid occupy it by other threads */ + ReservedAddress = map->address; + status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, + &ReservedSize, MEM_RESERVE, PAGE_NOACCESS); + if (!NT_SUCCESS(status)) { + ReservedAddress = NULL; + if (status != /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) + goto bailout_ntstatus /* no way to recovery */; + + if (may_move) + /* the base address could be changed */ + map->address = NULL; + } + +retry_file_and_section: + err = mdbx_filesize(map->fd, &map->filesize); + if (err != MDBX_SUCCESS) + goto bailout; + + if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) { + err = mdbx_ftruncate(map->fd, size); + if (err == MDBX_SUCCESS) + map->filesize = size; + /* ignore error, because Windows unable shrink file + * that already mapped (by another process) */ + } + + SectionSize.QuadPart = size; + status = NtCreateSection( + &map->section, + /* DesiredAccess */ + (flags & MDBX_WRITEMAP) + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | + SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, + /* ObjectAttributes */ NULL, + /* MaximumSize (InitialSize) */ &SectionSize, + /* SectionPageProtection */ + (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, + /* AllocationAttributes */ SEC_RESERVE, map->fd); + + if (!NT_SUCCESS(status)) + goto bailout_ntstatus; + + if (ReservedAddress) { + /* release reserved address space */ + ReservedSize = 0; + status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, + &ReservedSize, MEM_RELEASE); + ReservedAddress = NULL; + if (!NT_SUCCESS(status)) + goto bailout_ntstatus; + } + +retry_mapview:; + SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit; + status = NtMapViewOfSection( + map->section, GetCurrentProcess(), &map->address, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ NULL, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); + + if (!NT_SUCCESS(status)) { + if (status == /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && + map->address && may_move) { + /* try remap at another base address */ + map->address = NULL; + goto retry_mapview; + } + NtClose(map->section); + map->section = NULL; + + if (map->address && (size != map->current || limit != map->limit)) { + /* try remap with previously size and limit, + * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */ + rc = MDBX_UNABLE_EXTEND_MAPSIZE; + size = map->current; + limit = map->limit; + goto retry_file_and_section; + } + + /* no way to recovery */ + goto bailout_ntstatus; + } + assert(map->address != MAP_FAILED); + + map->current = (size_t)SectionSize.QuadPart; + map->limit = ViewSize; + +#else + + uint64_t filesize = 0; + int rc = mdbx_filesize(map->fd, &filesize); + if (rc != MDBX_SUCCESS) + return rc; + + if (flags & MDBX_RDONLY) { + map->current = (filesize > limit) ? limit : (size_t)filesize; + if (map->current != size) + rc = MDBX_UNABLE_EXTEND_MAPSIZE; + } else if (filesize != size) { + rc = mdbx_ftruncate(map->fd, size); + if (rc != MDBX_SUCCESS) + return rc; + map->current = size; + } + + if (limit == map->limit) + return MDBX_SUCCESS; + + if (limit < map->limit) { + /* unmap an excess at end of mapping. */ + if (unlikely(munmap(map->dxb + limit, map->limit - limit))) + return errno; + map->limit = limit; + return MDBX_SUCCESS; + } + + assert(limit > map->limit); + uint8_t *ptr = MAP_FAILED; + +#if defined(MREMAP_MAYMOVE) + ptr = mremap(map->address, map->limit, limit, may_move ? MREMAP_MAYMOVE : 0); + if (ptr == MAP_FAILED) { + const int err = errno; + switch (err) { + default: + return err; + case EAGAIN: + case ENOMEM: + return MDBX_UNABLE_EXTEND_MAPSIZE; + case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */: + break; + } + } +#endif /* MREMAP_MAYMOVE */ + + const unsigned mmap_flags = + MAP_CONCEAL | MAP_SHARED | MAP_FILE | + (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0); + const unsigned mmap_prot = + (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ; + + if (ptr == MAP_FAILED) { + /* Try to mmap additional space beyond the end of mapping. */ + ptr = mmap(map->dxb + map->limit, limit - map->limit, mmap_prot, + mmap_flags +#if defined(MAP_FIXED_NOREPLACE) + | MAP_FIXED_NOREPLACE +#endif /* MAP_FIXED_NOREPLACE */ + , + map->fd, map->limit); + if (ptr == map->dxb + map->limit) + ptr = map->dxb; + else if (ptr != MAP_FAILED) { + /* the desired address is busy, unmap unsuitable one */ + if (unlikely(munmap(ptr, limit - map->limit))) + return errno; + ptr = MAP_FAILED; + } else { + const int err = errno; + switch (err) { + default: + return err; + case EAGAIN: + case ENOMEM: + return MDBX_UNABLE_EXTEND_MAPSIZE; + case EEXIST: /* address busy */ + case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */ + break; + } + } + } + + if (ptr == MAP_FAILED) { + /* unmap and map again whole region */ + if (!may_move) { + /* TODO: Perhaps here it is worth to implement suspend/resume threads + * and perform unmap/map as like for Windows. */ + return MDBX_UNABLE_EXTEND_MAPSIZE; + } + + if (unlikely(munmap(map->address, map->limit))) + return errno; + + ptr = mmap(map->address, limit, mmap_prot, mmap_flags, map->fd, 0); + if (unlikely(ptr == MAP_FAILED)) { + ptr = mmap(map->address, map->limit, mmap_prot, mmap_flags, map->fd, 0); + if (unlikely(ptr == MAP_FAILED)) { + VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current); + /* Unpoisoning is required for ASAN to avoid false-positive diagnostic + * when this memory will re-used by malloc or another mmapping. + * See https://github.com/erthink/libmdbx/pull/93#issuecomment-613687203 + */ + ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit); + map->limit = 0; + map->current = 0; + map->address = nullptr; + return errno; + } + rc = MDBX_UNABLE_EXTEND_MAPSIZE; + limit = map->limit; + } + } + + assert(ptr && ptr != MAP_FAILED); + if (map->address != ptr) { + VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current); + /* Unpoisoning is required for ASAN to avoid false-positive diagnostic + * when this memory will re-used by malloc or another mmapping. + * See https://github.com/erthink/libmdbx/pull/93#issuecomment-613687203 + */ + ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit); + + VALGRIND_MAKE_MEM_DEFINED(ptr, map->current); + ASAN_UNPOISON_MEMORY_REGION(ptr, map->current); + map->address = ptr; + } + map->limit = limit; + +#ifdef MADV_DONTFORK + if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0)) + return errno; +#endif /* MADV_DONTFORK */ + +#ifdef MADV_NOHUGEPAGE + (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE); +#endif /* MADV_NOHUGEPAGE */ + +#endif /* POSIX / Windows */ + + return rc; +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC __cold void mdbx_osal_jitter(bool tiny) { + for (;;) { +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \ + defined(__x86_64__) + const unsigned salt = 277u * (unsigned)__rdtsc(); +#else + const unsigned salt = rand(); +#endif + + const unsigned coin = salt % (tiny ? 29u : 43u); + if (coin < 43 / 3) + break; +#if defined(_WIN32) || defined(_WIN64) + SwitchToThread(); + if (coin > 43 * 2 / 3) + Sleep(1); +#else + sched_yield(); + if (coin > 43 * 2 / 3) + usleep(coin); +#endif + } +} + +#if defined(_WIN32) || defined(_WIN64) +#elif defined(__APPLE__) || defined(__MACH__) +#include <mach/mach_time.h> +#elif defined(__linux__) || defined(__gnu_linux__) +static __cold clockid_t choice_monoclock(void) { + struct timespec probe; +#if defined(CLOCK_BOOTTIME) + if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0) + return CLOCK_BOOTTIME; +#elif defined(CLOCK_MONOTONIC_RAW) + if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0) + return CLOCK_MONOTONIC_RAW; +#elif defined(CLOCK_MONOTONIC_COARSE) + if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0) + return CLOCK_MONOTONIC_COARSE; +#endif + return CLOCK_MONOTONIC; +} +#endif + +/*----------------------------------------------------------------------------*/ + +#if defined(_WIN32) || defined(_WIN64) +static LARGE_INTEGER performance_frequency; +#elif defined(__APPLE__) || defined(__MACH__) +static uint64_t ratio_16dot16_to_monotine; +#endif + +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16) { +#if defined(_WIN32) || defined(_WIN64) + if (unlikely(performance_frequency.QuadPart == 0)) + QueryPerformanceFrequency(&performance_frequency); + const uint64_t ratio = performance_frequency.QuadPart; +#elif defined(__APPLE__) || defined(__MACH__) + if (unlikely(ratio_16dot16_to_monotine == 0)) { + mach_timebase_info_data_t ti; + mach_timebase_info(&ti); + ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer; + } + const uint64_t ratio = ratio_16dot16_to_monotine; +#else + const uint64_t ratio = UINT64_C(1000000000); +#endif + return (ratio * seconds_16dot16 + 32768) >> 16; +} + +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime) { + static uint64_t limit; + if (unlikely(monotime > limit)) { + if (limit != 0) + return UINT32_MAX; + limit = mdbx_osal_16dot16_to_monotime(UINT32_MAX - 1); + if (monotime > limit) + return UINT32_MAX; + } +#if defined(_WIN32) || defined(_WIN64) + return (uint32_t)((monotime << 16) / performance_frequency.QuadPart); +#elif defined(__APPLE__) || defined(__MACH__) + return (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine); +#else + return (uint32_t)(monotime * 128 / 1953125); +#endif +} + +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void) { +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER counter; + counter.QuadPart = 0; + QueryPerformanceCounter(&counter); + return counter.QuadPart; +#elif defined(__APPLE__) || defined(__MACH__) + return mach_absolute_time(); +#else + +#if defined(__linux__) || defined(__gnu_linux__) + static clockid_t posix_clockid = -1; + if (unlikely(posix_clockid < 0)) + posix_clockid = choice_monoclock(); +#elif defined(CLOCK_MONOTONIC) +#define posix_clockid CLOCK_MONOTONIC +#else +#define posix_clockid CLOCK_REALTIME +#endif + + struct timespec ts; + if (unlikely(clock_gettime(posix_clockid, &ts) != 0)) { + ts.tv_nsec = 0; + ts.tv_sec = 0; + } + return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec; +#endif +} + +/*----------------------------------------------------------------------------*/ + +static void bootid_shake(bin128_t *p) { + /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */ + const uint32_t e = p->a - (p->b << 23 | p->b >> 9); + p->a = p->b ^ (p->c << 16 | p->c >> 16); + p->b = p->c + (p->d << 11 | p->d >> 21); + p->c = p->d + e; + p->d = e + p->a; +} + +static void bootid_collect(bin128_t *p, const void *s, size_t n) { + p->y += UINT64_C(64526882297375213); + bootid_shake(p); + for (size_t i = 0; i < n; ++i) { + bootid_shake(p); + p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i]; + bootid_shake(p); + p->y += 14621231; + } + bootid_shake(p); + + /* minor non-linear tomfoolery */ + const unsigned z = p->x % 61; + p->y = p->y << z | p->y >> (64 - z); + bootid_shake(p); + bootid_shake(p); + const unsigned q = p->x % 59; + p->y = p->y << q | p->y >> (64 - q); + bootid_shake(p); + bootid_shake(p); + bootid_shake(p); +} + +#if defined(_WIN32) || defined(_WIN64) + +static uint64_t windows_systemtime_ms() { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul; +} + +static uint64_t windows_bootime(void) { + unsigned confirmed = 0; + uint64_t boottime = 0; + uint64_t up0 = mdbx_GetTickCount64(); + uint64_t st0 = windows_systemtime_ms(); + for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) { + YieldProcessor(); + const uint64_t up1 = mdbx_GetTickCount64(); + const uint64_t st1 = windows_systemtime_ms(); + if (st1 > fuse && st1 == st0 && up1 == up0) { + uint64_t diff = st1 - up1; + if (boottime == diff) { + if (++confirmed > 4) + return boottime; + } else { + confirmed = 0; + boottime = diff; + } + fuse = st1; + Sleep(1); + } + st0 = st1; + up0 = up1; + } + return 0; +} + +static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue, + PVOID pvData, LPDWORD pcbData) { + LSTATUS rc; + if (!mdbx_RegGetValueA) { + /* an old Windows 2000/XP */ + HKEY hSubKey; + rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey); + if (rc == ERROR_SUCCESS) { + rc = RegQueryValueExA(hSubKey, lpValue, NULL, NULL, pvData, pcbData); + RegCloseKey(hSubKey); + } + return rc; + } + + rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, NULL, pvData, + pcbData); + if (rc != ERROR_FILE_NOT_FOUND) + return rc; + + rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue, + RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, + NULL, pvData, pcbData); + if (rc != ERROR_FILE_NOT_FOUND) + return rc; + return mdbx_RegGetValueA(hKey, lpSubKey, lpValue, + RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, + NULL, pvData, pcbData); +} +#endif + +static __cold __maybe_unused bool bootid_parse_uuid(bin128_t *s, const void *p, + const size_t n) { + if (n > 31) { + unsigned bits = 0; + for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ { + uint8_t c = ((const uint8_t *)p)[i]; + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'a' && c <= 'f') + c -= 'a' - 10; + else if (c >= 'A' && c <= 'F') + c -= 'A' - 10; + else + continue; + assert(c <= 15); + c ^= s->y >> 60; + s->y = s->y << 4 | s->x >> 60; + s->x = s->x << 4 | c; + bits += 4; + } + if (bits > 42 * 3) + /* UUID parsed successfully */ + return true; + } + + if (n > 15) /* is enough handle it as a binary? */ { + if (n == sizeof(bin128_t)) { + bin128_t aligned; + memcpy(&aligned, p, sizeof(bin128_t)); + s->x += aligned.x; + s->y += aligned.y; + } else + bootid_collect(s, p, n); + return true; + } + + if (n) + bootid_collect(s, p, n); + return false; +} + +__cold MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void) { + bin128_t bin = {{0, 0}}; + bool got_machineid = false, got_boottime = false, got_bootseq = false; + +#if defined(__linux__) || defined(__gnu_linux__) + { + const int fd = + open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_NOFOLLOW); + if (fd != -1) { + struct statfs fs; + char buf[42]; + const ssize_t len = + (fstatfs(fd, &fs) == 0 && fs.f_type == /* procfs */ 0x9FA0) + ? read(fd, buf, sizeof(buf)) + : -1; + const int err = close(fd); + assert(err == 0); + (void)err; + if (len > 0 && bootid_parse_uuid(&bin, buf, len)) + return bin; + } + } +#endif /* Linux */ + +#if defined(__APPLE__) || defined(__MACH__) + { + char buf[42]; + size_t len = sizeof(buf); + if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && + bootid_parse_uuid(&bin, buf, len)) + return bin; + +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \ + __MAC_OS_X_VERSION_MIN_REQUIRED > 1050 + uuid_t uuid; + struct timespec wait = {0, 1000000000u / 42}; + if (!gethostuuid(uuid, &wait) && + bootid_parse_uuid(&bin, uuid, sizeof(uuid))) + got_machineid = true; +#endif /* > 10.5 */ + + struct timeval boottime; + len = sizeof(boottime); + if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && + len == sizeof(boottime) && boottime.tv_sec) + got_boottime = true; + } +#endif /* Apple/Darwin */ + +#if defined(_WIN32) || defined(_WIN64) + { + union buf { + DWORD BootId; + DWORD BaseTime; + SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo; + struct { + LARGE_INTEGER BootTime; + LARGE_INTEGER CurrentTime; + LARGE_INTEGER TimeZoneBias; + ULONG TimeZoneId; + ULONG Reserved; + ULONGLONG BootTimeBias; + ULONGLONG SleepTimeBias; + } SysTimeOfDayInfoHacked; + wchar_t MachineGuid[42]; + char DigitalProductId[248]; + } buf; + + static const char HKLM_MicrosoftCryptography[] = + "SOFTWARE\\Microsoft\\Cryptography"; + DWORD len = sizeof(buf); + /* Windows is madness and must die */ + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, + "MachineGuid", &buf.MachineGuid, + &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) + got_machineid = bootid_parse_uuid(&bin, &buf.MachineGuid, len); + + if (!got_machineid) { + /* again, Windows is madness */ + static const char HKLM_WindowsNT[] = + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; + static const char HKLM_WindowsNT_DPK[] = + "SOFTWARE\\Microsoft\\Windows " + "NT\\CurrentVersion\\DefaultProductKey"; + static const char HKLM_WindowsNT_DPK2[] = + "SOFTWARE\\Microsoft\\Windows " + "NT\\CurrentVersion\\DefaultProductKey2"; + + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, + "DigitalProductId", &buf.DigitalProductId, + &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.DigitalProductId, len); + got_machineid = true; + } + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, + "DigitalProductId", &buf.DigitalProductId, + &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.DigitalProductId, len); + got_machineid = true; + } + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, + "DigitalProductId", &buf.DigitalProductId, + &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.DigitalProductId, len); + got_machineid = true; + } + } + + static const char HKLM_PrefetcherParams[] = + "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory " + "Management\\PrefetchParameters"; + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", + &buf.BootId, &len) == ERROR_SUCCESS && + len > 1 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.BootId, len); + got_bootseq = true; + } + + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", + &buf.BaseTime, &len) == ERROR_SUCCESS && + len >= sizeof(buf.BaseTime) && buf.BaseTime) { + bootid_collect(&bin, &buf.BaseTime, len); + got_boottime = true; + } + + /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */ + NTSTATUS status = NtQuerySystemInformation( + 0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo, + sizeof(buf.SysTimeOfDayInfo), &len); + if (NT_SUCCESS(status) && + len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTime) + + sizeof(buf.SysTimeOfDayInfoHacked.BootTime) && + buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) { + bootid_collect(&bin, &buf.SysTimeOfDayInfoHacked.BootTime, + sizeof(buf.SysTimeOfDayInfoHacked.BootTime)); + got_boottime = true; + } + + if (!got_boottime) { + uint64_t boottime = windows_bootime(); + if (boottime) { + bootid_collect(&bin, &boottime, sizeof(boottime)); + got_boottime = true; + } + } + } +#endif /* Windows */ + +#if defined(CTL_HW) && defined(HW_UUID) + if (!got_machineid) { + static const int mib[] = {CTL_HW, HW_UUID}; + char buf[42]; + size_t len = sizeof(buf); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0) + got_machineid = bootid_parse_uuid(&bin, buf, len); + } +#endif /* CTL_HW && HW_UUID */ + +#if defined(CTL_KERN) && defined(KERN_HOSTUUID) + if (!got_machineid) { + static const int mib[] = {CTL_KERN, KERN_HOSTUUID}; + char buf[42]; + size_t len = sizeof(buf); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0) + got_machineid = bootid_parse_uuid(&bin, buf, len); + } +#endif /* CTL_KERN && KERN_HOSTUUID */ + +#if defined(__NetBSD__) + if (!got_machineid) { + char buf[42]; + size_t len = sizeof(buf); + if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, NULL, 0) == 0) + got_machineid = bootid_parse_uuid(&bin, buf, len); + } +#endif /* __NetBSD__ */ + +#if _XOPEN_SOURCE_EXTENDED + if (!got_machineid) { + const int hostid = gethostid(); + if (hostid > 0) { + bootid_collect(&bin, &hostid, sizeof(hostid)); + got_machineid = true; + } + } +#endif /* _XOPEN_SOURCE_EXTENDED */ + + if (!got_machineid) { + lack: + bin.x = bin.y = 0; + return bin; + } + + /*--------------------------------------------------------------------------*/ + +#if defined(CTL_KERN) && defined(KERN_BOOTTIME) + if (!got_boottime) { + static const int mib[] = {CTL_KERN, KERN_BOOTTIME}; + struct timeval boottime; + size_t len = sizeof(boottime); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &boottime, &len, NULL, 0) == 0 && + len == sizeof(boottime) && boottime.tv_sec) { + bootid_collect(&bin, &boottime, len); + got_boottime = true; + } + } +#endif /* CTL_KERN && KERN_BOOTTIME */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) + if (!got_boottime) { + kstat_ctl_t *kc = kstat_open(); + if (kc) { + kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc"); + if (kp && kstat_read(kc, kp, 0) != -1) { + kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time"); + if (kn) { + switch (kn->data_type) { + case KSTAT_DATA_INT32: + case KSTAT_DATA_UINT32: + bootid_collect(&bin, &kn->value, sizeof(int32_t)); + got_boottime = true; + case KSTAT_DATA_INT64: + case KSTAT_DATA_UINT64: + bootid_collect(&bin, &kn->value, sizeof(int64_t)); + got_boottime = true; + } + } + } + kstat_close(kc); + } + } +#endif /* SunOS / Solaris */ + +#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME) + if (!got_boottime) { + setutxent(); + const struct utmpx id = {.ut_type = BOOT_TIME}; + const struct utmpx *entry = getutxid(&id); + if (entry) { + bootid_collect(&bin, entry, sizeof(*entry)); + got_boottime = true; + while (unlikely((entry = getutxid(&id)) != nullptr)) { + /* have multiple reboot records, assuming we can distinguish next + * bootsession even if RTC is wrong or absent */ + bootid_collect(&bin, entry, sizeof(*entry)); + got_bootseq = true; + } + } + endutxent(); + } +#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */ + + if (!got_bootseq) { + if (!got_boottime || !MDBX_TRUST_RTC) + goto lack; + +#if defined(_WIN32) || defined(_WIN64) + FILETIME now; + GetSystemTimeAsFileTime(&now); + if (0x1CCCCCC > now.dwHighDateTime) +#else + struct timespec mono, real; + if (clock_gettime(CLOCK_MONOTONIC, &mono) || + clock_gettime(CLOCK_REALTIME, &real) || + /* wrong time, RTC is mad or absent */ + 1555555555l > real.tv_sec || + /* seems no adjustment by RTC/NTP, i.e. a fake time */ + real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || + (real.tv_sec - mono.tv_sec) % 900u == 0) +#endif + goto lack; + } + + return bin; +} +/* This is CMake-template for libmdbx's version.c + ******************************************************************************/ + + +#if MDBX_VERSION_MAJOR != 0 || \ + MDBX_VERSION_MINOR != 9 +#error "API version mismatch! Had `git fetch --tags` done?" +#endif + +static const char sourcery[] = STRINGIFY(MDBX_BUILD_SOURCERY); + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || \ + __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const struct MDBX_version_info mdbx_version = { + 0, + 9, + 1, + 18, + {"2020-10-08T01:50:18+03:00", "5cbfdfdc65b62937f7fc927e55cec6304bf04f7c", "1d31ebdc1c930ceb1b223691ac27e507d86cf8e2", + "v0.9.1-18-g1d31ebdc1c"}, + sourcery}; + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || \ + __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const char *const mdbx_sourcery_anchor = sourcery; +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +#if defined(_WIN32) || defined(_WIN64) /* Windows LCK-implementation */ + +/* PREAMBLE FOR WINDOWS: + * + * We are not concerned for performance here. + * If you are running Windows a performance could NOT be the goal. + * Otherwise please use Linux. */ + + +static void mdbx_winnt_import(void); + +#if MDBX_BUILD_SHARED_LIBRARY +#if MDBX_AVOID_CRT && defined(NDEBUG) +/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks. + * + * Define dll's entry point only for Release build when NDEBUG is defined and + * MDBX_AVOID_CRT=ON. if the entry point isn't defined then MSVC's will + * automatically use DllMainCRTStartup() from CRT library, which also + * automatically call DllMain() from our mdbx.dll */ +#pragma comment(linker, "/ENTRY:DllMain") +#endif /* MDBX_AVOID_CRT */ + +BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved) +#else +#if !MDBX_CONFIG_MANUAL_TLS_CALLBACK +static +#endif /* !MDBX_CONFIG_MANUAL_TLS_CALLBACK */ + void NTAPI + mdbx_dll_handler(PVOID module, DWORD reason, PVOID reserved) +#endif /* MDBX_BUILD_SHARED_LIBRARY */ +{ + (void)reserved; + switch (reason) { + case DLL_PROCESS_ATTACH: + mdbx_winnt_import(); + mdbx_rthc_global_init(); + break; + case DLL_PROCESS_DETACH: + mdbx_rthc_global_dtor(); + break; + + case DLL_THREAD_ATTACH: + break; + case DLL_THREAD_DETACH: + mdbx_rthc_thread_dtor(module); + break; + } +#if MDBX_BUILD_SHARED_LIBRARY + return TRUE; +#endif +} + +#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(_MSC_VER) +# pragma const_seg(push) +# pragma data_seg(push) + +# ifdef _WIN64 + /* kick a linker to create the TLS directory if not already done */ +# pragma comment(linker, "/INCLUDE:_tls_used") + /* Force some symbol references. */ +# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor") + /* specific const-segment for WIN64 */ +# pragma const_seg(".CRT$XLB") + const +# else + /* kick a linker to create the TLS directory if not already done */ +# pragma comment(linker, "/INCLUDE:__tls_used") + /* Force some symbol references. */ +# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor") + /* specific data-segment for WIN32 */ +# pragma data_seg(".CRT$XLB") +# endif + + __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_dll_handler; +# pragma data_seg(pop) +# pragma const_seg(pop) + +#elif defined(__GNUC__) +# ifdef _WIN64 + const +# endif + PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_dll_handler; +#else +# error FIXME +#endif +/* *INDENT-ON* */ +/* clang-format on */ +#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK */ + +/*----------------------------------------------------------------------------*/ + +#define LCK_SHARED 0 +#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK +#define LCK_WAITFOR 0 +#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY + +static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset, + size_t bytes) { + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov); +} + +static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset, + size_t bytes) { + return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, + HIGH_DWORD(bytes)); +} + +/*----------------------------------------------------------------------------*/ +/* global `write` lock for write-txt processing, + * exclusive locking both meta-pages) */ + +#define LCK_MAXLEN (1u + (size_t)(MAXSSIZE_T)) +#define LCK_META_OFFSET 0 +#define LCK_META_LEN (MAX_PAGESIZE * NUM_METAS) +#define LCK_BODY_OFFSET LCK_META_LEN +#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET) +#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN +#define LCK_WHOLE 0, LCK_MAXLEN + +int mdbx_txn_lock(MDBX_env *env, bool dontwait) { + if (dontwait) { + if (!TryEnterCriticalSection(&env->me_windowsbug_lock)) + return MDBX_BUSY; + } else { + EnterCriticalSection(&env->me_windowsbug_lock); + } + + if ((env->me_flags & MDBX_EXCLUSIVE) || + flock(env->me_lazy_fd, + dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) + : (LCK_EXCLUSIVE | LCK_WAITFOR), + LCK_BODY)) + return MDBX_SUCCESS; + int rc = GetLastError(); + LeaveCriticalSection(&env->me_windowsbug_lock); + return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY; +} + +void mdbx_txn_unlock(MDBX_env *env) { + int rc = (env->me_flags & MDBX_EXCLUSIVE) + ? TRUE + : funlock(env->me_lazy_fd, LCK_BODY); + LeaveCriticalSection(&env->me_windowsbug_lock); + if (!rc) + mdbx_panic("%s failed: err %u", __func__, GetLastError()); +} + +/*----------------------------------------------------------------------------*/ +/* global `read` lock for readers registration, + * exclusive locking `mti_numreaders` (second) cacheline */ + +#define LCK_LO_OFFSET 0 +#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders) +#define LCK_UP_OFFSET LCK_LO_LEN +#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET) +#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN +#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN + +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) { + mdbx_srwlock_AcquireShared(&env->me_remap_guard); + if (env->me_lfd == INVALID_HANDLE_VALUE) + return MDBX_SUCCESS; /* readonly database in readonly filesystem */ + + /* transition from S-? (used) to S-E (locked), + * e.g. exclusive lock upper-part */ + if ((env->me_flags & MDBX_EXCLUSIVE) || + flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) + return MDBX_SUCCESS; + + int rc = GetLastError(); + mdbx_srwlock_ReleaseShared(&env->me_remap_guard); + return rc; +} + +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) { + if (env->me_lfd != INVALID_HANDLE_VALUE) { + /* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */ + if ((env->me_flags & MDBX_EXCLUSIVE) == 0 && + !funlock(env->me_lfd, LCK_UPPER)) + mdbx_panic("%s failed: err %u", __func__, GetLastError()); + } + mdbx_srwlock_ReleaseShared(&env->me_remap_guard); +} + +static int suspend_and_append(mdbx_handle_array_t **array, + const DWORD ThreadId) { + const unsigned limit = (*array)->limit; + if ((*array)->count == limit) { + void *ptr = mdbx_realloc( + (limit > ARRAY_LENGTH((*array)->handles)) + ? *array + : /* don't free initial array on the stack */ NULL, + sizeof(mdbx_handle_array_t) + + sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles))); + if (!ptr) + return MDBX_ENOMEM; + if (limit == ARRAY_LENGTH((*array)->handles)) + memcpy(ptr, *array, sizeof(mdbx_handle_array_t)); + *array = (mdbx_handle_array_t *)ptr; + (*array)->limit = limit * 2; + } + + HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, + FALSE, ThreadId); + if (hThread == NULL) + return GetLastError(); + + if (SuspendThread(hThread) == -1) { + int err = GetLastError(); + DWORD ExitCode; + if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || + !GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE) + err = MDBX_SUCCESS; + CloseHandle(hThread); + return err; + } + + (*array)->handles[(*array)->count++] = hThread; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { + const uintptr_t CurrentTid = GetCurrentThreadId(); + int rc; + if (env->me_lck) { + /* Scan LCK for threads of the current process */ + const MDBX_reader *const begin = env->me_lck->mti_readers; + const MDBX_reader *const end = begin + env->me_lck->mti_numreaders; + const uintptr_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0; + for (const MDBX_reader *reader = begin; reader < end; ++reader) { + if (reader->mr_pid != env->me_pid || !reader->mr_tid) { + skip_lck: + continue; + } + if (reader->mr_tid == CurrentTid || reader->mr_tid == WriteTxnOwner) + goto skip_lck; + if (env->me_flags & MDBX_NOTLS) { + /* Skip duplicates in no-tls mode */ + for (const MDBX_reader *scan = reader; --scan >= begin;) + if (scan->mr_tid == reader->mr_tid) + goto skip_lck; + } + + rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid); + if (rc != MDBX_SUCCESS) { + bailout_lck: + (void)mdbx_resume_threads_after_remap(*array); + return rc; + } + } + if (WriteTxnOwner && WriteTxnOwner != CurrentTid) { + rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner); + if (rc != MDBX_SUCCESS) + goto bailout_lck; + } + } else { + /* Without LCK (i.e. read-only mode). + * Walk through a snapshot of all running threads */ + mdbx_assert(env, + env->me_txn0 == NULL || (env->me_flags & MDBX_EXCLUSIVE) != 0); + const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (hSnapshot == INVALID_HANDLE_VALUE) + return GetLastError(); + + THREADENTRY32 entry; + entry.dwSize = sizeof(THREADENTRY32); + + if (!Thread32First(hSnapshot, &entry)) { + rc = GetLastError(); + bailout_toolhelp: + CloseHandle(hSnapshot); + (void)mdbx_resume_threads_after_remap(*array); + return rc; + } + + do { + if (entry.th32OwnerProcessID != env->me_pid || + entry.th32ThreadID == CurrentTid) + continue; + + rc = suspend_and_append(array, entry.th32ThreadID); + if (rc != MDBX_SUCCESS) + goto bailout_toolhelp; + + } while (Thread32Next(hSnapshot, &entry)); + + rc = GetLastError(); + if (rc != ERROR_NO_MORE_FILES) + goto bailout_toolhelp; + CloseHandle(hSnapshot); + } + + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) { + int rc = MDBX_SUCCESS; + for (unsigned i = 0; i < array->count; ++i) { + const HANDLE hThread = array->handles[i]; + if (ResumeThread(hThread) == -1) { + const int err = GetLastError(); + DWORD ExitCode; + if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && + GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE) + rc = err; + } + CloseHandle(hThread); + } + return rc; +} + +/*----------------------------------------------------------------------------*/ +/* global `initial` lock for lockfile initialization, + * exclusive/shared locking first cacheline */ + +/* Briefly description of locking schema/algorithm: + * - Windows does not support upgrading or downgrading for file locking. + * - Therefore upgrading/downgrading is emulated by shared and exclusive + * locking of upper and lower halves. + * - In other words, we have FSM with possible 9 states, + * i.e. free/shared/exclusive x free/shared/exclusive == 9. + * Only 6 states of FSM are used, which 2 of ones are transitive. + * + * States: + * ?-? = free, i.e. unlocked + * S-? = used, i.e. shared lock + * E-? = exclusive-read, i.e. operational exclusive + * ?-S + * ?-E = middle (transitive state) + * S-S + * S-E = locked (transitive state) + * E-S + * E-E = exclusive-write, i.e. exclusive due (re)initialization + * + * The mdbx_lck_seize() moves the locking-FSM from the initial free/unlocked + * state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible, + * or to the "used" (and returns MDBX_RESULT_FALSE). + * + * The mdbx_lck_downgrade() moves the locking-FSM from "exclusive write" + * state to the "used" (i.e. shared) state. + * + * The mdbx_lck_upgrade() moves the locking-FSM from "used" (i.e. shared) + * state to the "exclusive write" state. + */ + +static void lck_unlock(MDBX_env *env) { + int err; + + if (env->me_lfd != INVALID_HANDLE_VALUE) { + /* double `unlock` for robustly remove overlapped shared/exclusive locks */ + while (funlock(env->me_lfd, LCK_LOWER)) + ; + err = GetLastError(); + assert(err == ERROR_NOT_LOCKED || + (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); + (void)err; + SetLastError(ERROR_SUCCESS); + + while (funlock(env->me_lfd, LCK_UPPER)) + ; + err = GetLastError(); + assert(err == ERROR_NOT_LOCKED || + (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); + (void)err; + SetLastError(ERROR_SUCCESS); + } + + if (env->me_lazy_fd != INVALID_HANDLE_VALUE) { + /* explicitly unlock to avoid latency for other processes (windows kernel + * releases such locks via deferred queues) */ + while (funlock(env->me_lazy_fd, LCK_BODY)) + ; + err = GetLastError(); + assert(err == ERROR_NOT_LOCKED || + (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); + (void)err; + SetLastError(ERROR_SUCCESS); + + while (funlock(env->me_lazy_fd, LCK_WHOLE)) + ; + err = GetLastError(); + assert(err == ERROR_NOT_LOCKED || + (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); + (void)err; + SetLastError(ERROR_SUCCESS); + } +} + +/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE) + * or as 'used' (S-? and returns MDBX_RESULT_FALSE). + * Otherwise returns an error. */ +static int internal_seize_lck(HANDLE lfd) { + int rc; + assert(lfd != INVALID_HANDLE_VALUE); + + /* 1) now on ?-? (free), get ?-E (middle) */ + mdbx_jitter4testing(false); + if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) { + rc = GetLastError() /* 2) something went wrong, give up */; + mdbx_error("%s, err %u", "?-?(free) >> ?-E(middle)", rc); + return rc; + } + + /* 3) now on ?-E (middle), try E-E (exclusive-write) */ + mdbx_jitter4testing(false); + if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER)) + return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */; + + /* 5) still on ?-E (middle) */ + rc = GetLastError(); + mdbx_jitter4testing(false); + if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) { + /* 6) something went wrong, give up */ + if (!funlock(lfd, LCK_UPPER)) + mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)", + GetLastError()); + return rc; + } + + /* 7) still on ?-E (middle), try S-E (locked) */ + mdbx_jitter4testing(false); + rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE + : GetLastError(); + + mdbx_jitter4testing(false); + if (rc != MDBX_RESULT_FALSE) + mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc); + + /* 8) now on S-E (locked) or still on ?-E (middle), + * transition to S-? (used) or ?-? (free) */ + if (!funlock(lfd, LCK_UPPER)) + mdbx_panic("%s(%s) failed: err %u", __func__, + "X-E(locked/middle) >> X-?(used/free)", GetLastError()); + + /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */ + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) { + int rc; + + assert(env->me_lazy_fd != INVALID_HANDLE_VALUE); + if (env->me_flags & MDBX_EXCLUSIVE) + return MDBX_RESULT_TRUE /* nope since files were must be opened + non-shareable */ + ; + + if (env->me_lfd == INVALID_HANDLE_VALUE) { + /* LY: without-lck mode (e.g. on read-only filesystem) */ + mdbx_jitter4testing(false); + if (!flock(env->me_lazy_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) { + rc = GetLastError(); + mdbx_error("%s, err %u", "without-lck", rc); + return rc; + } + return MDBX_RESULT_FALSE; + } + + rc = internal_seize_lck(env->me_lfd); + mdbx_jitter4testing(false); + if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) { + /* Check that another process don't operates in without-lck mode. + * Doing such check by exclusive locking the body-part of db. Should be + * noted: + * - we need an exclusive lock for do so; + * - we can't lock meta-pages, otherwise other process could get an error + * while opening db in valid (non-conflict) mode. */ + if (!flock(env->me_lazy_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) { + rc = GetLastError(); + mdbx_error("%s, err %u", "lock-against-without-lck", rc); + mdbx_jitter4testing(false); + lck_unlock(env); + } else { + mdbx_jitter4testing(false); + if (!funlock(env->me_lazy_fd, LCK_BODY)) + mdbx_panic("%s(%s) failed: err %u", __func__, + "unlock-against-without-lck", GetLastError()); + } + } + + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) { + /* Transite from exclusive-write state (E-E) to used (S-?) */ + assert(env->me_lazy_fd != INVALID_HANDLE_VALUE); + assert(env->me_lfd != INVALID_HANDLE_VALUE); + + if (env->me_flags & MDBX_EXCLUSIVE) + return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ + ; + /* 1) now at E-E (exclusive-write), transition to ?_E (middle) */ + if (!funlock(env->me_lfd, LCK_LOWER)) + mdbx_panic("%s(%s) failed: err %u", __func__, + "E-E(exclusive-write) >> ?-E(middle)", GetLastError()); + + /* 2) now at ?-E (middle), transition to S-E (locked) */ + if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) { + int rc = GetLastError() /* 3) something went wrong, give up */; + mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc); + return rc; + } + + /* 4) got S-E (locked), continue transition to S-? (used) */ + if (!funlock(env->me_lfd, LCK_UPPER)) + mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)", + GetLastError()); + + return MDBX_SUCCESS /* 5) now at S-? (used), done */; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_upgrade(MDBX_env *env) { + /* Transite from used state (S-?) to exclusive-write (E-E) */ + assert(env->me_lfd != INVALID_HANDLE_VALUE); + + if (env->me_flags & MDBX_EXCLUSIVE) + return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ + ; + + int rc; + /* 1) now on S-? (used), try S-E (locked) */ + mdbx_jitter4testing(false); + if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_UPPER)) { + rc = GetLastError() /* 2) something went wrong, give up */; + mdbx_verbose("%s, err %u", "S-?(used) >> S-E(locked)", rc); + return rc; + } + + /* 3) now on S-E (locked), transition to ?-E (middle) */ + if (!funlock(env->me_lfd, LCK_LOWER)) + mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)", + GetLastError()); + + /* 4) now on ?-E (middle), try E-E (exclusive-write) */ + mdbx_jitter4testing(false); + if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER)) { + rc = GetLastError() /* 5) something went wrong, give up */; + mdbx_verbose("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc); + return rc; + } + + return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag) { + (void)env; + (void)inprocess_neighbor; + (void)global_uniqueness_flag; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor) { + /* LY: should unmap before releasing the locks to avoid race condition and + * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */ + if (env->me_map) + mdbx_munmap(&env->me_dxb_mmap); + if (env->me_lck) { + const bool synced = env->me_lck_mmap.lck->mti_unsynced_pages == 0; + mdbx_munmap(&env->me_lck_mmap); + if (synced && !inprocess_neighbor && env->me_lfd != INVALID_HANDLE_VALUE && + mdbx_lck_upgrade(env) == MDBX_SUCCESS) + /* this will fail if LCK is used/mmapped by other process(es) */ + mdbx_ftruncate(env->me_lfd, 0); + } + lck_unlock(env); + return MDBX_SUCCESS; +} + +/*----------------------------------------------------------------------------*/ +/* reader checking (by pid) */ + +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) { + (void)env; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) { + (void)env; + return MDBX_SUCCESS; +} + +/* Checks reader by pid. + * + * Returns: + * MDBX_RESULT_TRUE, if pid is live (unable to acquire lock) + * MDBX_RESULT_FALSE, if pid is dead (lock acquired) + * or otherwise the errcode. */ +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) { + (void)env; + HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid); + int rc; + if (likely(hProcess)) { + rc = WaitForSingleObject(hProcess, 0); + if (unlikely(rc == WAIT_FAILED)) + rc = GetLastError(); + CloseHandle(hProcess); + } else { + rc = GetLastError(); + } + + switch (rc) { + case ERROR_INVALID_PARAMETER: + /* pid seems invalid */ + return MDBX_RESULT_FALSE; + case WAIT_OBJECT_0: + /* process just exited */ + return MDBX_RESULT_FALSE; + case ERROR_ACCESS_DENIED: + /* The ERROR_ACCESS_DENIED would be returned for CSRSS-processes, etc. + * assume pid exists */ + return MDBX_RESULT_TRUE; + case WAIT_TIMEOUT: + /* pid running */ + return MDBX_RESULT_TRUE; + default: + /* failure */ + return rc; + } +} + +//---------------------------------------------------------------------------- +// Stub for slim read-write lock +// Copyright (C) 1995-2002 Brad Wilson + +static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) { + srwl->readerCount = srwl->writerCount = 0; +} + +static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) { + while (true) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + + // If there's a writer already, spin without unnecessarily + // interlocking the CPUs + if (srwl->writerCount != 0) { + YieldProcessor(); + continue; + } + + // Add to the readers list + _InterlockedIncrement(&srwl->readerCount); + + // Check for writers again (we may have been preempted). If + // there are no writers writing or waiting, then we're done. + if (srwl->writerCount == 0) + break; + + // Remove from the readers list, spin, try again + _InterlockedDecrement(&srwl->readerCount); + YieldProcessor(); + } +} + +static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) { + assert(srwl->readerCount > 0); + _InterlockedDecrement(&srwl->readerCount); +} + +static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) { + while (true) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + + // If there's a writer already, spin without unnecessarily + // interlocking the CPUs + if (srwl->writerCount != 0) { + YieldProcessor(); + continue; + } + + // See if we can become the writer (expensive, because it inter- + // locks the CPUs, so writing should be an infrequent process) + if (_InterlockedExchange(&srwl->writerCount, 1) == 0) + break; + } + + // Now we're the writer, but there may be outstanding readers. + // Spin until there aren't any more; new readers will wait now + // that we're the writer. + while (srwl->readerCount != 0) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + YieldProcessor(); + } +} + +static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) { + assert(srwl->writerCount == 1 && srwl->readerCount >= 0); + srwl->writerCount = 0; +} + +MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared, + mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive, + mdbx_srwlock_ReleaseExclusive; + +/*----------------------------------------------------------------------------*/ + +#if 0 /* LY: unused for now */ +static DWORD WINAPI stub_DiscardVirtualMemory(PVOID VirtualAddress, + SIZE_T Size) { + return VirtualAlloc(VirtualAddress, Size, MEM_RESET, PAGE_NOACCESS) + ? ERROR_SUCCESS + : GetLastError(); +} +#endif /* unused for now */ + +static uint64_t WINAPI stub_GetTickCount64(void) { + LARGE_INTEGER Counter, Frequency; + return (QueryPerformanceFrequency(&Frequency) && + QueryPerformanceCounter(&Counter)) + ? Counter.QuadPart * 1000ul / Frequency.QuadPart + : 0; +} + +/*----------------------------------------------------------------------------*/ +#ifndef MDBX_ALLOY +MDBX_NtExtendSection mdbx_NtExtendSection; +MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx; +MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW; +MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; +MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle; +MDBX_NtFsControlFile mdbx_NtFsControlFile; +MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; +MDBX_GetTickCount64 mdbx_GetTickCount64; +MDBX_RegGetValueA mdbx_RegGetValueA; +#if 0 /* LY: unused for now */ +MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; +MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; +#endif /* unused for now */ +#endif /* MDBX_ALLOY */ + +static void mdbx_winnt_import(void) { + const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll"); + +#define GET_PROC_ADDR(dll, ENTRY) \ + mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(dll, #ENTRY) + + if (GetProcAddress(hNtdll, "wine_get_version")) { + assert(mdbx_RunningUnderWine()); + } else { + GET_PROC_ADDR(hNtdll, NtFsControlFile); + GET_PROC_ADDR(hNtdll, NtExtendSection); + assert(!mdbx_RunningUnderWine()); + } + + const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll"); + GET_PROC_ADDR(hKernel32dll, GetFileInformationByHandleEx); + GET_PROC_ADDR(hKernel32dll, GetTickCount64); + if (!mdbx_GetTickCount64) + mdbx_GetTickCount64 = stub_GetTickCount64; + if (!mdbx_RunningUnderWine()) { + GET_PROC_ADDR(hKernel32dll, SetFileInformationByHandle); + GET_PROC_ADDR(hKernel32dll, GetVolumeInformationByHandleW); + GET_PROC_ADDR(hKernel32dll, GetFinalPathNameByHandleW); + GET_PROC_ADDR(hKernel32dll, PrefetchVirtualMemory); + } + + const HINSTANCE hAdvapi32dll = GetModuleHandleA("advapi32.dll"); + GET_PROC_ADDR(hAdvapi32dll, RegGetValueA); + +#if 0 /* LY: unused for now */ + if (!mdbx_RunningUnderWine()) { + GET_PROC_ADDR(hKernel32dll, DiscardVirtualMemory); + GET_PROC_ADDR(hKernel32dll, OfferVirtualMemory); + GET_PROC_ADDR(hKernel32dll, ReclaimVirtualMemory); + } + if (!mdbx_DiscardVirtualMemory) + mdbx_DiscardVirtualMemory = stub_DiscardVirtualMemory; + if (!mdbx_OfferVirtualMemory) + mdbx_OfferVirtualMemory = stub_OfferVirtualMemory; + if (!mdbx_ReclaimVirtualMemory) + mdbx_ReclaimVirtualMemory = stub_ReclaimVirtualMemory; +#endif /* unused for now */ + +#undef GET_PROC_ADDR + + const MDBX_srwlock_function init = + (MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock"); + if (init != NULL) { + mdbx_srwlock_Init = init; + mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "AcquireSRWLockShared"); + mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "ReleaseSRWLockShared"); + mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "AcquireSRWLockExclusive"); + mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "ReleaseSRWLockExclusive"); + } else { + mdbx_srwlock_Init = stub_srwlock_Init; + mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared; + mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared; + mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive; + mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive; + } +} + +#endif /* Windows LCK-implementation */ +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +#if !(defined(_WIN32) || defined(_WIN64)) /* !Windows LCK-implementation */ + +#include <sys/sem.h> + +/*----------------------------------------------------------------------------*/ +/* global constructor/destructor */ + +#if defined(__linux__) || defined(__gnu_linux__) + +#include <sys/utsname.h> + +#ifndef MDBX_ALLOY +uint32_t mdbx_linux_kernel_version; +bool mdbx_RunningOnWSL; +#endif /* MDBX_ALLOY */ + +static __cold bool probe_for_WSL(const char *tag) { + /* "Official" way of detecting WSL but not WSL2 + * https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 */ + return strstr(tag, "Microsoft") || strstr(tag, "WSL"); +} + +#endif /* Linux */ + +static __cold __attribute__((__constructor__)) void +mdbx_global_constructor(void) { +#if defined(__linux__) || defined(__gnu_linux__) + struct utsname buffer; + if (uname(&buffer) == 0) { + mdbx_RunningOnWSL = probe_for_WSL(buffer.version) || + probe_for_WSL(buffer.sysname) || + probe_for_WSL(buffer.release); + int i = 0; + char *p = buffer.release; + while (*p && i < 4) { + if (*p >= '0' && *p <= '9') { + long number = strtol(p, &p, 10); + if (number > 0) { + if (number > 255) + number = 255; + mdbx_linux_kernel_version += number << (24 - i * 8); + } + ++i; + } else { + ++p; + } + } + } +#endif /* Linux */ + + mdbx_rthc_global_init(); +} + +static __cold __attribute__((__destructor__)) void +mdbx_global_destructor(void) { + mdbx_rthc_global_dtor(); +} + +/*----------------------------------------------------------------------------*/ +/* lck */ + +/* Описание реализации блокировок для POSIX & Linux: + * + * lck-файл отображается в память, в нём организуется таблица читателей и + * размещаются совместно используемые posix-мьютексы (futex). Посредством + * этих мьютексов (см struct MDBX_lockinfo) реализуются: + * - Блокировка таблицы читателей для регистрации, + * т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock(). + * - Блокировка БД для пишущих транзакций, + * т.е. функции mdbx_txn_lock() и mdbx_txn_unlock(). + * + * Остальной функционал реализуется отдельно посредством файловых блокировок: + * - Первоначальный захват БД в режиме exclusive/shared и последующий перевод + * в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade(). + * - Проверка присутствие процессов-читателей, + * т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check(). + * + * Для блокировки файлов используется fcntl(F_SETLK), так как: + * - lockf() оперирует только эксклюзивной блокировкой и требует + * открытия файла в RW-режиме. + * - flock() не гарантирует атомарности при смене блокировок + * и оперирует только всем файлом целиком. + * - Для контроля процессов-читателей используются однобайтовые + * range-блокировки lck-файла посредством fcntl(F_SETLK). При этом + * в качестве позиции используется pid процесса-читателя. + * - Для первоначального захвата и shared/exclusive выполняется блокировка + * основного файла БД и при успехе lck-файла. + * + * ---------------------------------------------------------------------------- + * УДЕРЖИВАЕМЫЕ БЛОКИРОВКИ В ЗАВИСИМОСТИ ОТ РЕЖИМА И СОСТОЯНИЯ + * + * Эксклюзивный режим без lck-файла: + * = заблокирован весь dxb-файл посредством F_RDLCK или F_WRLCK, + * в зависимости от MDBX_RDONLY. + * + * Не-операционный режим на время пере-инициализации и разрушении lck-файла: + * = F_WRLCK блокировка первого байта lck-файла, другие процессы ждут её + * снятия при получении F_RDLCK через F_SETLKW. + * - блокировки dxb-файла могут меняться до снятие эксклюзивной блокировки + * lck-файла: + * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + * + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + * + * ОПЕРАЦИОННЫЙ режим с lck-файлом: + * = F_RDLCK блокировка первого байта lck-файла, другие процессы не могут + * получить F_WRLCK и таким образом видят что БД используется. + * + F_WRLCK блокировка pid-байта в clk-файле после первой транзакции чтения. + * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + * + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + */ + +#if MDBX_USE_OFDLOCKS +static int op_setlk, op_setlkw, op_getlk; +static void __cold choice_fcntl() { + assert(!op_setlk && !op_setlkw && !op_getlk); + if ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 +#if defined(__linux__) || defined(__gnu_linux__) + && mdbx_linux_kernel_version > + 0x030f0000 /* OFD locks are available since 3.15, but engages here + only for 3.16 and later kernels (i.e. LTS) because + of reliability reasons */ +#endif /* linux */ + ) { + op_setlk = F_OFD_SETLK; + op_setlkw = F_OFD_SETLKW; + op_getlk = F_OFD_GETLK; + return; + } + op_setlk = F_SETLK; + op_setlkw = F_SETLKW; + op_getlk = F_GETLK; +} +#else +#define op_setlk F_SETLK +#define op_setlkw F_SETLKW +#define op_getlk F_GETLK +#endif /* MDBX_USE_OFDLOCKS */ + +#ifndef OFF_T_MAX +#define OFF_T_MAX \ + ((sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff) +#endif + +static int lck_op(mdbx_filehandle_t fd, int cmd, int lck, off_t offset, + off_t len) { + mdbx_jitter4testing(true); + for (;;) { + struct flock lock_op; + memset(&lock_op, 0, sizeof(lock_op)); + lock_op.l_type = lck; + lock_op.l_whence = SEEK_SET; + lock_op.l_start = offset; + lock_op.l_len = len; + int rc = fcntl(fd, cmd, &lock_op); + mdbx_jitter4testing(true); + if (rc != -1) { + if (cmd == op_getlk) { + /* Checks reader by pid. Returns: + * MDBX_RESULT_TRUE - if pid is live (reader holds a lock). + * MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */ + return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE + : MDBX_RESULT_TRUE; + } + return MDBX_SUCCESS; + } + rc = errno; + if (rc != EINTR || cmd == op_setlkw) { + mdbx_assert(nullptr, MDBX_IS_ERROR(rc)); + return rc; + } + } +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + assert(env->me_pid > 0); + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; + return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1); +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + assert(env->me_pid > 0); + return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1); +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + assert(pid > 0); + return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1); +} + +/*---------------------------------------------------------------------------*/ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + return sem_init(ipc, false, 1) ? errno : 0; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + return pthread_mutex_init(ipc, nullptr); +#else +#error "FIXME" +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + return sem_destroy(ipc) ? errno : 0; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + return pthread_mutex_destroy(ipc); +#else +#error "FIXME" +#endif +} +#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */ + +static int check_fstat(MDBX_env *env) { + struct stat st; + + int rc = MDBX_SUCCESS; + if (fstat(env->me_lazy_fd, &st)) { + rc = errno; + mdbx_error("fstat(%s), err %d", "DXB", rc); + return rc; + } + + if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { +#ifdef EBADFD + rc = EBADFD; +#else + rc = EPERM; +#endif + mdbx_error("%s %s, err %d", "DXB", + (st.st_nlink < 1) ? "file was removed" : "not a regular file", + rc); + return rc; + } + + if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) { + mdbx_verbose("dxb-file is too short (%u), exclusive-lock needed", + (unsigned)st.st_size); + rc = MDBX_RESULT_TRUE; + } + + //---------------------------------------------------------------------------- + + if (fstat(env->me_lfd, &st)) { + rc = errno; + mdbx_error("fstat(%s), err %d", "LCK", rc); + return rc; + } + + if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { +#ifdef EBADFD + rc = EBADFD; +#else + rc = EPERM; +#endif + mdbx_error("%s %s, err %d", "LCK", + (st.st_nlink < 1) ? "file was removed" : "not a regular file", + rc); + return rc; + } + + /* Checking file size for detect the situation when we got the shared lock + * immediately after mdbx_lck_destroy(). */ + if (st.st_size < (off_t)(sizeof(MDBX_lockinfo) + sizeof(MDBX_reader))) { + mdbx_verbose("lck-file is too short (%u), exclusive-lock needed", + (unsigned)st.st_size); + rc = MDBX_RESULT_TRUE; + } + + return rc; +} + +MDBX_INTERNAL_FUNC int __cold mdbx_lck_seize(MDBX_env *env) { + assert(env->me_lazy_fd != INVALID_HANDLE_VALUE); + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; +#if MDBX_USE_OFDLOCKS + if (unlikely(op_setlk == 0)) + choice_fcntl(); +#endif /* MDBX_USE_OFDLOCKS */ + + int rc = MDBX_SUCCESS; +#if defined(__linux__) || defined(__gnu_linux__) + if (unlikely(mdbx_RunningOnWSL)) { + rc = ENOLCK /* No record locks available */; + mdbx_error("%s, err %u", + "WSL (Windows Subsystem for Linux) is mad and trouble-full, " + "injecting failure to avoid data loss", + rc); + return rc; + } +#endif /* Linux */ + + if (env->me_lfd == INVALID_HANDLE_VALUE) { + /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ + rc = + lck_op(env->me_lazy_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s, err %u", "without-lck", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; + } +#if defined(_POSIX_PRIORITY_SCHEDULING) && _POSIX_PRIORITY_SCHEDULING > 0 + sched_yield(); +#endif + +retry: + if (rc == MDBX_RESULT_TRUE) { + rc = lck_op(env->me_lfd, op_setlk, F_UNLCK, 0, 1); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s, err %u", "unlock-before-retry", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + } + + /* Firstly try to get exclusive locking. */ + rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); + if (rc == MDBX_SUCCESS) { + rc = check_fstat(env); + if (MDBX_IS_ERROR(rc)) + return rc; + + continue_dxb_exclusive: + rc = + lck_op(env->me_lazy_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + if (rc == MDBX_SUCCESS) + return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; + + int err = check_fstat(env); + if (MDBX_IS_ERROR(err)) + return err; + + /* the cause may be a collision with POSIX's file-lock recovery. */ + if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || + rc == EDEADLK)) { + mdbx_error("%s, err %u", "dxb-exclusive", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Fallback to lck-shared */ + } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || + rc == EWOULDBLOCK || rc == EDEADLK)) { + mdbx_error("%s, err %u", "try-exclusive", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Here could be one of two: + * - mdbx_lck_destroy() from the another process was hold the lock + * during a destruction. + * - either mdbx_lck_seize() from the another process was got the exclusive + * lock and doing initialization. + * For distinguish these cases will use size of the lck-file later. */ + + /* Wait for lck-shared now. */ + /* Here may be await during transient processes, for instance until another + * competing process doesn't call lck_downgrade(). */ + rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s, err %u", "try-shared", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + rc = check_fstat(env); + if (rc == MDBX_RESULT_TRUE) + goto retry; + if (rc != MDBX_SUCCESS) { + mdbx_error("%s, err %u", "lck_fstat", rc); + return rc; + } + + /* got shared, retry exclusive */ + rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); + if (rc == MDBX_SUCCESS) + goto continue_dxb_exclusive; + + if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || + rc == EDEADLK)) { + mdbx_error("%s, err %u", "try-exclusive", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Lock against another process operating in without-lck or exclusive mode. */ + rc = + lck_op(env->me_lazy_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s, err %u", "lock-against-without-lck", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Done: return with shared locking. */ + return MDBX_RESULT_FALSE; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; + + int rc = MDBX_SUCCESS; + if ((env->me_flags & MDBX_EXCLUSIVE) == 0) { + rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, 0, env->me_pid); + if (rc == MDBX_SUCCESS) + rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, env->me_pid + 1, + OFF_T_MAX - env->me_pid - 1); + } + if (rc == MDBX_SUCCESS) + rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1); + if (unlikely(rc != 0)) { + mdbx_error("%s, err %u", "lck", rc); + assert(MDBX_IS_ERROR(rc)); + } + return rc; +} + +MDBX_INTERNAL_FUNC int __cold mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor) { + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; + + int rc = MDBX_SUCCESS; + struct stat lck_info; + if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor && + env->me_lck && + /* try get exclusive access */ + lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 && + /* if LCK was not removed */ + fstat(env->me_lfd, &lck_info) == 0 && lck_info.st_nlink > 0 && + lck_op(env->me_lazy_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, + OFF_T_MAX) == 0) { + + mdbx_verbose("%p got exclusive, drown locks", (void *)env); +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + if (env->me_sysv_ipc.semid != -1) + rc = semctl(env->me_sysv_ipc.semid, 2, IPC_RMID) ? errno : 0; +#else + rc = mdbx_ipclock_destroy(&env->me_lck->mti_rlock); + if (rc == 0) + rc = mdbx_ipclock_destroy(&env->me_lck->mti_wlock); +#endif /* MDBX_LOCKING */ + + mdbx_assert(env, rc == 0); + if (rc == 0) { + const bool synced = env->me_lck_mmap.lck->mti_unsynced_pages == 0; + mdbx_munmap(&env->me_lck_mmap); + if (synced) + rc = ftruncate(env->me_lfd, 0) ? errno : 0; + } + + mdbx_jitter4testing(false); + } + + /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored + * after file was closed. + * + * 2) File locks would be released (by kernel) while the file-descriptors will + * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel, + * locks should be released here explicitly with properly order. */ + + /* close dxb and restore lock */ + if (env->me_dsync_fd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->me_dsync_fd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->me_dsync_fd = INVALID_HANDLE_VALUE; + } + if (env->me_lazy_fd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->me_lazy_fd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->me_lazy_fd = INVALID_HANDLE_VALUE; + if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { + /* restore file-lock */ + rc = lck_op( + inprocess_neighbor->me_lazy_fd, F_SETLKW, + (inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, + (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) + ? 0 + : inprocess_neighbor->me_pid, + (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1); + } + } + + /* close clk and restore locks */ + if (env->me_lfd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->me_lfd = INVALID_HANDLE_VALUE; + if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { + /* restore file-locks */ + rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1); + if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader) + rc = mdbx_rpid_set(inprocess_neighbor); + } + } + + if (inprocess_neighbor && rc != MDBX_SUCCESS) + inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR; + return rc; +} + +/*---------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int __cold mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag) { +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + int semid = -1; + /* don't initialize semaphores twice */ + (void)inprocess_neighbor; + if (global_uniqueness_flag == MDBX_RESULT_TRUE) { + struct stat st; + if (fstat(env->me_lazy_fd, &st)) + return errno; + sysv_retry_create: + semid = semget(env->me_sysv_ipc.key, 2, + IPC_CREAT | IPC_EXCL | + (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))); + if (unlikely(semid == -1)) { + int err = errno; + if (err != EEXIST) + return err; + + /* remove and re-create semaphore set */ + semid = semget(env->me_sysv_ipc.key, 2, 0); + if (semid == -1) { + err = errno; + if (err != ENOENT) + return err; + goto sysv_retry_create; + } + if (semctl(semid, 2, IPC_RMID)) { + err = errno; + if (err != EIDRM) + return err; + } + goto sysv_retry_create; + } + + unsigned short val_array[2] = {1, 1}; + if (semctl(semid, 2, SETALL, val_array)) + return errno; + } else { + semid = semget(env->me_sysv_ipc.key, 2, 0); + if (semid == -1) + return errno; + + /* check read & write access */ + struct semid_ds data[2]; + if (semctl(semid, 2, IPC_STAT, data) || semctl(semid, 2, IPC_SET, data)) + return errno; + } + + env->me_sysv_ipc.semid = semid; + return MDBX_SUCCESS; + +#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX + (void)inprocess_neighbor; + if (global_uniqueness_flag != MDBX_RESULT_TRUE) + return MDBX_SUCCESS; +#error "FIXME: Not implemented" +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + + /* don't initialize semaphores twice */ + (void)inprocess_neighbor; + if (global_uniqueness_flag == MDBX_RESULT_TRUE) { + if (sem_init(&env->me_lck->mti_rlock, true, 1)) + return errno; + if (sem_init(&env->me_lck->mti_wlock, true, 1)) + return errno; + } + return MDBX_SUCCESS; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + if (inprocess_neighbor) + return MDBX_SUCCESS /* don't need any initialization for mutexes + if LCK already opened/used inside current process */ + ; + + /* FIXME: Unfortunately, there is no other reliable way but to long testing + * on each platform. On the other hand, behavior like FreeBSD is incorrect + * and we can expect it to be rare. Moreover, even on FreeBSD without + * additional in-process initialization, the probability of an problem + * occurring is vanishingly small, and the symptom is a return of EINVAL + * while locking a mutex. In other words, in the worst case, the problem + * results in an EINVAL error at the start of the transaction, but NOT data + * loss, nor database corruption, nor other fatal troubles. Thus, the code + * below I am inclined to think the workaround for erroneous platforms (like + * FreeBSD), rather than a defect of libmdbx. */ +#if defined(__FreeBSD__) + /* seems that shared mutexes on FreeBSD required in-process initialization */ + (void)global_uniqueness_flag; +#else + /* shared mutexes on many other platforms (including Darwin and Linux's + * futexes) doesn't need any addition in-process initialization */ + if (global_uniqueness_flag != MDBX_RESULT_TRUE) + return MDBX_SUCCESS; +#endif + + pthread_mutexattr_t ma; + int rc = pthread_mutexattr_init(&ma); + if (rc) + return rc; + + rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED); + if (rc) + goto bailout; + +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust) + rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); +#elif defined(PTHREAD_MUTEX_ROBUST_NP) || \ + defined(pthread_mutexattr_setrobust_np) + rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); +#elif _POSIX_THREAD_PROCESS_SHARED < 200809L + rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); +#else + rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); +#endif + if (rc) + goto bailout; +#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ + +#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && \ + !defined(MDBX_SAFE4QEMU) + rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT); + if (rc == ENOTSUP) + rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE); + if (rc && rc != ENOTSUP) + goto bailout; +#endif /* PTHREAD_PRIO_INHERIT */ + + rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK); + if (rc && rc != ENOTSUP) + goto bailout; + + rc = pthread_mutex_init(&env->me_lck->mti_rlock, &ma); + if (rc) + goto bailout; + rc = pthread_mutex_init(&env->me_lck->mti_wlock, &ma); + +bailout: + pthread_mutexattr_destroy(&ma); + return rc; +#else +#error "FIXME" +#endif /* MDBX_LOCKING > 0 */ +} + +static int __cold mdbx_ipclock_failed(MDBX_env *env, mdbx_ipclock_t *ipc, + const int err) { + int rc = err; +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV + if (err == EOWNERDEAD) { + /* We own the mutex. Clean up after dead previous owner. */ + + const bool rlocked = (env->me_lck && ipc == &env->me_lck->mti_rlock); + rc = MDBX_SUCCESS; + if (!rlocked) { + if (unlikely(env->me_txn)) { + /* env is hosed if the dead thread was ours */ + env->me_flags |= MDBX_FATAL_ERROR; + env->me_txn = NULL; + rc = MDBX_PANIC; + } + } + mdbx_warning("%clock owner died, %s", (rlocked ? 'r' : 'w'), + (rc ? "this process' env is hosed" : "recovering")); + + int check_rc = mdbx_cleanup_dead_readers(env, rlocked, NULL); + check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc; + +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + rc = (rc == MDBX_SUCCESS) ? check_rc : rc; +#else +#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutex_consistent) + int mreco_rc = pthread_mutex_consistent(ipc); +#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutex_consistent_np) + int mreco_rc = pthread_mutex_consistent_np(ipc); +#elif _POSIX_THREAD_PROCESS_SHARED < 200809L + int mreco_rc = pthread_mutex_consistent_np(ipc); +#else + int mreco_rc = pthread_mutex_consistent(ipc); +#endif + check_rc = (mreco_rc == 0) ? check_rc : mreco_rc; + + if (unlikely(mreco_rc)) + mdbx_error("lock recovery failed, %s", mdbx_strerror(mreco_rc)); + + rc = (rc == MDBX_SUCCESS) ? check_rc : rc; + if (MDBX_IS_ERROR(rc)) + pthread_mutex_unlock(ipc); +#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ + return rc; + } +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 + (void)ipc; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + (void)ipc; +#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX +#ifdef _MSC_VER +#pragma message("warning: TODO") +#else +#warning "TODO" +#endif + (void)ipc; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + + mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err)); + if (rc != EDEADLK) + env->me_flags |= MDBX_FATAL_ERROR; + return rc; +} + +static int mdbx_ipclock_lock(MDBX_env *env, mdbx_ipclock_t *ipc, + const bool dont_wait) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + int rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc); + rc = (rc == EBUSY && dont_wait) ? MDBX_BUSY : rc; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + int rc = MDBX_SUCCESS; + if (dont_wait) { + if (sem_trywait(ipc)) { + rc = errno; + if (rc == EAGAIN) + rc = MDBX_BUSY; + } + } else if (sem_wait(ipc)) + rc = errno; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + struct sembuf op = {.sem_num = (ipc != env->me_wlock), + .sem_op = -1, + .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO}; + int rc; + if (semop(env->me_sysv_ipc.semid, &op, 1)) { + rc = errno; + if (dont_wait && rc == EAGAIN) + rc = MDBX_BUSY; + } else { + rc = *ipc ? EOWNERDEAD : MDBX_SUCCESS; + *ipc = env->me_pid; + } +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + + if (unlikely(rc != MDBX_SUCCESS && rc != MDBX_BUSY)) + rc = mdbx_ipclock_failed(env, ipc, rc); + return rc; +} + +static int mdbx_ipclock_unlock(MDBX_env *env, mdbx_ipclock_t *ipc) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + int rc = pthread_mutex_unlock(ipc); + (void)env; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + int rc = sem_post(ipc) ? errno : MDBX_SUCCESS; + (void)env; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + if (unlikely(*ipc != (pid_t)env->me_pid)) + return EPERM; + *ipc = 0; + struct sembuf op = { + .sem_num = (ipc != env->me_wlock), .sem_op = 1, .sem_flg = SEM_UNDO}; + int rc = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) { + mdbx_trace("%s", ">>"); + mdbx_jitter4testing(true); + int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_rlock, false); + mdbx_trace("<< rc %d", rc); + return rc; +} + +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) { + mdbx_trace("%s", ">>"); + int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_rlock); + mdbx_trace("<< rc %d", rc); + if (unlikely(rc != MDBX_SUCCESS)) + mdbx_panic("%s() failed: err %d\n", __func__, rc); + mdbx_jitter4testing(true); +} + +int mdbx_txn_lock(MDBX_env *env, bool dont_wait) { + mdbx_trace("%swait %s", dont_wait ? "dont-" : "", ">>"); + mdbx_jitter4testing(true); + int rc = mdbx_ipclock_lock(env, env->me_wlock, dont_wait); + mdbx_trace("<< rc %d", rc); + return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS; +} + +void mdbx_txn_unlock(MDBX_env *env) { + mdbx_trace("%s", ">>"); + int rc = mdbx_ipclock_unlock(env, env->me_wlock); + mdbx_trace("<< rc %d", rc); + if (unlikely(rc != MDBX_SUCCESS)) + mdbx_panic("%s() failed: err %d\n", __func__, rc); + mdbx_jitter4testing(true); +} + +#else +#ifdef _MSC_VER +#pragma warning(disable : 4206) /* nonstandard extension used: translation \ + unit is empty */ +#endif /* _MSC_VER (warnings) */ +#endif /* !Windows LCK-implementation */ diff --git a/libs/libmdbx/src/mdbx.c++ b/libs/libmdbx/src/mdbx.c++ new file mode 100644 index 0000000000..ef4cb789fb --- /dev/null +++ b/libs/libmdbx/src/mdbx.c++ @@ -0,0 +1,4524 @@ +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#define MDBX_ALLOY 1 +#define MDBX_BUILD_SOURCERY 3b5677a6062b714f1e138b0066c5590ee3c9ebf3bf8cfa3bb9503515ea0d1f02_v0_9_1_18_g1d31ebdc1c +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_FULL_VER < 190024234 + /* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual Studio 2015 Update 3). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for alignment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant aggregate initializer */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ + +#if defined(MDBX_TOOLS) +#undef MDBX_DEPRECATED +#define MDBX_DEPRECATED +#endif /* MDBX_TOOLS */ + +#include "mdbx.h++" +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# else +# define __noinline +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(7, 0) && \ + (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +# define __fallthrough __attribute__((__fallthrough__)) +# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ + __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define __fallthrough [[clang::fallthrough]] +# else +# define __fallthrough +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline __maybe_unused const void* MDBX_PURE_FUNCTION + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include <valgrind/memcheck.h> +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include <sanitizer/asan_interface.h> +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template <typename T, size_t N> + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_STATIC_ASSERT) +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif defined(_MSC_VER) +# include <crtdbg.h> +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wnested-anon-types") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wnested-anon-types" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wnested-anon-types" +# else +# pragma warning disable "nested-anon-types" +# endif +#endif /* -Wnested-anon-types */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) && MDBX_AVOID_CRT +#define _NO_CRT_STDIO_INLINE +#endif +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include <assert.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +/* C11 stdalign.h */ +#if __has_include(<stdalign.h>) +#include <stdalign.h> +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif /* Apple OSX & iOS */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) +#include <sys/cdefs.h> +#include <sys/mount.h> +#include <sys/sysctl.h> +#include <sys/types.h> +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <vm/vm_param.h> +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include <uvm/uvm_param.h> +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#ifndef __MACH__ +#include <sys/vmmeter.h> +#endif +#else +#include <malloc.h> +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(_WIN32) || defined(_WIN64)) +#include <mntent.h> +#endif /* !Solaris */ +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) +#include <malloc_np.h> +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) +#include <malloc/malloc.h> +#endif /* MacOS */ + +#if defined(__MACH__) +#include <mach/host_info.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> +#include <uuid/uuid.h> +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include <linux/sysctl.h> +#include <sched.h> +#include <sys/sendfile.h> +#include <sys/statfs.h> +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include <utmpx.h> +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include <kstat.h> +#include <sys/mnttab.h> +/* On Solaris, it's easier to add a missing prototype rather than find a + * combination of #defines that break nothing. */ +__extern_C key_t ftok(const char *, int); +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <tlhelp32.h> +#include <windows.h> +#include <winnt.h> +#include <winternl.h> +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event[2]; +} mdbx_condpair_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <sys/file.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond[2]; +} mdbx_condpair_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +#ifdef __ANDROID_API__ +#include <android/log.h> +#if __ANDROID_API__ >= 21 +#include <sys/sendfile.h> +#endif +#endif /* Android */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) +#include <sys/stat.h> +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) +#include <sys/types.h> +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) +#include <sys/file.h> +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include <intrin.h> +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif /* __ia32__ */ +#if defined(__ia32__) +#include <cpuid.h> +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include <mbarrier.h> +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include <machine/sys/inline.h> +#elif defined(__IBMC__) && defined(__powerpc) +#include <atomic.h> +#elif defined(_AIX) +#include <builtins.h> +#include <sys/atomic_op.h> +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include <c_asm.h> +#include <machine/builtins.h> +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include <machine/inline.h> +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline size_t +mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; +#endif /* Windows */ + +#ifdef __cplusplus +extern void mdbx_osal_jitter(bool tiny); +#else +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +#if __has_include(<sys/cachectl.h>) +#include <sys/cachectl.h> +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include <sys/cachectl.h> +#endif + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC MDBX_PRINTF_ARGS(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +MDBX_INTERNAL_VAR bool + mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; +#endif /* Linux */ + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_NONE = 0, + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + const enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); + +enum mdbx_openfile_purpose { + MDBX_OPEN_DXB_READ = 0, + MDBX_OPEN_DXB_LAZY = 1, + MDBX_OPEN_DXB_DSYNC = 2, + MDBX_OPEN_LCK = 3, + MDBX_OPEN_COPY = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +#define MMAP_OPTION_TRUNCATE 1 +#define MMAP_OPTION_SEMAPHORE 2 +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const unsigned options); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna, const bool may_move); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { + mdbx_tid_t thunk; + STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); +#if defined(_WIN32) || defined(_WIN64) + thunk = GetCurrentThreadId(); +#else + thunk = pthread_self(); +#endif + return (uintptr_t)thunk; +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by argument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (especially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); +MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; + +static __inline bool mdbx_RunningUnderWine(void) { + return !mdbx_NtExtendSection; +} + +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, + LPCSTR lpValue, DWORD dwFlags, + LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData); +MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include <stdatomic.h> +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include <libkern/OSAtomic.h> +#else +#error FIXME atomic-ops +#endif + +#endif /* !__cplusplus */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/******************************************************************************* + ******************************************************************************* + ******************************************************************************* + * + * + * #### ##### ##### # #### # # #### + * # # # # # # # # ## # # + * # # # # # # # # # # # #### + * # # ##### # # # # # # # # + * # # # # # # # # ## # # + * #### # # # #### # # #### + * + * + */ + +/** \defgroup build_option Build options + * The libmdbx build options. + @{ */ + +#ifdef DOXYGEN +/* !!! Actually this is a fake definitions !!! + * !!! for documentation generation by Doxygen !!! */ + +/** Controls enabling of debugging features. + * + * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all, + * including logging and assertion controls. + * Logging level and corresponding debug flags changing + * by \ref mdbx_setup_debug() will not have effect. + * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging, + * assertions checking and internal audit). + * Simultaneously sets the default logging level + * to the `MDBX_DEBUG` value. + * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. + * + * \ingroup build_option */ +#define MDBX_DEBUG 0...7 + +/** Disables using of GNU libc extensions. */ +#define MDBX_DISABLE_GNU_SOURCE 0 or 1 + +#endif /* DOXYGEN */ + +/** Enables support for huge write-transactions */ +#ifndef MDBX_HUGE_TRANSACTIONS +#define MDBX_HUGE_TRANSACTIONS 0 +#endif /* MDBX_HUGE_TRANSACTIONS */ + +/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_DURABILITY 0 +/** Using fsync() with chance of data lost on power failure */ +#define MDBX_OSX_WANNA_SPEED 1 + +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED + * for OSX & iOS */ +#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ + +/** Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_ENV_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be omitted: + * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_ENV_CHECKPID 0 +#else +#define MDBX_ENV_CHECKPID 1 +#endif +#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_ENV_CHECKPID) +#else +#define MDBX_ENV_CHECKPID_CONFIG STRINGIFY(MDBX_ENV_CHECKPID) +#endif /* MDBX_ENV_CHECKPID */ + +/** Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +/** Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +//------------------------------------------------------------------------------ + +/** Win32 File Locking API for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_WIN32FILES -1 + +/** SystemV IPC semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_SYSV 5 + +/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX1988 1988 + +/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2001 2001 + +/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2008 2008 + +/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_BENAPHORE 1995 + +/** Advanced: Choices the locking implementation (autodetection by default). */ +#if defined(_WIN32) || defined(_WIN64) +#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES +#else +#ifndef MDBX_LOCKING +#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ + _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ + _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ + _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 +#else +#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 +#endif +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 +#else +#define MDBX_LOCKING MDBX_LOCKING_SYSV +#endif +#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) +#else +#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) +#endif /* MDBX_LOCKING */ +#endif /* !Windows */ + +/** Advanced: Using POSIX OFD-locks (autodetection by default). */ +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) && \ + !defined(__sun) /* OFD-lock are broken on Solaris */ +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/** Advanced: Using sendfile() syscall (autodetection by default). */ +#ifndef MDBX_USE_SENDFILE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) +#define MDBX_USE_SENDFILE 1 +#else +#define MDBX_USE_SENDFILE 0 +#endif +#endif /* MDBX_USE_SENDFILE */ + +/** Advanced: Using copy_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_COPYFILERANGE +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) +#define MDBX_USE_COPYFILERANGE 1 +#else +#define MDBX_USE_COPYFILERANGE 0 +#endif +#endif /* MDBX_USE_COPYFILERANGE */ + +/** Advanced: Using sync_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_SYNCFILERANGE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) +#define MDBX_USE_SYNCFILERANGE 1 +#else +#define MDBX_USE_SYNCFILERANGE 0 +#endif +#endif /* MDBX_USE_SYNCFILERANGE */ + +//------------------------------------------------------------------------------ + +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) || defined(DOXYGEN) +#define MDBX_CPU_WRITEBACK_INCOHERENT 0 +#else +#define MDBX_CPU_WRITEBACK_INCOHERENT 1 +#endif +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#ifdef __OpenBSD__ +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 +#else +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +/** @} end of build options */ +/******************************************************************************* + ******************************************************************************* + ******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) +#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) +#define INVALID_TXNID UINT64_MAX +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + MDBX_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + bin128_t mm_bootid; + +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + __anonymous_struct_extension__ struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + indx_t mp_ptrs[] /* dynamic size */; +#endif /* C99 */ +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES +#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) +typedef void mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t mdbx_ipclock_t; +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#define MDBX_CLOCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) +typedef sem_t mdbx_ipclock_t; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); +#endif /* MDBX_LOCKING */ + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Write transaction lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_wlock; +#endif /* MDBX_LOCKING > 0 */ + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Readeaders registration lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_rlock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[] /* dynamic size */; +#endif /* C99 */ +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + __anonymous_struct_extension__ struct { + pgno_t pgno; + MDBX_page *ptr; + }; + __anonymous_struct_extension__ struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#if MDBX_HUGE_TRANSACTIONS +#define MDBX_PNL_MAX \ + ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) +#else +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) +#endif /* MDBX_HUGE_TRANSACTIONS */ + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ + size_t md_klen_min, md_klen_max; /* min/max key length for the database */ + size_t md_vlen_min, + md_vlen_max; /* min/max value/data length for the database */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) +#define MDBX_TXN_RW_BEGIN_FLAGS \ + (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + +#define TXN_FLAGS \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ + MDBX_TXN_HAS_CHILD) + +#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ + ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ + MDBX_SHRINK_ALLOWED) +#error "Oops, some flags overlapped or wrong" +#endif + + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + + /* Transaction DBI Flags */ +#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ +#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ +#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ +#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ +#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ +#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* Array of flags for each DB */ + uint8_t *mt_dbistate; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + MDBX_canary mt_canary; + void *mt_userctx; /* User-settable context */ + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbistate for this database */ + uint8_t *mc_dbistate; + unsigned mc_snum; /* number of pushed pages */ + unsigned mc_top; /* index of top page, normally mc_snum-1 */ + + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + + /* Cursor checking flags. */ +#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ +#define C_UPDATING 0x200 /* update/rebalance pending */ +#define C_RETIRING 0x400 /* refs to child pages may be invalid */ +#define C_SKIPORD 0x800 /* don't check keys ordering */ + + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbistate for this Dup DB */ + uint8_t mx_dbistate; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + void *mc_userctx; /* User-settable context */ + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + uint32_t me_signature; + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + /* Legacy MDBX_MAPASYNC (prior v0.9) */ +#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) +#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) + uint32_t me_flags; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_lazy_fd me_dxb_mmap.fd + mdbx_filehandle_t me_dsync_fd; + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + unsigned me_psize; /* DB page size, inited from me_os_psize */ + uint8_t me_psize2log; /* log2 of DB page size */ + int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ + + /* write-txn lock */ +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING > 0 + mdbx_ipclock_t *me_wlock; +#endif /* MDBX_LOCKING > 0 */ + + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + unsigned me_branch_nodemax; /* max size of a branch-node */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ + struct { +#if MDBX_LOCKING > 0 + mdbx_ipclock_t wlock; +#endif /* MDBX_LOCKING > 0 */ + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +#ifndef __cplusplus +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +extern uint8_t mdbx_runtime_flags; +extern uint8_t mdbx_loglevel; +extern MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) + mdbx_debug_log(int level, const char *function, int line, const char *fmt, + ...) MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function, + int line, const char *fmt, + va_list args); + +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#endif /* MDBX_DEBUG */ + +#if !MDBX_DEBUG && defined(__ANDROID_API__) +#define mdbx_assert_fail(env, msg, func, line) \ + __android_log_assert(msg, "mdbx", "%s:%u", func, line) +#else +void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, + int line); +#endif + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ + +#if MDBX_CPU_WRITEBACK_INCOHERENT +#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() +#else +#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +static __maybe_unused __inline void +mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = + (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + mdbx_assert(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#if MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +#endif /* !__cplusplus */ + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) + +/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + }; +#endif /* __BYTE_ORDER__ */ + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + uint8_t mn_data[] /* key and data are appended here */; +#endif /* C99 */ +} MDBX_node; + +#define DB_PERSISTENT_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP) + +/* mdbx_dbi_open() flags */ +#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) + +#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define DB_INTERNAL_FLAGS DB_VALID + +#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif +#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused bool +is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define ENV_CHANGEABLE_FLAGS \ + (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ + MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) +#define ENV_CHANGELESS_FLAGS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) +#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) + +#if !defined(__cplusplus) || defined(__cpp_constexpr) +static __maybe_unused void static_checks(void) { + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, + "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & + (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, + "Oops, some flags overlapped or wrong"); +} +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +#ifdef __cplusplus +} +#endif +// +// Copyright (c) 2020, Leonid Yuriev <leo@yuriev.ru>. +// SPDX-License-Identifier: Apache-2.0 +// +// Non-inline part of the libmdbx C++ API (preliminary) +// + +#ifdef _MSC_VER +#define _CRT_SECURE_NO_WARNINGS +#endif + + + +#include <atomic> +#include <cctype> // for isxdigit(), etc +#include <system_error> + +namespace { + +#if 0 /* Unused for now */ + +class trouble_location { + +#ifndef TROUBLE_PROVIDE_LINENO +#define TROUBLE_PROVIDE_LINENO 1 +#endif + +#ifndef TROUBLE_PROVIDE_CONDITION +#define TROUBLE_PROVIDE_CONDITION 1 +#endif + +#ifndef TROUBLE_PROVIDE_FUNCTION +#define TROUBLE_PROVIDE_FUNCTION 1 +#endif + +#ifndef TROUBLE_PROVIDE_FILENAME +#define TROUBLE_PROVIDE_FILENAME 1 +#endif + +#if TROUBLE_PROVIDE_LINENO + const unsigned line_; +#endif +#if TROUBLE_PROVIDE_CONDITION + const char *const condition_; +#endif +#if TROUBLE_PROVIDE_FUNCTION + const char *const function_; +#endif +#if TROUBLE_PROVIDE_FILENAME + const char *const filename_; +#endif + +public: + MDBX_CXX11_CONSTEXPR trouble_location(unsigned line, const char *condition, + const char *function, const char *filename) + : +#if TROUBLE_PROVIDE_LINENO + line_(line) +#endif +#if TROUBLE_PROVIDE_CONDITION + , + condition_(condition) +#endif +#if TROUBLE_PROVIDE_FUNCTION + , + function_(function) +#endif +#if TROUBLE_PROVIDE_FILENAME + , + filename_(filename) +#endif + { +#if !TROUBLE_PROVIDE_LINENO + (void)line; +#endif +#if !TROUBLE_PROVIDE_CONDITION + (void)condition; +#endif +#if !TROUBLE_PROVIDE_FUNCTION + (void)function; +#endif +#if !TROUBLE_PROVIDE_FILENAME + (void)filename; +#endif + } + + trouble_location(const trouble_location &&) = delete; + + unsigned line() const { +#if TROUBLE_PROVIDE_LINENO + return line_; +#else + return 0; +#endif + } + + const char *condition() const { +#if TROUBLE_PROVIDE_CONDITION + return condition_; +#else + return ""; +#endif + } + + const char *function() const { +#if TROUBLE_PROVIDE_FUNCTION + return function_; +#else + return ""; +#endif + } + + const char *filename() const { +#if TROUBLE_PROVIDE_FILENAME + return filename_; +#else + return ""; +#endif + } +}; + +//------------------------------------------------------------------------------ + +__cold std::string format_va(const char *fmt, va_list ap) { + va_list ones; + va_copy(ones, ap); +#ifdef _MSC_VER + int needed = _vscprintf(fmt, ap); +#else + int needed = vsnprintf(nullptr, 0, fmt, ap); +#endif + assert(needed >= 0); + std::string result; + result.reserve(size_t(needed + 1)); + result.resize(size_t(needed), '\0'); + assert(int(result.capacity()) > needed); + int actual = vsnprintf(const_cast<char *>(result.data()), result.capacity(), + fmt, ones); + assert(actual == needed); + (void)actual; + va_end(ones); + return result; +} + +__cold std::string format(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + std::string result = format_va(fmt, ap); + va_end(ap); + return result; +} + +class bug : public std::runtime_error { + const trouble_location &location_; + +public: + bug(const trouble_location &) noexcept; + /* temporary workaround for "private field 'FOO' is not used" from CLANG + * and for "function 'BAR' was declared but never referenced" from LCC. */ +#ifndef __LCC__ + const trouble_location &location() const noexcept { return location_; } +#endif + virtual ~bug() noexcept; +}; + +__cold bug::bug(const trouble_location &location) noexcept + : std::runtime_error(format("mdbx.bug: %s.%s at %s:%u", location.function(), + location.condition(), location.filename(), + location.line())), + location_(location) {} + +__cold bug::~bug() noexcept {} + +[[noreturn]] __cold void raise_bug(const trouble_location &what_and_where) { + throw bug(what_and_where); +} + +#define RAISE_BUG(line, condition, function, file) \ + do { \ + static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, \ + function, file); \ + raise_bug(bug); \ + } while (0) + +#define ENSURE(condition) \ + do \ + if (MDBX_UNLIKELY(!(condition))) \ + RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \ + while (0) + +#define NOT_IMPLEMENTED() \ + RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__); + +#endif /* Unused*/ + +//------------------------------------------------------------------------------ + +template <typename PATH> struct path_to_pchar { + const std::string str; + path_to_pchar(const PATH &path) : str(path.generic_string()) {} + operator const char *() const { return str.c_str(); } +}; + +template <typename PATH> PATH pchar_to_path(const char *c_str) { + return PATH(c_str); +} + +template <> struct path_to_pchar<std::string> { + const char *const ptr; + path_to_pchar(const std::string &path) : ptr(path.c_str()) {} + operator const char *() const { return ptr; } +}; + +#if defined(_WIN32) || defined(_WIN64) + +template <> struct path_to_pchar<std::wstring> { + std::string str; + path_to_pchar(const std::wstring &path) { + if (!path.empty()) { + const int chars = + WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, path.data(), + int(path.size()), nullptr, 0, nullptr, nullptr); + if (chars == 0) + mdbx::error::throw_exception(GetLastError()); + str.append(chars, '\0'); + WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, path.data(), + int(path.size()), const_cast<char *>(str.data()), + chars, nullptr, nullptr); + } + } + operator const char *() const { return str.c_str(); } +}; + +template <> std::wstring pchar_to_path<std::wstring>(const char *c_str) { + std::wstring wstr; + if (c_str && *c_str) { + const int chars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, c_str, + int(strlen(c_str)), nullptr, 0); + if (chars == 0) + mdbx::error::throw_exception(GetLastError()); + wstr.append(chars, '\0'); + MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, c_str, + int(strlen(c_str)), const_cast<wchar_t *>(wstr.data()), + chars); + } + return wstr; +} + +#endif /* Windows */ + +} // namespace + +//------------------------------------------------------------------------------ + +namespace mdbx { + +[[noreturn]] __cold void throw_max_length_exceeded() { + throw std::length_error( + "mdbx:: exceeded the maximal length of data/slice/buffer"); +} + +[[noreturn]] __cold void throw_too_small_target_buffer() { + throw std::length_error("mdbx:: the target buffer is too small"); +} + +[[noreturn]] __cold void throw_out_range() { + throw std::out_of_range("mdbx:: slice or buffer method was called with " + "an argument that exceeds the length"); +} + +__cold exception::exception(const ::mdbx::error &error) noexcept + : base(error.what()), error_(error) {} + +__cold exception::~exception() noexcept {} + +static std::atomic_int fatal_countdown; + +__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) { + ++fatal_countdown; +} + +__cold fatal::~fatal() noexcept { + if (--fatal_countdown == 0) + std::terminate(); +} + +#define DEFINE_EXCEPTION(NAME) \ + __cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \ + __cold NAME::~NAME() noexcept {} + +DEFINE_EXCEPTION(bad_map_id) +DEFINE_EXCEPTION(bad_transaction) +DEFINE_EXCEPTION(bad_value_size) +DEFINE_EXCEPTION(db_corrupted) +DEFINE_EXCEPTION(db_full) +DEFINE_EXCEPTION(db_invalid) +DEFINE_EXCEPTION(db_too_large) +DEFINE_EXCEPTION(db_unable_extend) +DEFINE_EXCEPTION(db_version_mismatch) +DEFINE_EXCEPTION(db_wanna_write_for_recovery) +DEFINE_EXCEPTION(incompatible_operation) +DEFINE_EXCEPTION(internal_page_full) +DEFINE_EXCEPTION(internal_problem) +DEFINE_EXCEPTION(key_exists) +DEFINE_EXCEPTION(key_mismatch) +DEFINE_EXCEPTION(max_maps_reached) +DEFINE_EXCEPTION(max_readers_reached) +DEFINE_EXCEPTION(multivalue) +DEFINE_EXCEPTION(no_data) +DEFINE_EXCEPTION(not_found) +DEFINE_EXCEPTION(operation_not_permitted) +DEFINE_EXCEPTION(permission_denied_or_not_writeable) +DEFINE_EXCEPTION(reader_slot_busy) +DEFINE_EXCEPTION(remote_media) +DEFINE_EXCEPTION(something_busy) +DEFINE_EXCEPTION(thread_mismatch) +DEFINE_EXCEPTION(transaction_full) +DEFINE_EXCEPTION(transaction_overlapping) + +#undef DEFINE_EXCEPTION + +__cold const char *error::what() const noexcept { + if (is_mdbx_error()) + return mdbx_liberr2str(code()); + + switch (code()) { +#define ERROR_CASE(CODE) \ + case CODE: \ + return STRINGIFY(CODE) + ERROR_CASE(MDBX_ENODATA); + ERROR_CASE(MDBX_EINVAL); + ERROR_CASE(MDBX_EACCESS); + ERROR_CASE(MDBX_ENOMEM); + ERROR_CASE(MDBX_EROFS); + ERROR_CASE(MDBX_ENOSYS); + ERROR_CASE(MDBX_EIO); + ERROR_CASE(MDBX_EPERM); + ERROR_CASE(MDBX_EINTR); + ERROR_CASE(MDBX_ENOFILE); + ERROR_CASE(MDBX_EREMOTE); +#undef ERROR_CASE + default: + return "SYSTEM"; + } +} + +__cold std::string error::message() const { + char buf[1024]; + const char *msg = ::mdbx_strerror_r(code(), buf, sizeof(buf)); + return std::string(msg ? msg : "unknown"); +} + +[[noreturn]] __cold void error::panic(const char *context, + const char *func) const noexcept { + assert(code() != MDBX_SUCCESS); + ::mdbx_panic("mdbx::%s.%s(): \"%s\" (%d)", context, func, what(), code()); + std::terminate(); +} + +__cold void error::throw_exception() const { + switch (code()) { + case MDBX_EINVAL: + throw std::invalid_argument("mdbx"); + case MDBX_ENOMEM: + throw std::bad_alloc(); + case MDBX_SUCCESS: + static_assert(MDBX_SUCCESS == MDBX_RESULT_FALSE, "WTF?"); + throw std::logic_error("MDBX_SUCCESS (MDBX_RESULT_FALSE)"); + case MDBX_RESULT_TRUE: + throw std::logic_error("MDBX_RESULT_TRUE"); +#define CASE_EXCEPTION(NAME, CODE) \ + case CODE: \ + throw NAME(code()) + CASE_EXCEPTION(bad_map_id, MDBX_BAD_DBI); + CASE_EXCEPTION(bad_transaction, MDBX_BAD_TXN); + CASE_EXCEPTION(bad_value_size, MDBX_BAD_VALSIZE); + CASE_EXCEPTION(db_corrupted, MDBX_CORRUPTED); + CASE_EXCEPTION(db_corrupted, MDBX_CURSOR_FULL); /* branch-pages loop */ + CASE_EXCEPTION(db_corrupted, MDBX_PAGE_NOTFOUND); + CASE_EXCEPTION(db_full, MDBX_MAP_FULL); + CASE_EXCEPTION(db_invalid, MDBX_INVALID); + CASE_EXCEPTION(db_too_large, MDBX_TOO_LARGE); + CASE_EXCEPTION(db_unable_extend, MDBX_UNABLE_EXTEND_MAPSIZE); + CASE_EXCEPTION(db_version_mismatch, MDBX_VERSION_MISMATCH); + CASE_EXCEPTION(db_wanna_write_for_recovery, MDBX_WANNA_RECOVERY); + CASE_EXCEPTION(fatal, MDBX_EBADSIGN); + CASE_EXCEPTION(fatal, MDBX_PANIC); + CASE_EXCEPTION(incompatible_operation, MDBX_INCOMPATIBLE); + CASE_EXCEPTION(internal_page_full, MDBX_PAGE_FULL); + CASE_EXCEPTION(internal_problem, MDBX_PROBLEM); + CASE_EXCEPTION(key_mismatch, MDBX_EKEYMISMATCH); + CASE_EXCEPTION(max_maps_reached, MDBX_DBS_FULL); + CASE_EXCEPTION(max_readers_reached, MDBX_READERS_FULL); + CASE_EXCEPTION(multivalue, MDBX_EMULTIVAL); + CASE_EXCEPTION(no_data, MDBX_ENODATA); + CASE_EXCEPTION(not_found, MDBX_NOTFOUND); + CASE_EXCEPTION(operation_not_permitted, MDBX_EPERM); + CASE_EXCEPTION(permission_denied_or_not_writeable, MDBX_EACCESS); + CASE_EXCEPTION(reader_slot_busy, MDBX_BAD_RSLOT); + CASE_EXCEPTION(remote_media, MDBX_EREMOTE); + CASE_EXCEPTION(something_busy, MDBX_BUSY); + CASE_EXCEPTION(thread_mismatch, MDBX_THREAD_MISMATCH); + CASE_EXCEPTION(transaction_full, MDBX_TXN_FULL); + CASE_EXCEPTION(transaction_overlapping, MDBX_TXN_OVERLAPPING); +#undef CASE_EXCEPTION + default: + if (is_mdbx_error()) + throw exception(*this); + throw std::system_error(std::error_code(code(), std::system_category())); + } +} + +//------------------------------------------------------------------------------ + +bool slice::is_printable(bool disable_utf8) const noexcept { + enum : byte { + LS = 5, // shift for UTF8 sequence length + P_ = 1 << (LS - 1), // printable ASCII flag + N_ = 0, // non-printable ASCII + second_range_mask = P_ - 1, // mask for range flag + r80_BF = P_ | 0, // flag for UTF8 2nd byte range + rA0_BF = P_ | 1, // flag for UTF8 2nd byte range + r80_9F = P_ | 2, // flag for UTF8 2nd byte range + r90_BF = P_ | 3, // flag for UTF8 2nd byte range + r80_8F = P_ | 4, // flag for UTF8 2nd byte range + + // valid utf-8 byte sequences + // http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 + // Code | Bytes | | | + // Points | 1st | 2nd | 3rd |4th + // --------------------|--------|--------|--------|--- + C2 = 2 << LS | r80_BF, // U+000080..U+0007FF | C2..DF | 80..BF | | + E0 = 3 << LS | rA0_BF, // U+000800..U+000FFF | E0 | A0..BF | 80..BF | + E1 = 3 << LS | r80_BF, // U+001000..U+00CFFF | E1..EC | 80..BF | 80..BF | + ED = 3 << LS | r80_9F, // U+00D000..U+00D7FF | ED | 80..9F | 80..BF | + EE = 3 << LS | r80_BF, // U+00E000..U+00FFFF | EE..EF | 80..BF | 80..BF | + F0 = 4 << LS | r90_BF, // U+010000..U+03FFFF | F0 | 90..BF | 80..BF |... + F1 = 4 << LS | r80_BF, // U+040000..U+0FFFFF | F1..F3 | 80..BF | 80..BF |... + F4 = 4 << LS | r80_BF, // U+100000..U+10FFFF | F4 | 80..8F | 80..BF |... + }; + + static const byte range_from[] = {0x80, 0xA0, 0x80, 0x90, 0x80}; + static const byte range_to[] = {0xBF, 0xBF, 0x9F, 0xBF, 0x8F}; + + static const byte map[256] = { + // 1 2 3 4 5 6 7 8 9 a b c d e f + N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, // 00 + N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, N_, // 10 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 20 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 30 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 40 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 50 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // 60 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, // 70 + N_, N_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, P_, N_, // 80 + N_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, N_, P_, P_, // 90 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // a0 + P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, // b0 + P_, P_, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, // c0 + C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, // df + E0, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, E1, ED, EE, EE, // e0 + F0, F1, F1, F1, F4, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_, P_ // f0 + }; + + if (length() < 1) + return false; + + auto src = byte_ptr(); + const auto end = src + length(); + if (MDBX_UNLIKELY(disable_utf8)) { + do + if (MDBX_UNLIKELY((P_ & map[*src]) == 0)) + return false; + while (++src < end); + return true; + } + + do { + const auto bits = map[*src]; + const auto second_from = range_from[bits & second_range_mask]; + const auto second_to = range_to[bits & second_range_mask]; + switch (bits >> LS) { + default: + return false; + case 1: + src += 1; + continue; + case 2: + if (unlikely(src + 1 >= end)) + return false; + if (unlikely(src[1] < second_from || src[1] > second_to)) + return false; + src += 2; + continue; + case 3: + if (unlikely(src + 3 >= end)) + return false; + if (unlikely(src[1] < second_from || src[1] > second_to)) + return false; + if (unlikely(src[2] < 0x80 || src[2] > 0xBF)) + return false; + src += 3; + continue; + case 4: + if (unlikely(src + 4 >= end)) + return false; + if (unlikely(src[1] < second_from || src[1] > second_to)) + return false; + if (unlikely(src[2] < 0x80 || src[2] > 0xBF)) + return false; + if (unlikely(src[3] < 0x80 || src[3] > 0xBF)) + return false; + src += 4; + continue; + } + } while (src < end); + + return true; +} + +//------------------------------------------------------------------------------ + +char *slice::to_hex(char *__restrict dest, size_t dest_size, bool uppercase, + unsigned wrap_width) const { + if (MDBX_UNLIKELY(to_hex_bytes(wrap_width) > dest_size)) + throw_too_small_target_buffer(); + + auto src = byte_ptr(); + const char alphabase = (uppercase ? 'A' : 'a') - 10; + auto line = dest; + for (const auto end = src + length(); src != end; ++src) { + const int8_t hi = *src >> 4; + const int8_t lo = *src & 15; + dest[0] = char(alphabase + hi + (((hi - 10) >> 7) & -7)); + dest[1] = char(alphabase + lo + (((lo - 10) >> 7) & -7)); + dest += 2; + if (wrap_width && size_t(dest - line) >= wrap_width) { + *dest = '\n'; + line = ++dest; + } + } + return dest; +} + +byte *slice::from_hex(byte *__restrict dest, size_t dest_size, + bool ignore_spaces) const { + if (MDBX_UNLIKELY(length() % 2 && !ignore_spaces)) + throw std::domain_error( + "mdbx::from_hex:: odd length of hexadecimal string"); + if (MDBX_UNLIKELY(from_hex_bytes() > dest_size)) + throw_too_small_target_buffer(); + + auto src = byte_ptr(); + for (auto left = length(); left > 0;) { + if (MDBX_UNLIKELY(*src <= ' ') && + MDBX_LIKELY(ignore_spaces && isspace(*src))) { + ++src; + --left; + continue; + } + + if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1]))) + throw std::domain_error("mdbx::from_hex:: invalid hexadecimal string"); + + int8_t hi = src[0]; + hi = (hi | 0x20) - 'a'; + hi += 10 + ((hi >> 7) & 7); + + int8_t lo = src[1]; + lo = (lo | 0x20) - 'a'; + lo += 10 + ((lo >> 7) & 7); + + *dest++ = hi << 4 | lo; + src += 2; + left -= 2; + } + return dest; +} + +bool slice::is_hex(bool ignore_spaces) const noexcept { + if (MDBX_UNLIKELY(length() % 2 && !ignore_spaces)) + return false; + + bool got = false; + auto src = byte_ptr(); + for (auto left = length(); left > 0;) { + if (MDBX_UNLIKELY(*src <= ' ') && + MDBX_LIKELY(ignore_spaces && isspace(*src))) { + ++src; + --left; + continue; + } + + if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1]))) + return false; + + got = true; + src += 2; + left -= 2; + } + return got; +} + +//------------------------------------------------------------------------------ + +enum : signed char { + OO /* ASCII NUL */ = -8, + EQ /* BASE64 '=' pad */ = -4, + SP /* SPACE */ = -2, + IL /* invalid */ = -1 +}; + +static const byte b58_alphabet[58] = { + '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', + 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', + 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + +#ifndef bswap64 +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +static inline uint64_t bswap64(uint64_t v) noexcept { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ + __has_builtin(__builtin_bswap64) + return __builtin_bswap64(v); +#elif defined(_MSC_VER) && !defined(__clang__) + return _byteswap_uint64(v); +#elif defined(__bswap_64) + return __bswap_64(v); +#elif defined(bswap_64) + return bswap_64(v); +#else + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | + ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | + ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | + ((v >> 40) & UINT64_C(0x000000000000ff00)); +#endif +} +#endif /* __BYTE_ORDER__ */ +#endif /* ifdef bswap64 */ + +static inline char b58_8to11(uint64_t &v) noexcept { + const unsigned i = unsigned(v % 58); + v /= 58; + return b58_alphabet[i]; +} + +char *slice::to_base58(char *__restrict dest, size_t dest_size, + unsigned wrap_width) const { + if (MDBX_UNLIKELY(to_base58_bytes(wrap_width) > dest_size)) + throw_too_small_target_buffer(); + + auto src = byte_ptr(); + size_t left = length(); + auto line = dest; + while (MDBX_LIKELY(left > 7)) { + left -= 8; + uint64_t v; + std::memcpy(&v, src, 8); + src += 8; +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + v = bswap64(v); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + dest[10] = b58_8to11(v); + dest[9] = b58_8to11(v); + dest[8] = b58_8to11(v); + dest[7] = b58_8to11(v); + dest[6] = b58_8to11(v); + dest[5] = b58_8to11(v); + dest[4] = b58_8to11(v); + dest[3] = b58_8to11(v); + dest[2] = b58_8to11(v); + dest[1] = b58_8to11(v); + dest[0] = b58_8to11(v); + assert(v == 0); + dest += 11; + if (wrap_width && size_t(dest - line) >= wrap_width) { + *dest = '\n'; + line = ++dest; + } + } + + if (left) { + uint64_t v = 0; + unsigned parrots = 31; + do { + v = (v << 8) + *src++; + parrots += 43; + } while (--left); + + auto ptr = dest += parrots >> 5; + do { + *--ptr = b58_8to11(v); + parrots -= 32; + } while (parrots > 31); + assert(v == 0); + } + + return dest; +} + +const signed char b58_map[256] = { + // 1 2 3 4 5 6 7 8 9 a b c d e f + OO, IL, IL, IL, IL, IL, IL, IL, IL, SP, SP, SP, SP, SP, IL, IL, // 00 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 10 + SP, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 20 + IL, 0, 1, 2, 3, 4, 5, 6, 7, 8, IL, IL, IL, IL, IL, IL, // 30 + IL, 9, 10, 11, 12, 13, 14, 15, 16, IL, 17, 18, 19, 20, 21, IL, // 40 + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, IL, IL, IL, IL, IL, // 50 + IL, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, IL, 44, 45, 46, // 60 + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, IL, IL, IL, IL, IL, // 70 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 80 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 90 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // a0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // b0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // c0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // d0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // e0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0 +}; + +static inline signed char b58_11to8(uint64_t &v, const byte c) noexcept { + const signed char m = b58_map[c]; + v = v * 58 + m; + return m; +} + +byte *slice::from_base58(byte *__restrict dest, size_t dest_size, + bool ignore_spaces) const { + if (MDBX_UNLIKELY(from_base58_bytes() > dest_size)) + throw_too_small_target_buffer(); + + auto src = byte_ptr(); + for (auto left = length(); left > 0;) { + if (MDBX_UNLIKELY(isspace(*src)) && ignore_spaces) { + ++src; + --left; + continue; + } + + if (MDBX_LIKELY(left > 10)) { + uint64_t v = 0; + if (MDBX_UNLIKELY((b58_11to8(v, src[0]) | b58_11to8(v, src[1]) | + b58_11to8(v, src[2]) | b58_11to8(v, src[3]) | + b58_11to8(v, src[4]) | b58_11to8(v, src[5]) | + b58_11to8(v, src[6]) | b58_11to8(v, src[7]) | + b58_11to8(v, src[8]) | b58_11to8(v, src[9]) | + b58_11to8(v, src[10])) < 0)) + goto bailout; +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + v = bswap64(v); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + std::memcpy(dest, &v, 8); + dest += 8; + src += 11; + left -= 11; + continue; + } + + constexpr unsigned invalid_length_mask = 1 << 1 | 1 << 4 | 1 << 8; + if (invalid_length_mask & (1 << left)) + goto bailout; + + uint64_t v = 1; + unsigned parrots = 0; + do { + if (MDBX_UNLIKELY(b58_11to8(v, *src++) < 0)) + goto bailout; + parrots += 32; + } while (--left); + + auto ptr = dest += parrots / 43; + do { + *--ptr = byte(v); + v >>= 8; + } while (v > 255); + break; + } + return dest; + +bailout: + throw std::domain_error("mdbx::from_base58:: invalid base58 string"); +} + +bool slice::is_base58(bool ignore_spaces) const noexcept { + bool got = false; + auto src = byte_ptr(); + for (auto left = length(); left > 0;) { + if (MDBX_UNLIKELY(*src <= ' ') && + MDBX_LIKELY(ignore_spaces && isspace(*src))) { + ++src; + --left; + continue; + } + + if (MDBX_LIKELY(left > 10)) { + if (MDBX_UNLIKELY((b58_map[src[0]] | b58_map[src[1]] | b58_map[src[2]] | + b58_map[src[3]] | b58_map[src[4]] | b58_map[src[5]] | + b58_map[src[6]] | b58_map[src[7]] | b58_map[src[8]] | + b58_map[src[9]] | b58_map[src[10]]) < 0)) + return false; + src += 11; + left -= 11; + got = true; + continue; + } + + constexpr unsigned invalid_length_mask = 1 << 1 | 1 << 4 | 1 << 8; + if (invalid_length_mask & (1 << left)) + return false; + + do + if (MDBX_UNLIKELY(b58_map[*src++] < 0)) + return false; + while (--left); + got = true; + break; + } + return got; +} + +//------------------------------------------------------------------------------ + +static inline void b64_3to4(const byte x, const byte y, const byte z, + char *__restrict dest) noexcept { + static const byte alphabet[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + dest[0] = alphabet[(x & 0xfc) >> 2]; + dest[1] = alphabet[((x & 0x03) << 4) + ((y & 0xf0) >> 4)]; + dest[2] = alphabet[((y & 0x0f) << 2) + ((z & 0xc0) >> 6)]; + dest[3] = alphabet[z & 0x3f]; +} + +char *slice::to_base64(char *__restrict dest, size_t dest_size, + unsigned wrap_width) const { + if (MDBX_UNLIKELY(to_base64_bytes(wrap_width) > dest_size)) + throw_too_small_target_buffer(); + + auto src = byte_ptr(); + size_t left = length(); + auto line = dest; + while (true) { + switch (left) { + default: + MDBX_CXX20_LIKELY left -= 3; + b64_3to4(src[0], src[1], src[2], dest); + dest += 4; + src += 3; + if (wrap_width && size_t(dest - line) >= wrap_width) { + *dest = '\n'; + line = ++dest; + } + continue; + case 2: + b64_3to4(src[0], 0, 0, dest); + dest[2] = dest[3] = '='; + return dest + 4; + case 1: + b64_3to4(src[0], src[1], 0, dest); + dest[3] = '='; + return dest + 4; + case 0: + return dest; + } + } +} + +static const signed char b64_map[256] = { + // 1 2 3 4 5 6 7 8 9 a b c d e f + OO, IL, IL, IL, IL, IL, IL, IL, IL, SP, SP, SP, SP, SP, IL, IL, // 00 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 10 + SP, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, 62, IL, IL, IL, 63, // 20 + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, IL, IL, IL, EQ, IL, IL, // 30 + IL, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 40 + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, IL, IL, IL, IL, IL, // 50 + IL, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, // 60 + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, IL, IL, IL, IL, IL, // 70 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 80 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // 90 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // a0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // b0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // c0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // d0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, // e0 + IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0 +}; + +static inline signed char b64_4to3(signed char a, signed char b, signed char c, + signed char d, + byte *__restrict dest) noexcept { + dest[0] = byte((a << 2) + ((b & 0x30) >> 4)); + dest[1] = byte(((b & 0xf) << 4) + ((c & 0x3c) >> 2)); + dest[2] = byte(((c & 0x3) << 6) + d); + return a | b | c | d; +} + +byte *slice::from_base64(byte *__restrict dest, size_t dest_size, + bool ignore_spaces) const { + if (MDBX_UNLIKELY(length() % 4 && !ignore_spaces)) + throw std::domain_error("mdbx::from_base64:: odd length of base64 string"); + if (MDBX_UNLIKELY(from_base64_bytes() > dest_size)) + throw_too_small_target_buffer(); + + auto src = byte_ptr(); + for (auto left = length(); left > 0;) { + if (MDBX_UNLIKELY(*src <= ' ') && + MDBX_LIKELY(ignore_spaces && isspace(*src))) { + ++src; + --left; + continue; + } + + if (MDBX_UNLIKELY(left < 3)) { + bailout: + throw std::domain_error("mdbx::from_base64:: invalid base64 string"); + } + const signed char a = b64_map[src[0]], b = b64_map[src[1]], + c = b64_map[src[2]], d = b64_map[src[3]]; + if (MDBX_UNLIKELY(b64_4to3(a, b, c, d, dest) < 0)) { + if (left == 4 && (a | b) >= 0 && d == EQ) { + if (c >= 0) + return dest + 2; + if (c == d) + return dest + 1; + } + goto bailout; + } + src += 4; + left -= 4; + } + return dest; +} + +bool slice::is_base64(bool ignore_spaces) const noexcept { + if (MDBX_UNLIKELY(length() % 4 && !ignore_spaces)) + return false; + + bool got = false; + auto src = byte_ptr(); + for (auto left = length(); left > 0;) { + if (MDBX_UNLIKELY(*src <= ' ') && + MDBX_LIKELY(ignore_spaces && isspace(*src))) { + ++src; + --left; + continue; + } + + if (MDBX_UNLIKELY(left < 3)) + return false; + const signed char a = b64_map[src[0]], b = b64_map[src[1]], + c = b64_map[src[2]], d = b64_map[src[3]]; + if (MDBX_UNLIKELY((a | b | c | d) < 0)) { + if (left == 4 && (a | b) >= 0 && d == EQ && (c >= 0 || c == d)) + return true; + return false; + } + got = true; + src += 4; + left -= 4; + } + return got; +} + +//------------------------------------------------------------------------------ + +template class LIBMDBX_API_TYPE buffer<legacy_allocator>; + +#if defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L +template class LIBMDBX_API_TYPE buffer<polymorphic_allocator>; +#endif /* __cpp_lib_memory_resource >= 201603L */ + +//------------------------------------------------------------------------------ + +size_t env::default_pagesize() noexcept { return ::mdbx_syspagesize(); } + +static inline MDBX_env_flags_t mode2flags(env::mode mode) { + switch (mode) { + default: + MDBX_CXX20_UNLIKELY throw std::invalid_argument("db::mode is invalid"); + case env::mode::readonly: + return MDBX_RDONLY; + case env::mode::write_file_io: + return MDBX_ENV_DEFAULTS; + case env::mode::write_mapped_io: + return MDBX_WRITEMAP; + } +} + +__cold MDBX_env_flags_t +env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { + MDBX_env_flags_t flags = mode2flags(mode); + if (accede) + flags |= MDBX_ACCEDE; + if (!use_subdirectory) + flags |= MDBX_NOSUBDIR; + if (options.exclusive) + flags |= MDBX_EXCLUSIVE; + if (options.orphan_read_transactions) + flags |= MDBX_NOTLS; + if (options.disable_readahead) + flags |= MDBX_NORDAHEAD; + if (options.disable_clear_memory) + flags |= MDBX_NOMEMINIT; + + if (mode != readonly) { + if (options.nested_write_transactions) + flags &= ~MDBX_WRITEMAP; + if (reclaiming.coalesce) + flags |= MDBX_COALESCE; + if (reclaiming.lifo) + flags |= MDBX_LIFORECLAIM; + switch (durability) { + default: + MDBX_CXX20_UNLIKELY throw std::invalid_argument( + "db::durability is invalid"); + case env::durability::robust_synchronous: + break; + case env::durability::half_synchronous_weak_last: + flags |= MDBX_NOMETASYNC; + break; + case env::durability::lazy_weak_tail: + static_assert(MDBX_MAPASYNC == MDBX_SAFE_NOSYNC, "WTF? Obsolete C API?"); + flags |= MDBX_SAFE_NOSYNC; + break; + case env::durability::whole_fragile: + flags |= MDBX_UTTERLY_NOSYNC; + break; + } + } + return flags; +} + +env::mode +env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept { + if (flags & MDBX_RDONLY) + return env::mode::readonly; + return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io + : env::mode::write_file_io; +} + +env::durability env::operate_parameters::durability_from_flags( + MDBX_env_flags_t flags) noexcept { + if ((flags & MDBX_UTTERLY_NOSYNC) == MDBX_UTTERLY_NOSYNC) + return env::durability::whole_fragile; + if (flags & MDBX_SAFE_NOSYNC) + return env::durability::lazy_weak_tail; + if (flags & MDBX_NOMETASYNC) + return env::durability::half_synchronous_weak_last; + return env::durability::robust_synchronous; +} + +env::reclaiming_options::reclaiming_options(MDBX_env_flags_t flags) noexcept + : lifo((flags & MDBX_LIFORECLAIM) ? true : false), + coalesce((flags & MDBX_COALESCE) ? true : false) {} + +env::operate_options::operate_options(MDBX_env_flags_t flags) noexcept + : orphan_read_transactions( + ((flags & (MDBX_NOTLS | MDBX_EXCLUSIVE)) == MDBX_NOTLS) ? true + : false), + nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false + : true), + exclusive((flags & MDBX_EXCLUSIVE) ? true : false), + disable_readahead((flags & MDBX_NORDAHEAD) ? true : false), + disable_clear_memory((flags & MDBX_NOMEMINIT) ? true : false) {} + +env::operate_parameters::operate_parameters(const env &env) + : max_maps(env.max_maps()), max_readers(env.max_readers()) { + const auto flags = env.get_flags(); + mode = mode_from_flags(flags); + durability = durability_from_flags(flags); + reclaiming = reclaiming_from_flags(flags); + options = options_from_flags(flags); +} + +bool env::is_pristine() const { + return get_stat().ms_mod_txnid == 0 && + get_info().mi_recent_txnid == INITIAL_TXNID; +} + +bool env::is_empty() const { return get_stat().ms_branch_pages == 0; } + +env &env::copy(const path &destination, bool compactify, + bool force_dynamic_size) { + const path_to_pchar<path> utf8(destination); + error::success_or_throw( + ::mdbx_env_copy(handle_, utf8, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE + : MDBX_CP_DEFAULTS))); + return *this; +} + +env &env::copy(filehandle fd, bool compactify, bool force_dynamic_size) { + error::success_or_throw( + ::mdbx_env_copy2fd(handle_, fd, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE + : MDBX_CP_DEFAULTS))); + return *this; +} + +path env::get_path() const { + const char *c_str; + error::success_or_throw(::mdbx_env_get_path(handle_, &c_str)); + return pchar_to_path<path>(c_str); +} + +//------------------------------------------------------------------------------ + +static inline MDBX_env *create_env() { + MDBX_env *ptr; + error::success_or_throw(::mdbx_env_create(&ptr)); + assert(ptr != nullptr); + return ptr; +} + +env_managed::~env_managed() noexcept { + if (handle_) + error::success_or_panic(::mdbx_env_close(handle_), "mdbx::~env()", + "mdbx_env_close"); +} + +void env_managed::close(bool dont_sync) { + const error rc = + static_cast<MDBX_error_t>(::mdbx_env_close_ex(handle_, dont_sync)); + switch (rc.code()) { + case MDBX_EBADSIGN: + handle_ = nullptr; + __fallthrough /* fall through */; + default: + rc.throw_exception(); + case MDBX_SUCCESS: + handle_ = nullptr; + } +} + +__cold void env_managed::setup(unsigned max_maps, unsigned max_readers) { + if (max_readers > 0) + error::success_or_throw(::mdbx_env_set_maxreaders(handle_, max_readers)); + if (max_maps > 0) + error::success_or_throw(::mdbx_env_set_maxdbs(handle_, max_maps)); +} + +__cold env_managed::env_managed(const path &pathname, + const operate_parameters &op, bool accede) + : env_managed(create_env()) { + setup(op.max_maps, op.max_readers); + const path_to_pchar<path> utf8(pathname); + error::success_or_throw( + ::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0)); + + if (op.options.nested_write_transactions && + !get_options().nested_write_transactions) + error::throw_exception(MDBX_INCOMPATIBLE); +} + +__cold env_managed::env_managed(const path &pathname, + const env_managed::create_parameters &cp, + const env::operate_parameters &op, bool accede) + : env_managed(create_env()) { + setup(op.max_maps, op.max_readers); + const path_to_pchar<path> utf8(pathname); + set_geometry(cp.geometry); + error::success_or_throw( + ::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory), + cp.file_mode_bits)); + + if (op.options.nested_write_transactions && + !get_options().nested_write_transactions) + error::throw_exception(MDBX_INCOMPATIBLE); +} + +//------------------------------------------------------------------------------ + +txn_managed txn::start_nested() { + MDBX_txn *nested; + error::throw_on_nullptr(handle_, MDBX_BAD_TXN); + error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_, + MDBX_TXN_READWRITE, &nested)); + assert(nested != nullptr); + return txn_managed(nested); +} + +txn_managed::~txn_managed() noexcept { + if (handle_) + error::success_or_panic(::mdbx_txn_abort(handle_), "mdbx::~txn", + "mdbx_txn_abort"); +} + +void txn_managed::abort() { + const error err = static_cast<MDBX_error_t>(::mdbx_txn_abort(handle_)); + if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS)) { + if (err.code() != MDBX_THREAD_MISMATCH) + handle_ = nullptr; + err.throw_exception(); + } +} + +void txn_managed::commit() { + const error err = static_cast<MDBX_error_t>(::mdbx_txn_commit(handle_)); + if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS)) { + if (err.code() != MDBX_THREAD_MISMATCH) + handle_ = nullptr; + err.throw_exception(); + } +} + +//------------------------------------------------------------------------------ + +bool txn::drop_map(const char *name, bool throw_if_absent) { + map_handle map; + const int err = ::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi); + switch (err) { + case MDBX_SUCCESS: + drop_map(map); + return true; + case MDBX_NOTFOUND: + case MDBX_BAD_DBI: + if (!throw_if_absent) + return false; + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +bool txn::clear_map(const char *name, bool throw_if_absent) { + map_handle map; + const int err = ::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi); + switch (err) { + case MDBX_SUCCESS: + clear_map(map); + return true; + case MDBX_NOTFOUND: + case MDBX_BAD_DBI: + if (!throw_if_absent) + return false; + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +//------------------------------------------------------------------------------ + +void cursor_managed::close() { + if (MDBX_UNLIKELY(!handle_)) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + ::mdbx_cursor_close(handle_); + handle_ = nullptr; +} + +//------------------------------------------------------------------------------ + +__cold ::std::ostream &operator<<(::std::ostream &out, const slice &it) { + out << "{"; + if (!it.is_valid()) + out << "INVALID." << it.length(); + else if (it.is_null()) + out << "NULL"; + else if (it.empty()) + out << "EMPTY->" << it.data(); + else { + const slice root(it.head(std::min(it.length(), size_t(64)))); + out << it.length() << "->" + << (root.is_printable() ? root.string() : root.base58_encode()) + << ((root == it) ? "" : "..."); + } + return out << "}"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, const pair &it) { + return out << "{" << it.key << " => " << it.value << "}"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const ::mdbx::env::geometry::size &it) { + switch (it.bytes) { + case ::mdbx::env::geometry::default_value: + return out << "default"; + case ::mdbx::env::geometry::minimal_value: + return out << "minimal"; + case ::mdbx::env::geometry::maximal_value: + return out << "maximal"; + } + + const auto bytes = (it.bytes < 0) ? out << "-", + size_t(-it.bytes) : size_t(it.bytes); + struct { + size_t one; + const char *suffix; + } static const scales[] = { +#if MDBX_WORDBITS > 32 + {env_managed::geometry::EiB, "EiB"}, + {env_managed::geometry::EB, "EB"}, + {env_managed::geometry::PiB, "PiB"}, + {env_managed::geometry::PB, "PB"}, + {env_managed::geometry::TiB, "TiB"}, + {env_managed::geometry::TB, "TB"}, +#endif + {env_managed::geometry::GiB, "GiB"}, + {env_managed::geometry::GB, "GB"}, + {env_managed::geometry::MiB, "MiB"}, + {env_managed::geometry::MB, "MB"}, + {env_managed::geometry::KiB, "KiB"}, + {env_managed::geometry::kB, "kB"}, + {1, " bytes"} + }; + + for (const auto i : scales) + if (bytes % i.one == 0) + return out << bytes / i.one << i.suffix; + + assert(false); + __unreachable(); + return out; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const env::geometry &it) { + return // + out << "\tlower " << env::geometry::size(it.size_lower) // + << ",\n\tnow " << env::geometry::size(it.size_now) // + << ",\n\tupper " << env::geometry::size(it.size_upper) // + << ",\n\tgrowth " << env::geometry::size(it.growth_step) // + << ",\n\tshrink " << env::geometry::size(it.shrink_threshold) // + << ",\n\tpagesize " << env::geometry::size(it.pagesize) << "\n"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const env::operate_parameters &it) { + return out << "{\n" // + << "\tmax_maps " << it.max_maps // + << ",\n\tmax_readers " << it.max_readers // + << ",\n\tmode " << it.mode // + << ",\n\tdurability " << it.durability // + << ",\n\treclaiming " << it.reclaiming // + << ",\n\toptions " << it.options // + << "\n}"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, const env::mode &it) { + switch (it) { + case env::mode::readonly: + return out << "readonly"; + case env::mode::write_file_io: + return out << "write_file_io"; + case env::mode::write_mapped_io: + return out << "write_mapped_io"; + default: + return out << "mdbx::env::mode::invalid"; + } +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const env::durability &it) { + switch (it) { + case env::durability::robust_synchronous: + return out << "robust_synchronous"; + case env::durability::half_synchronous_weak_last: + return out << "half_synchronous_weak_last"; + case env::durability::lazy_weak_tail: + return out << "lazy_weak_tail"; + case env::durability::whole_fragile: + return out << "whole_fragile"; + default: + return out << "mdbx::env::durability::invalid"; + } +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const env::reclaiming_options &it) { + return out << "{" // + << "lifo: " << (it.lifo ? "yes" : "no") // + << ", coalesce: " << (it.coalesce ? "yes" : "no") // + << "}"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const env::operate_options &it) { + static const char comma[] = ", "; + const char *delimiter = ""; + out << "{"; + if (it.orphan_read_transactions) { + out << delimiter << "orphan_read_transactions"; + delimiter = comma; + } + if (it.nested_write_transactions) { + out << delimiter << "nested_write_transactions"; + delimiter = comma; + } + if (it.exclusive) { + out << delimiter << "exclusive"; + delimiter = comma; + } + if (it.disable_readahead) { + out << delimiter << "disable_readahead"; + delimiter = comma; + } + if (it.disable_clear_memory) { + out << delimiter << "disable_clear_memory"; + delimiter = comma; + } + if (delimiter != comma) + out << "default"; + return out << "}"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const env_managed::create_parameters &it) { + return out << "{\n" // + << "\tfile_mode " << std::oct << it.file_mode_bits << std::dec // + << ",\n\tsubdirectory " << (it.use_subdirectory ? "yes" : "no") // + << ",\n" + << it.geometry << "}"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const MDBX_log_level_t &it) { + switch (it) { + case MDBX_LOG_FATAL: + return out << "LOG_FATAL"; + case MDBX_LOG_ERROR: + return out << "LOG_ERROR"; + case MDBX_LOG_WARN: + return out << "LOG_WARN"; + case MDBX_LOG_NOTICE: + return out << "LOG_NOTICE"; + case MDBX_LOG_VERBOSE: + return out << "LOG_VERBOSE"; + case MDBX_LOG_DEBUG: + return out << "LOG_DEBUG"; + case MDBX_LOG_TRACE: + return out << "LOG_TRACE"; + case MDBX_LOG_EXTRA: + return out << "LOG_EXTRA"; + case MDBX_LOG_DONTCHANGE: + return out << "LOG_DONTCHANGE"; + default: + return out << "mdbx::log_level::invalid"; + } +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const MDBX_debug_flags_t &it) { + if (it == MDBX_DBG_DONTCHANGE) + return out << "DBG_DONTCHANGE"; + + static const char comma[] = "|"; + const char *delimiter = ""; + out << "{"; + if (it & MDBX_DBG_ASSERT) { + out << delimiter << "DBG_ASSERT"; + delimiter = comma; + } + if (it & MDBX_DBG_AUDIT) { + out << delimiter << "DBG_AUDIT"; + delimiter = comma; + } + if (it & MDBX_DBG_JITTER) { + out << delimiter << "DBG_JITTER"; + delimiter = comma; + } + if (it & MDBX_DBG_DUMP) { + out << delimiter << "DBG_DUMP"; + delimiter = comma; + } + if (it & MDBX_DBG_LEGACY_MULTIOPEN) { + out << delimiter << "DBG_LEGACY_MULTIOPEN"; + delimiter = comma; + } + if (it & MDBX_DBG_LEGACY_OVERLAP) { + out << delimiter << "DBG_LEGACY_OVERLAP"; + delimiter = comma; + } + if (delimiter != comma) + out << "DBG_NONE"; + return out << "}"; +} + +__cold ::std::ostream &operator<<(::std::ostream &out, + const ::mdbx::error &err) { + return out << err.what() << " (" << long(err.code()) << ")"; +} + +} // namespace mdbx diff --git a/libs/libmdbx/src/mdbx.h b/libs/libmdbx/src/mdbx.h index 97750bc05d..982f5973ae 100644 --- a/libs/libmdbx/src/mdbx.h +++ b/libs/libmdbx/src/mdbx.h @@ -1,536 +1,69 @@ -/**** BRIEFLY ****************************************************************** - * - * libmdbx is superior to LMDB (https://bit.ly/26ts7tL) in terms of features - * and reliability, not inferior in performance. In comparison to LMDB, libmdbx - * makes many things just work perfectly, not silently and catastrophically - * break down. libmdbx supports Linux, Windows, MacOS, FreeBSD, DragonFly, - * Solaris, OpenSolaris, OpenIndiana, NetBSD, OpenBSD and other systems - * compliant with POSIX.1-2008. - * - * Look below for API description, for other information (build, embedding and - * amalgamation, improvements over LMDB, benchmarking, etc) please refer to - * README.md at https://abf.io/erthink/libmdbx. - * - * --- - * - * The next version is under active non-public development and will be released - * as MithrilDB and libmithrildb for libraries & packages. Admittedly mythical - * Mithril is resembling silver but being stronger and lighter than steel. - * Therefore MithrilDB is rightly relevant name. - * - * MithrilDB will be radically different from libmdbx by the new database format - * and API based on C++17, as well as the Apache 2.0 License. The goal of this - * revolution is to provide a clearer and robust API, add more features and new - * valuable properties of database. - * - * The Future will (be) Positive. Всё будет хорошо. - * - * - **** INTRODUCTION ************************************************************* - * - * // For the most part, this section is a copy of the corresponding text - * // from LMDB description, but with some edits reflecting the improvements - * // and enhancements were made in MDBX. - * - * MDBX is a Btree-based database management library modeled loosely on the - * BerkeleyDB API, but much simplified. The entire database (aka "environment") - * is exposed in a memory map, and all data fetches return data directly from - * the mapped memory, so no malloc's or memcpy's occur during data fetches. - * As such, the library is extremely simple because it requires no page caching - * layer of its own, and it is extremely high performance and memory-efficient. - * It is also fully transactional with full ACID semantics, and when the memory - * map is read-only, the database integrity cannot be corrupted by stray pointer - * writes from application code. - * - * The library is fully thread-aware and supports concurrent read/write access - * from multiple processes and threads. Data pages use a copy-on-write strategy - * so no active data pages are ever overwritten, which also provides resistance - * to corruption and eliminates the need of any special recovery procedures - * after a system crash. Writes are fully serialized; only one write transaction - * may be active at a time, which guarantees that writers can never deadlock. - * The database structure is multi-versioned so readers run with no locks; - * writers cannot block readers, and readers don't block writers. - * - * Unlike other well-known database mechanisms which use either write-ahead - * transaction logs or append-only data writes, MDBX requires no maintenance - * during operation. Both write-ahead loggers and append-only databases require - * periodic checkpointing and/or compaction of their log or database files - * otherwise they grow without bound. MDBX tracks retired/freed pages within the - * database and re-uses them for new write operations, so the database size does - * not grow without bound in normal use. It is worth noting that the "next" - * version libmdbx (MithrilDB) will solve this problem. - * - * The memory map can be used as a read-only or read-write map. It is read-only - * by default as this provides total immunity to corruption. Using read-write - * mode offers much higher write performance, but adds the possibility for stray - * application writes thru pointers to silently corrupt the database. - * Of course if your application code is known to be bug-free (...) then this is - * not an issue. - * - * If this is your first time using a transactional embedded key-value store, - * you may find the "GETTING STARTED" section below to be helpful. - * - * - **** GETTING STARTED ********************************************************** - * - * // This section is based on Bert Hubert's intro "LMDB Semantics", with - * // edits reflecting the improvements and enhancements were made in MDBX. - * // See https://bit.ly/2maejGY for Bert Hubert's original. - * - * Everything starts with an environment, created by mdbx_env_create(). - * Once created, this environment must also be opened with mdbx_env_open(), - * and after use be closed by mdbx_env_close(). At that a non-zero value of the - * last argument "mode" supposes MDBX will create database and directory if ones - * does not exist. In this case the non-zero "mode" argument specifies the file - * mode bits be applied when a new files are created by open() function. - * - * Within that directory, a lock file (aka LCK-file) and a storage file (aka - * DXB-file) will be generated. If you don't want to use a directory, you can - * pass the MDBX_NOSUBDIR option, in which case the path you provided is used - * directly as the DXB-file, and another file with a "-lck" suffix added - * will be used for the LCK-file. - * - * Once the environment is open, a transaction can be created within it using - * mdbx_txn_begin(). Transactions may be read-write or read-only, and read-write - * transactions may be nested. A transaction must only be used by one thread at - * a time. Transactions are always required, even for read-only access. The - * transaction provides a consistent view of the data. - * - * Once a transaction has been created, a database (i.e. key-value space inside - * the environment) can be opened within it using mdbx_dbi_open(). If only one - * database will ever be used in the environment, a NULL can be passed as the - * database name. For named databases, the MDBX_CREATE flag must be used to - * create the database if it doesn't already exist. Also, mdbx_env_set_maxdbs() - * must be called after mdbx_env_create() and before mdbx_env_open() to set the - * maximum number of named databases you want to support. - * - * NOTE: a single transaction can open multiple databases. Generally databases - * should only be opened once, by the first transaction in the process. - * - * Within a transaction, mdbx_get() and mdbx_put() can store single key-value - * pairs if that is all you need to do (but see CURSORS below if you want to do - * more). - * - * A key-value pair is expressed as two MDBX_val structures. This struct that is - * exactly similar to POSIX's struct iovec and has two fields, iov_len and - * iov_base. The data is a void pointer to an array of iov_len bytes. - * (!) The notable difference between MDBX and LMDB is that MDBX support zero - * length keys. - * - * Because MDBX is very efficient (and usually zero-copy), the data returned in - * an MDBX_val structure may be memory-mapped straight from disk. In other words - * look but do not touch (or free() for that matter). Once a transaction is - * closed, the values can no longer be used, so make a copy if you need to keep - * them after that. - * - * - * CURSORS -- To do more powerful things, we must use a cursor. - * - * Within the transaction, a cursor can be created with mdbx_cursor_open(). - * With this cursor we can store/retrieve/delete (multiple) values using - * mdbx_cursor_get(), mdbx_cursor_put(), and mdbx_cursor_del(). - * - * mdbx_cursor_get() positions itself depending on the cursor operation - * requested, and for some operations, on the supplied key. For example, to list - * all key-value pairs in a database, use operation MDBX_FIRST for the first - * call to mdbx_cursor_get(), and MDBX_NEXT on subsequent calls, until the end - * is hit. - * - * To retrieve all keys starting from a specified key value, use MDBX_SET. For - * more cursor operations, see the API description below. - * - * When using mdbx_cursor_put(), either the function will position the cursor - * for you based on the key, or you can use operation MDBX_CURRENT to use the - * current position of the cursor. NOTE that key must then match the current - * position's key. - * - * - * SUMMARIZING THE OPENING - * - * So we have a cursor in a transaction which opened a database in an - * environment which is opened from a filesystem after it was separately - * created. - * - * Or, we create an environment, open it from a filesystem, create a transaction - * within it, open a database within that transaction, and create a cursor - * within all of the above. - * - * Got it? - * - * - * THREADS AND PROCESSES - * - * Do not have open an database twice in the same process at the same time, MDBX - * will track and prevent this. Instead, share the MDBX environment that has - * opened the file across all threads. The reason for this is: - * - When the "Open file description" locks (aka OFD-locks) are not available, - * MDBX uses POSIX locks on files, and these locks have issues if one process - * opens a file multiple times. - * - If a single process opens the same environment multiple times, closing it - * once will remove all the locks held on it, and the other instances will be - * vulnerable to corruption from other processes. - * + For compatibility with LMDB which allows multi-opening, MDBX can be - * configured at runtime by mdbx_setup_debug(MDBX_DBG_LEGACY_MULTIOPEN, ...) - * prior to calling other MDBX funcitons. In this way MDBX will track - * databases opening, detect multi-opening cases and then recover POSIX file - * locks as necessary. However, lock recovery can cause unexpected pauses, - * such as when another process opened the database in exclusive mode before - * the lock was restored - we have to wait until such a process releases the - * database, and so on. - * - * Do not use opened MDBX environment(s) after fork() in a child process(es), - * MDBX will check and prevent this at critical points. Instead, ensure there is - * no open MDBX-instance(s) during fork(), or atleast close it immediately after - * fork() in the child process and reopen if required - for instance by using - * pthread_atfork(). The reason for this is: - * - For competitive consistent reading, MDBX assigns a slot in the shared - * table for each process that interacts with the database. This slot is - * populated with process attributes, including the PID. - * - After fork(), in order to remain connected to a database, the child - * process must have its own such "slot", which can't be assigned in any - * simple and robust way another than the regular. - * - A write transaction from a parent process cannot continue in a child - * process for obvious reasons. - * - Moreover, in a multithreaded process at the fork() moment any number of - * threads could run in critical and/or intermediate sections of MDBX code - * with interaction and/or racing conditions with threads from other - * process(es). For instance: shrinking a database or copying it to a pipe, - * opening or closing environment, begining or finishing a transaction, - * and so on. - * = Therefore, any solution other than simply close database (and reopen if - * necessary) in a child process would be both extreme complicated and so - * fragile. - * - * Do not start more than one transaction for a one thread. If you think about - * this, it's really strange to do something with two data snapshots at once, - * which may be different. MDBX checks and preventing this by returning - * corresponding error code (MDBX_TXN_OVERLAPPING, MDBX_BAD_RSLOT, MDBX_BUSY) - * unless you using MDBX_NOTLS option on the environment. Nonetheless, with the - * MDBX_NOTLS option, you must know exactly what you are doing, otherwise you - * will get deadlocks or reading an alien data. - * - * Also note that a transaction is tied to one thread by default using Thread - * Local Storage. If you want to pass read-only transactions across threads, - * you can use the MDBX_NOTLS option on the environment. Nevertheless, a write - * transaction entirely should only be used in one thread from start to finish. - * MDBX checks this in a reasonable manner and return the MDBX_THREAD_MISMATCH - * error in rules violation. - * - * - * TRANSACTIONS, ROLLBACKS, etc. - * - * To actually get anything done, a transaction must be committed using - * mdbx_txn_commit(). Alternatively, all of a transaction's operations - * can be discarded using mdbx_txn_abort(). - * - * (!) An important difference between MDBX and LMDB is that MDBX required that - * any opened cursors can be reused and must be freed explicitly, regardless - * ones was opened in a read-only or write transaction. The REASON for this is - * eliminates ambiguity which helps to avoid errors such as: use-after-free, - * double-free, i.e. memory corruption and segfaults. - * - * For read-only transactions, obviously there is nothing to commit to storage. - * (!) An another notable difference between MDBX and LMDB is that MDBX make - * handles opened for existing databases immediately available for other - * transactions, regardless this transaction will be aborted or reset. The - * REASON for this is to avoiding the requirement for multiple opening a same - * handles in concurrent read transactions, and tracking of such open but hidden - * handles until the completion of read transactions which opened them. - * - * In addition, as long as a transaction is open, a consistent view of the - * database is kept alive, which requires storage. A read-only transaction that - * no longer requires this consistent view should be terminated (committed or - * aborted) when the view is no longer needed (but see below for an - * optimization). - * - * There can be multiple simultaneously active read-only transactions but only - * one that can write. Once a single read-write transaction is opened, all - * further attempts to begin one will block until the first one is committed or - * aborted. This has no effect on read-only transactions, however, and they may - * continue to be opened at any time. - * - * - * DUPLICATE KEYS - * - * mdbx_get() and mdbx_put() respectively have no and only some support or - * multiple key-value pairs with identical keys. If there are multiple values - * for a key, mdbx_get() will only return the first value. - * - * When multiple values for one key are required, pass the MDBX_DUPSORT flag to - * mdbx_dbi_open(). In an MDBX_DUPSORT database, by default mdbx_put() will not - * replace the value for a key if the key existed already. Instead it will add - * the new value to the key. In addition, mdbx_del() will pay attention to the - * value field too, allowing for specific values of a key to be deleted. - * - * Finally, additional cursor operations become available for traversing through - * and retrieving duplicate values. - * - * - * SOME OPTIMIZATION - * - * If you frequently begin and abort read-only transactions, as an optimization, - * it is possible to only reset and renew a transaction. - * - * mdbx_txn_reset() releases any old copies of data kept around for a read-only - * transaction. To reuse this reset transaction, call mdbx_txn_renew() on it. - * Any cursors in this transaction can also be renewed using mdbx_cursor_renew() - * or freed by mdbx_cursor_close(). - * - * To permanently free a transaction, reset or not, use mdbx_txn_abort(). - * - * - * CLEANING UP - * - * Any created cursors must be closed using mdbx_cursor_close(). It is advisable - * to repeat: - * (!) An important difference between MDBX and LMDB is that MDBX required that - * any opened cursors can be reused and must be freed explicitly, regardless - * ones was opened in a read-only or write transaction. The REASON for this is - * eliminates ambiguity which helps to avoid errors such as: use-after-free, - * double-free, i.e. memory corruption and segfaults. - * - * It is very rarely necessary to close a database handle, and in general they - * should just be left open. When you close a handle, it immediately becomes - * unavailable for all transactions in the environment. Therefore, you should - * avoid closing the handle while at least one transaction is using it. - * - * - * THE FULL API - * - * The full MDBX documentation lists further details below, - * like how to: - * - * - configure database size and automatic size management - * - drop and clean a database - * - detect and report errors - * - optimize (bulk) loading speed - * - (temporarily) reduce robustness to gain even more speed - * - gather statistics about the database - * - estimate size of range query result - * - double perfomance by LIFO reclaiming on storages with write-back - * - use sequences and canary markers - * - use lack-of-space callback (aka OOM-KICK) - * - use exclusive mode - * - define custom sort orders (but this is recommended to be avoided) - * - * - **** RESTRICTIONS & CAVEATS *************************************************** - * in addition to those listed for some functions. - * - * - Troubleshooting the LCK-file. - * 1. A broken LCK-file can cause sync issues, including appearance of - * wrong/inconsistent data for readers. When database opened in the - * cooperative read-write mode the LCK-file requires to be mapped to - * memory in read-write access. In this case it is always possible for - * stray/malfunctioned application could writes thru pointers to - * silently corrupt the LCK-file. - * - * Unfortunately, there is no any portable way to prevent such - * corruption, since the LCK-file is updated concurrently by - * multiple processes in a lock-free manner and any locking is - * unwise due to a large overhead. - * - * The "next" version of libmdbx (MithrilDB) will solve this issue. - * - * Workaround: Just make all programs using the database close it; - * the LCK-file is always reset on first open. - * - * 2. Stale reader transactions left behind by an aborted program cause - * further writes to grow the database quickly, and stale locks can - * block further operation. - * MDBX checks for stale readers while opening environment and before - * growth the database. But in some cases, this may not be enough. - * - * Workaround: Check for stale readers periodically, using the - * mdbx_reader_check() function or the mdbx_stat tool. - * - * 3. Stale writers will be cleared automatically by MDBX on supprted - * platforms. But this is platform-specific, especially of - * implementation of shared POSIX-mutexes and support for robust - * mutexes. For instance there are no known issues on Linux, OSX, - * Windows and FreeBSD. - * - * Workaround: Otherwise just make all programs using the database - * close it; the LCK-file is always reset on first open - * of the environment. - * - * - Do not use MDBX databases on remote filesystems, even between processes - * on the same host. This breaks file locks on some platforms, possibly - * memory map sync, and certainly sync between programs on different hosts. - * - * On the other hand, MDBX support the exclusive database operation over - * a network, and cooperative read-only access to the database placed on - * a read-only network shares. - * - * - Do not use opened MDBX_env instance(s) in a child processes after fork(). - * It would be insane to call fork() and any MDBX-functions simultaneously - * from multiple threads. The best way is to prevent the presence of open - * MDBX-instances during fork(). - * - * The MDBX_TXN_CHECKPID build-time option, which is ON by default on - * non-Windows platforms (i.e. where fork() is available), enables PID - * checking at a few critical points. But this does not give any guarantees, - * but only allows you to detect such errors a little sooner. Depending on - * the platform, you should expect an application crash and/or database - * corruption in such cases. - * - * On the other hand, MDBX allow calling mdbx_close_env() in such cases to - * release resources, but no more and in general this is a wrong way. - * - * - There is no pure read-only mode in a normal explicitly way, since - * readers need write access to LCK-file to be ones visible for writer. - * MDBX always tries to open/create LCK-file for read-write, but switches - * to without-LCK mode on appropriate errors (EROFS, EACCESS, EPERM) - * if the read-only mode was requested by the MDBX_RDONLY flag which is - * described below. - * - * The "next" version of libmdbx (MithrilDB) will solve this issue. - * - * - A thread can only use one transaction at a time, plus any nested - * read-write transactions in the non-writemap mode. Each transaction - * belongs to one thread. The MDBX_NOTLS flag changes this for read-only - * transactions. See below. - * - * Do not start more than one transaction for a one thread. If you think - * about this, it's really strange to do something with two data snapshots - * at once, which may be different. MDBX checks and preventing this by - * returning corresponding error code (MDBX_TXN_OVERLAPPING, MDBX_BAD_RSLOT, - * MDBX_BUSY) unless you using MDBX_NOTLS option on the environment. - * Nonetheless, with the MDBX_NOTLS option, you must know exactly what you - * are doing, otherwise you will get deadlocks or reading an alien data. - * - * - Do not have open an MDBX database twice in the same process at the same - * time. By default MDBX prevent this in most cases by tracking databases - * opening and return MDBX_BUSY if anyone LCK-file is already open. - * - * The reason for this is that when the "Open file description" locks (aka - * OFD-locks) are not available, MDBX uses POSIX locks on files, and these - * locks have issues if one process opens a file multiple times. If a single - * process opens the same environment multiple times, closing it once will - * remove all the locks held on it, and the other instances will be - * vulnerable to corruption from other processes. - * - * For compatibility with LMDB which allows multi-opening, MDBX can be - * configured at runtime by mdbx_setup_debug(MDBX_DBG_LEGACY_MULTIOPEN, ...) - * prior to calling other MDBX funcitons. In this way MDBX will track - * databases opening, detect multi-opening cases and then recover POSIX file - * locks as necessary. However, lock recovery can cause unexpected pauses, - * such as when another process opened the database in exclusive mode before - * the lock was restored - we have to wait until such a process releases the - * database, and so on. - * - * - Avoid long-lived read transactions, especially in the scenarios with a - * high rate of write transactions. Long-lived read transactions prevents - * recycling pages retired/freed by newer write transactions, thus the - * database can grow quickly. - * - * Understanding the problem of long-lived read transactions requires some - * explanation, but can be difficult for quick perception. So is is - * reasonable to simplify this as follows: - * 1. Garbage collection problem exists in all databases one way or - * another, e.g. VACUUM in PostgreSQL. But in MDBX it's even more - * discernible because of high transaction rate and intentional - * internals simplification in favor of performance. - * - * 2. MDBX employs Multiversion concurrency control on the Copy-on-Write - * basis, that allows multiple readers runs in parallel with a write - * transaction without blocking. An each write transaction needs free - * pages to put the changed data, that pages will be placed in the new - * b-tree snapshot at commit. MDBX efficiently recycling pages from - * previous created unused snapshots, BUT this is impossible if anyone - * a read transaction use such snapshot. - * - * 3. Thus massive altering of data during a parallel long read operation - * will increase the process's work set and may exhaust entire free - * database space. - * - * A good example of long readers is a hot backup to the slow destination - * or debugging of a client application while retaining an active read - * transaction. LMDB this results in MAP_FULL error and subsequent write - * performance degradation. - * - * MDBX mostly solve "long-lived" readers issue by the lack-of-space callback - * which allow to aborts long readers, and by the MDBX_LIFORECLAIM mode which - * addresses subsequent performance degradation. - * The "next" version of libmdbx (MithrilDB) will completely solve this. - * - * - Avoid suspending a process with active transactions. These would then be - * "long-lived" as above. - * - * The "next" version of libmdbx (MithrilDB) will solve this issue. - * - * - Avoid aborting a process with an active read-only transaction in scenaries - * with high rate of write transactions. The transaction becomes "long-lived" - * as above until a check for stale readers is performed or the LCK-file is - * reset, since the process may not remove it from the lockfile. This does - * not apply to write transactions if the system clears stale writers, see - * above. - * - * - An MDBX database configuration will often reserve considerable unused - * memory address space and maybe file size for future growth. This does - * not use actual memory or disk space, but users may need to understand - * the difference so they won't be scared off. - * - * - The Write Amplification Factor. - * TBD. - * - **** LICENSE AND COPYRUSTING ************************************************** - * - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - * - * --- - * - * This code is derived from "LMDB engine" written by - * Howard Chu (Symas Corporation), which itself derived from btree.c - * written by Martin Hedenfalk. - * - * --- - * - * Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - * - * --- - * - * Portions Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se> - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - **** ACKNOWLEDGEMENTS ********************************************************* - * - * Howard Chu (Symas Corporation) - the author of LMDB, - * from which originated the MDBX in 2015. - * - * Martin Hedenfalk <martin@bzero.se> - the author of `btree.c` code, - * which was used for begin development of LMDB. - * - ******************************************************************************/ +/** + +_libmdbx_ is an extremely fast, compact, powerful, embedded, +transactional [key-value +store](https://en.wikipedia.org/wiki/Key-value_database) database, with +[permissive license](./LICENSE). _MDBX_ has a specific set of properties and +capabilities, focused on creating unique lightweight solutions with +extraordinary performance. + +_libmdbx_ is superior to [LMDB](https://bit.ly/26ts7tL) in terms of features +and reliability, not inferior in performance. In comparison to LMDB, _libmdbx_ +makes many things just work perfectly, not silently and catastrophically +break down. _libmdbx_ supports Linux, Windows, MacOS, OSX, iOS, Android, +FreeBSD, DragonFly, Solaris, OpenSolaris, OpenIndiana, NetBSD, OpenBSD and other +systems compliant with POSIX.1-2008. + +_The Future will (be) [Positive](https://www.ptsecurity.com). Всё будет хорошо._ + + +\section copyright LICENSE & COPYRIGHT + +\authors Copyright (c) 2015-2020, Leonid Yuriev <leo@yuriev.ru> +and other _libmdbx_ authors: please see [AUTHORS](./AUTHORS) file. + +\copyright Redistribution and use in source and binary forms, with or without +modification, are permitted only as authorized by the OpenLDAP Public License. + +A copy of this license is available in the file LICENSE in the +top-level directory of the distribution or, alternatively, at +<http://www.OpenLDAP.org/license.html>. + + --- + +This code is derived from "LMDB engine" written by +Howard Chu (Symas Corporation), which itself derived from btree.c +written by Martin Hedenfalk. + + --- + +Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted only as authorized by the OpenLDAP +Public License. + +A copy of this license is available in the file LICENSE in the +top-level directory of the distribution or, alternatively, at +<http://www.OpenLDAP.org/license.html>. + + --- + +Portions Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se> + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +*******************************************************************************/ #pragma once #ifndef LIBMDBX_H @@ -547,84 +80,245 @@ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ +/* *INDENT-OFF* */ +/* clang-format off */ + +/** + \file mdbx.h + \brief The libmdbx C API header file + + \defgroup c_api C API + @{ + \defgroup c_err Error handling + \defgroup c_opening Opening & Closing + \defgroup c_transactions Transactions + \defgroup c_dbi Databases + \defgroup c_crud Create/Read/Update/Delete (see Quick Reference in details) + + \details + \anchor c_crud_hints +# Quick Reference for Insert/Update/Delete operations + +Historically, libmdbx inherits the API basis from LMDB, where it is often +difficult to select flags/options and functions for the desired operation. +So it is recommend using this hints. + +## Databases with UNIQUE keys + +In databases created without the \ref MDBX_DUPSORT option, keys are always +unique. Thus always a single value corresponds to the each key, and so there +are only a few cases of changing data. + +| Case | Flags to use | Result | +|---------------------------------------------|---------------------|------------------------| +| _INSERTING_||| +|Key is absent → Insertion |\ref MDBX_NOOVERWRITE|Insertion | +|Key exist → Error since key present |\ref MDBX_NOOVERWRITE|Error \ref MDBX_KEYEXIST and return Present value| +| _UPSERTING_||| +|Key is absent → Insertion |\ref MDBX_UPSERT |Insertion | +|Key exist → Update |\ref MDBX_UPSERT |Update | +| _UPDATING_||| +|Key is absent → Error since no such key |\ref MDBX_CURRENT |Error \ref MDBX_NOTFOUND| +|Key exist → Update |\ref MDBX_CURRENT |Update value | +| _DELETING_||| +|Key is absent → Error since no such key |\ref mdbx_del() or \ref mdbx_replace()|Error \ref MDBX_NOTFOUND| +|Key exist → Delete by key |\ref mdbx_del() with the parameter `data = NULL`|Deletion| +|Key exist → Delete by key with with data matching check|\ref mdbx_del() with the parameter `data` filled with the value which should be match for deletion|Deletion or \ref MDBX_NOTFOUND if the value does not match| +|Delete at the current cursor position |\ref mdbx_cursor_del() with \ref MDBX_CURRENT flag|Deletion| +|Extract (read & delete) value by the key |\ref mdbx_replace() with zero flag and parameter `new_data = NULL`|Returning a deleted value| + + +## Databases with NON-UNIQUE keys + +In databases created with the \ref MDBX_DUPSORT (Sorted Duplicates) option, keys +may be non unique. Such non-unique keys in a key-value database may be treated +as a duplicates or as like a multiple values corresponds to keys. + + +| Case | Flags to use | Result | +|---------------------------------------------|---------------------|------------------------| +| _INSERTING_||| +|Key is absent → Insertion |\ref MDBX_NOOVERWRITE|Insertion| +|Key exist → Needn't to add new values |\ref MDBX_NOOVERWRITE|Error \ref MDBX_KEYEXIST with returning the first value from those already present| +| _UPSERTING_||| +|Key is absent → Insertion |\ref MDBX_UPSERT |Insertion| +|Key exist → Wanna to add new values |\ref MDBX_UPSERT |Add one more value to the key| +|Key exist → Replace all values with a new one|\ref MDBX_UPSERT + \ref MDBX_ALLDUPS|Overwrite by single new value| +| _UPDATING_||| +|Key is absent → Error since no such key |\ref MDBX_CURRENT |Error \ref MDBX_NOTFOUND| +|Key exist, Single value → Update |\ref MDBX_CURRENT |Update single value | +|Key exist, Multiple values → Replace all values with a new one|\ref MDBX_CURRENT + \ref MDBX_ALLDUPS|Overwrite by single new value| +|Key exist, Multiple values → Error since it is unclear which of the values should be updated|\ref mdbx_put() with \ref MDBX_CURRENT|Error \ref MDBX_EMULTIVAL| +|Key exist, Multiple values → Update particular entry of multi-value|\ref mdbx_replace() with \ref MDBX_CURRENT + \ref MDBX_NOOVERWRITE and the parameter `old_value` filled with the value that wanna to update|Update one multi-value entry| +|Key exist, Multiple values → Update the current entry of multi-value|\ref mdbx_cursor_put() with \ref MDBX_CURRENT|Update one multi-value entry| +| _DELETING_||| +|Key is absent → Error since no such key |\ref mdbx_del() or \ref mdbx_replace()|Error \ref MDBX_NOTFOUND| +|Key exist → Delete all values corresponds given key|\ref mdbx_del() with the parameter `data = NULL`|Deletion| +|Key exist → Delete particular value corresponds given key|\ref mdbx_del() with the parameter `data` filled with the value that wanna to delete, or \ref mdbx_replace() with \ref MDBX_CURRENT + \ref MDBX_NOOVERWRITE and the `old_value` parameter filled with the value that wanna to delete and `new_data = NULL`| Deletion or \ref MDBX_NOTFOUND if no such key-value pair| +|Delete one value at the current cursor position|\ref mdbx_cursor_del() with \ref MDBX_CURRENT flag|Deletion only the current entry| +|Delete all values of key at the current cursor position|\ref mdbx_cursor_del() with with \ref MDBX_ALLDUPS flag|Deletion all duplicates of key (all multi-values) at the current cursor position| + + \defgroup c_cursors Cursors + \defgroup c_statinfo Statistics & Information + \defgroup c_settings Settings + \defgroup c_debug Logging and runtime debug + \defgroup c_rqest Range query estimation + \defgroup c_extra Extra operations +*/ + +/* *INDENT-ON* */ +/* clang-format on */ + #include <stdarg.h> #include <stddef.h> #include <stdint.h> #if defined(_WIN32) || defined(_WIN64) - #include <windows.h> #include <winnt.h> - -#ifndef FSCTL_GET_EXTERNAL_BACKING -#define FSCTL_GET_EXTERNAL_BACKING CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) -#endif - #ifndef __mode_t_defined -typedef unsigned short mode_t; -#endif +typedef unsigned short mdbx_mode_t; +#else +typedef mode_t mdbx_mode_t; +#endif /* __mode_t_defined */ typedef HANDLE mdbx_filehandle_t; typedef DWORD mdbx_pid_t; typedef DWORD mdbx_tid_t; -#define MDBX_ENODATA ERROR_HANDLE_EOF -#define MDBX_EINVAL ERROR_INVALID_PARAMETER -#define MDBX_EACCESS ERROR_ACCESS_DENIED -#define MDBX_ENOMEM ERROR_OUTOFMEMORY -#define MDBX_EROFS ERROR_FILE_READ_ONLY -#define MDBX_ENOSYS ERROR_NOT_SUPPORTED -#define MDBX_EIO ERROR_WRITE_FAULT -#define MDBX_EPERM ERROR_INVALID_FUNCTION -#define MDBX_EINTR ERROR_CANCELLED -#define MDBX_ENOFILE ERROR_FILE_NOT_FOUND -#define MDBX_EREMOTE ERROR_REMOTE_STORAGE_MEDIA_ERROR - -#else - +#else /* Windows */ #include <errno.h> /* for error codes */ #include <pthread.h> /* for pthread_t */ #include <sys/types.h> /* for pid_t */ -#include <sys/uio.h> /* for truct iovec */ +#include <sys/uio.h> /* for struct iovec */ #define HAVE_STRUCT_IOVEC 1 typedef int mdbx_filehandle_t; typedef pid_t mdbx_pid_t; typedef pthread_t mdbx_tid_t; -#ifdef ENODATA -#define MDBX_ENODATA ENODATA -#else -#define MDBX_ENODATA -1 -#endif -#define MDBX_EINVAL EINVAL -#define MDBX_EACCESS EACCES -#define MDBX_ENOMEM ENOMEM -#define MDBX_EROFS EROFS -#define MDBX_ENOSYS ENOSYS -#define MDBX_EIO EIO -#define MDBX_EPERM EPERM -#define MDBX_EINTR EINTR -#define MDBX_ENOFILE ENOENT -#define MDBX_EREMOTE ENOTBLK - -#endif +typedef mode_t mdbx_mode_t; +#endif /* !Windows */ #ifdef _MSC_VER #pragma warning(pop) #endif +/** @} close c_api + * \defgroup api_macros Common Macros + * @{ */ + /*----------------------------------------------------------------------------*/ #ifndef __has_attribute #define __has_attribute(x) (0) #endif /* __has_attribute */ -#ifndef __deprecated -#if defined(__GNUC__) || __has_attribute(__deprecated__) -#define __deprecated __attribute__((__deprecated__)) +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif /* __has_cpp_attribute */ + +#ifndef __has_feature +#define __has_feature(x) (0) +#endif /* __has_feature */ + +#ifndef __has_extension +#define __has_extension(x) (0) +#endif /* __has_extension */ + +#ifndef __has_builtin +#define __has_builtin(x) (0) +#endif /* __has_builtin */ + +/** Many functions have no effects except the return value and their + * return value depends only on the parameters and/or global variables. + * Such a function can be subject to common subexpression elimination + * and loop optimization just as an arithmetic operator would be. + * These functions should be declared with the attribute pure. */ +#if (defined(__GNUC__) || __has_attribute(__pure__)) && \ + (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \ + || !defined(__cplusplus) || !__has_feature(cxx_exceptions)) +#define MDBX_PURE_FUNCTION __attribute__((__pure__)) +#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 +#define MDBX_PURE_FUNCTION +#elif defined(__cplusplus) && __has_cpp_attribute(gnu::pure) && \ + (!defined(__clang__) || !__has_feature(cxx_exceptions)) +#define MDBX_PURE_FUNCTION [[gnu::pure]] +#else +#define MDBX_PURE_FUNCTION +#endif /* MDBX_PURE_FUNCTION */ + +/** Like \ref MDBX_PURE_FUNCTION with addition `noexcept` restriction + * that is compatible to CLANG and proposed [[pure]]. */ +#if defined(__GNUC__) || \ + (__has_attribute(__pure__) && __has_attribute(__nothrow__)) +#define MDBX_NOTHROW_PURE_FUNCTION __attribute__((__pure__, __nothrow__)) +#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 +#if __has_cpp_attribute(pure) +#define MDBX_NOTHROW_PURE_FUNCTION [[pure]] +#else +#define MDBX_NOTHROW_PURE_FUNCTION +#endif +#elif defined(__cplusplus) && __has_cpp_attribute(gnu::pure) +#if __has_cpp_attribute(gnu::nothrow) +#define MDBX_NOTHROW_PURE_FUNCTION [[gnu::pure, gnu::nothrow]] +#else +#define MDBX_NOTHROW_PURE_FUNCTION [[gnu::pure]] +#endif +#elif defined(__cplusplus) && __has_cpp_attribute(pure) +#define MDBX_NOTHROW_PURE_FUNCTION [[pure]] +#else +#define MDBX_NOTHROW_PURE_FUNCTION +#endif /* MDBX_NOTHROW_PURE_FUNCTION */ + +/** Many functions do not examine any values except their arguments, + * and have no effects except the return value. Basically this is just + * slightly more strict class than the PURE attribute, since function + * is not allowed to read global memory. + * + * Note that a function that has pointer arguments and examines the + * data pointed to must not be declared const. Likewise, a function + * that calls a non-const function usually must not be const. + * It does not make sense for a const function to return void. */ +#if (defined(__GNUC__) || __has_attribute(__pure__)) && \ + (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \ + || !defined(__cplusplus) || !__has_feature(cxx_exceptions)) +#define MDBX_CONST_FUNCTION __attribute__((__const__)) +#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 +#define MDBX_CONST_FUNCTION MDBX_PURE_FUNCTION +#elif defined(__cplusplus) && __has_cpp_attribute(gnu::const) && \ + (!defined(__clang__) || !__has_feature(cxx_exceptions)) +#define MDBX_CONST_FUNCTION [[gnu::const]] +#else +#define MDBX_CONST_FUNCTION MDBX_PURE_FUNCTION +#endif /* MDBX_CONST_FUNCTION */ + +/** Like \ref MDBX_CONST_FUNCTION with addition `noexcept` restriction + * that is compatible to CLANG and future [[const]]. */ +#if defined(__GNUC__) || \ + (__has_attribute(__const__) && __has_attribute(__nothrow__)) +#define MDBX_NOTHROW_CONST_FUNCTION __attribute__((__const__, __nothrow__)) +#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 +#define MDBX_NOTHROW_CONST_FUNCTION MDBX_NOTHROW_PURE_FUNCTION +#elif defined(__cplusplus) && __has_cpp_attribute(gnu::const) +#if __has_cpp_attribute(gnu::nothrow) +#define MDBX_NOTHROW_PURE_FUNCTION [[gnu::const, gnu::nothrow]] +#else +#define MDBX_NOTHROW_PURE_FUNCTION [[gnu::const]] +#endif +#elif defined(__cplusplus) && __has_cpp_attribute(const) +#define MDBX_NOTHROW_CONST_FUNCTION [[const]] +#else +#define MDBX_NOTHROW_CONST_FUNCTION MDBX_NOTHROW_PURE_FUNCTION +#endif /* MDBX_NOTHROW_PURE_FUNCTION */ + +#ifndef MDBX_DEPRECATED /* may be predefined to avoid warnings "deprecated" */ +#ifdef __deprecated +#define MDBX_DEPRECATED __deprecated +#elif defined(__GNUC__) || __has_attribute(__deprecated__) +#define MDBX_DEPRECATED __attribute__((__deprecated__)) #elif defined(_MSC_VER) -#define __deprecated __declspec(deprecated) +#define MDBX_DEPRECATED __declspec(deprecated) #else -#define __deprecated +#define MDBX_DEPRECATED #endif -#endif /* __deprecated */ +#endif /* MDBX_DEPRECATED */ #ifndef __dll_export #if defined(_WIN32) || defined(__CYGWIN__) @@ -658,9 +352,135 @@ typedef pthread_t mdbx_tid_t; /*----------------------------------------------------------------------------*/ -/* MDBX version 0.8.2, released 2020-07-06 */ +#ifndef __cplusplus +#ifndef bool +#define bool _Bool +#endif +#ifndef true +#define true (1) +#endif +#ifndef false +#define false (0) +#endif +#endif /* bool without __cplusplus */ + +#if !defined(__cpp_noexcept_function_type) || \ + __cpp_noexcept_function_type < 201510L +#define MDBX_CXX17_NOEXCEPT +#else +#define MDBX_CXX17_NOEXCEPT noexcept +#endif /* MDBX_CXX17_NOEXCEPT */ + +/* Workaround for old compilers without properly support for constexpr. */ +#if !defined(__cplusplus) +#define MDBX_CXX01_CONSTEXPR __inline +#define MDBX_CXX01_CONSTEXPR_VAR const +#elif !defined(__cpp_constexpr) || __cpp_constexpr < 200704L || \ + (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && \ + !defined(__clang__) && !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || \ + (defined(__clang__) && __clang_major__ < 4) +#define MDBX_CXX01_CONSTEXPR inline +#define MDBX_CXX01_CONSTEXPR_VAR const +#else +#define MDBX_CXX01_CONSTEXPR constexpr +#define MDBX_CXX01_CONSTEXPR_VAR constexpr +#endif /* MDBX_CXX01_CONSTEXPR */ + +#if !defined(__cplusplus) +#define MDBX_CXX11_CONSTEXPR __inline +#define MDBX_CXX11_CONSTEXPR_VAR const +#elif !defined(__cpp_constexpr) || __cpp_constexpr < 201304 || \ + (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && __GNUC__ < 6 && !defined(__clang__) && \ + !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || \ + (defined(__clang__) && __clang_major__ < 5) +#define MDBX_CXX11_CONSTEXPR inline +#define MDBX_CXX11_CONSTEXPR_VAR const +#else +#define MDBX_CXX11_CONSTEXPR constexpr +#define MDBX_CXX11_CONSTEXPR_VAR constexpr +#endif /* MDBX_CXX11_CONSTEXPR */ + +#if !defined(__cplusplus) +#define MDBX_CXX14_CONSTEXPR __inline +#define MDBX_CXX14_CONSTEXPR_VAR const +#elif defined(__cpp_constexpr) && __cpp_constexpr >= 201304L && \ + ((defined(_MSC_VER) && _MSC_VER >= 1910) || \ + (defined(__clang__) && __clang_major__ > 4) || \ + (defined(__GNUC__) && __GNUC__ > 6) || \ + (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) +#define MDBX_CXX14_CONSTEXPR constexpr +#define MDBX_CXX14_CONSTEXPR_VAR constexpr +#else +#define MDBX_CXX14_CONSTEXPR inline +#define MDBX_CXX14_CONSTEXPR_VAR const +#endif /* MDBX_CXX14_CONSTEXPR */ + +#if defined(__noreturn) +#define MDBX_NORETURN __noreturn +#elif defined(_Noreturn) +#define MDBX_NORETURN _Noreturn +#elif defined(__GNUC__) || __has_attribute(__noreturn__) +#define MDBX_NORETURN __attribute__((__noreturn__)) +#elif defined(_MSC_VER) && !defined(__clang__) +#define MDBX_NORETURN __declspec(noreturn) +#else +#define MDBX_NORETURN +#endif /* MDBX_NORETURN */ + +#ifndef MDBX_PRINTF_ARGS +#if defined(__GNUC__) || __has_attribute(__format__) +#define MDBX_PRINTF_ARGS(format_index, first_arg) \ + __attribute__((__format__(__printf__, format_index, first_arg))) +#else +#define MDBX_PRINTF_ARGS(format_index, first_arg) +#endif +#endif /* MDBX_PRINTF_ARGS */ + +#ifndef DEFINE_ENUM_FLAG_OPERATORS +#if defined(__cplusplus) +/// Define operator overloads to enable bit operations on enum values that are +/// used to define flags (based on Microsoft's DEFINE_ENUM_FLAG_OPERATORS). +#define DEFINE_ENUM_FLAG_OPERATORS(ENUM) \ + extern "C++" { \ + MDBX_CXX01_CONSTEXPR ENUM operator|(ENUM a, ENUM b) { \ + return ENUM(std::size_t(a) | std::size_t(b)); \ + } \ + MDBX_CXX14_CONSTEXPR ENUM &operator|=(ENUM &a, ENUM b) { return a = a | b; } \ + MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, ENUM b) { \ + return ENUM(std::size_t(a) & std::size_t(b)); \ + } \ + MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, ENUM b) { return a = a & b; } \ + MDBX_CXX01_CONSTEXPR ENUM operator~(ENUM a) { \ + return ENUM(~std::size_t(a)); \ + } \ + MDBX_CXX01_CONSTEXPR ENUM operator^(ENUM a, ENUM b) { \ + return ENUM(std::size_t(a) ^ std::size_t(b)); \ + } \ + MDBX_CXX14_CONSTEXPR ENUM &operator^=(ENUM &a, ENUM b) { return a = a ^ b; } \ + } +#else /* __cplusplus */ +#define DEFINE_ENUM_FLAG_OPERATORS(ENUM) /* nope, C allows these operators */ +#endif /* !__cplusplus */ +#endif /* DEFINE_ENUM_FLAG_OPERATORS */ + +/** @} end of Common Macros */ + +/*----------------------------------------------------------------------------*/ + +/** \addtogroup c_api + * @{ */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* MDBX version 0.9.x */ #define MDBX_VERSION_MAJOR 0 -#define MDBX_VERSION_MINOR 8 +#define MDBX_VERSION_MINOR 9 #ifndef LIBMDBX_API #if defined(LIBMDBX_EXPORTS) @@ -673,10 +493,14 @@ typedef pthread_t mdbx_tid_t; #endif /* LIBMDBX_API */ #ifdef __cplusplus -extern "C" { +#if defined(__clang__) || __has_attribute(type_visibility) +#define LIBMDBX_API_TYPE LIBMDBX_API __attribute__((type_visibility("default"))) +#else +#define LIBMDBX_API_TYPE LIBMDBX_API #endif - -/**** MDBX version information ************************************************/ +#else +#define LIBMDBX_API_TYPE +#endif /* LIBMDBX_API_TYPE */ #if defined(LIBMDBX_IMPORTS) #define LIBMDBX_VERINFO_API __dll_import @@ -684,32 +508,31 @@ extern "C" { #define LIBMDBX_VERINFO_API __dll_export #endif /* LIBMDBX_VERINFO_API */ -typedef struct mdbx_version_info { - uint8_t major; - uint8_t minor; - uint16_t release; - uint32_t revision; - struct /* source info from git */ { - const char *datetime /* committer date, strict ISO-8601 format */; - const char *tree /* commit hash (hexadecimal digits) */; - const char *commit /* tree hash, i.e. digest of the source code */; - const char *describe /* git-describe string */; - } git; - const char *sourcery /* sourcery anchor for pinning */; -} mdbx_version_info; -extern LIBMDBX_VERINFO_API const mdbx_version_info mdbx_version; - -/* MDBX build information. - * WARNING: Some strings could be NULL in case no corresponding information was - * provided at build time (i.e. flags). */ -typedef struct mdbx_build_info { - const char *datetime /* build timestamp (ISO-8601 or __DATE__ __TIME__) */; - const char *target /* cpu/arch-system-config triplet */; - const char *options /* mdbx-related options */; - const char *compiler /* compiler */; - const char *flags /* CFLAGS */; -} mdbx_build_info; -extern LIBMDBX_VERINFO_API const mdbx_build_info mdbx_build; +/** \brief libmdbx version information */ +extern LIBMDBX_VERINFO_API const struct MDBX_version_info { + uint8_t major; /**< Major version number */ + uint8_t minor; /**< Minor version number */ + uint16_t release; /**< Release number of Major.Minor */ + uint32_t revision; /**< Revision number of Release */ + struct { + const char *datetime; /**< committer date, strict ISO-8601 format */ + const char *tree; /**< commit hash (hexadecimal digits) */ + const char *commit; /**< tree hash, i.e. digest of the source code */ + const char *describe; /**< git-describe string */ + } git; /**< source information from git */ + const char *sourcery; /**< sourcery anchor for pinning */ +} /** \brief libmdbx version information */ mdbx_version; + +/** \brief libmdbx build information + * \attention Some strings could be NULL in case no corresponding information + * was provided at build time (i.e. flags). */ +extern LIBMDBX_VERINFO_API const struct MDBX_build_info { + const char *datetime; /**< build timestamp (ISO-8601 or __DATE__ __TIME__) */ + const char *target; /**< cpu/arch-system-config triplet */ + const char *options; /**< mdbx-related options */ + const char *compiler; /**< compiler */ + const char *flags; /**< CFLAGS and CXXFLAGS */ +} /** \brief libmdbx build information */ mdbx_build; #if defined(_WIN32) || defined(_WIN64) #if !MDBX_BUILD_SHARED_LIBRARY @@ -758,1280 +581,1670 @@ void LIBMDBX_API NTAPI mdbx_dll_handler(PVOID module, DWORD reason, #endif /* !MDBX_BUILD_SHARED_LIBRARY */ #endif /* Windows */ -/**** OPACITY STRUCTURES ******************************************************/ +/* OPACITY STRUCTURES *********************************************************/ -/* Opaque structure for a database environment. - * - * An environment supports multiple key-value databases (aka key-value spaces - * or tables), all residing in the same shared-memory map. */ +/** \brief Opaque structure for a database environment. + * \details An environment supports multiple key-value sub-databases (aka + * key-value spaces or tables), all residing in the same shared-memory map. + * \see mdbx_env_create() \see mdbx_env_close() */ +#ifndef __cplusplus typedef struct MDBX_env MDBX_env; +#else +struct MDBX_env; +#endif -/* Opaque structure for a transaction handle. - * - * All database operations require a transaction handle. Transactions may be - * read-only or read-write. */ +/** \brief Opaque structure for a transaction handle. + * \ingroup c_transactions + * \details All database operations require a transaction handle. Transactions + * may be read-only or read-write. + * \see mdbx_txn_begin() \see mdbx_txn_commit() \see mdbx_txn_abort() */ +#ifndef __cplusplus typedef struct MDBX_txn MDBX_txn; +#else +struct MDBX_txn; +#endif -/* A handle for an individual database (key-value spaces) in the environment. - * Zero handle is used internally (hidden Garbage Collection DB). - * So, any valid DBI-handle great than 0 and less than or equal MDBX_MAX_DBI. */ +/** \brief A handle for an individual database (key-value spaces) in the + * environment. \ingroup c_dbi \details Zero handle is used internally (hidden + * Garbage Collection DB). So, any valid DBI-handle great than 0 and less than + * or equal \ref MDBX_MAX_DBI. \see mdbx_dbi_open() \see mdbx_dbi_close() */ typedef uint32_t MDBX_dbi; -#define MDBX_MAX_DBI UINT32_C(32765) -/* Opaque structure for navigating through a database */ +/** \brief Opaque structure for navigating through a database + * \ingroup c_cursors + * \see mdbx_cursor_create() \see mdbx_cursor_bind() \see mdbx_cursor_close() + */ +#ifndef __cplusplus typedef struct MDBX_cursor MDBX_cursor; +#else +struct MDBX_cursor; +#endif -/* Generic structure used for passing keys and data in and out of the database. +/** \brief Generic structure used for passing keys and data in and out of the + * database. + * \anchor MDBX_val \see mdbx::slice \see mdbx::buffer * - * Values returned from the database are valid only until a subsequent + * \details Values returned from the database are valid only until a subsequent * update operation, or the end of the transaction. Do not modify or * free them, they commonly point into the database itself. * - * Key sizes must be between 0 and mdbx_env_get_maxkeysize() inclusive. - * The same applies to data sizes in databases with the MDBX_DUPSORT flag. - * Other data items can in theory be from 0 to 0x7fffffff bytes long. + * Key sizes must be between 0 and \ref mdbx_env_get_maxkeysize() inclusive. + * The same applies to data sizes in databases with the \ref MDBX_DUPSORT flag. + * Other data items can in theory be from 0 to \ref MDBX_MAXDATASIZE bytes long. * - * (!) The notable difference between MDBX and LMDB is that MDBX support zero + * \note The notable difference between MDBX and LMDB is that MDBX support zero * length keys. */ #ifndef HAVE_STRUCT_IOVEC struct iovec { - void *iov_base /* pointer to some data */; - size_t iov_len /* the length of data in bytes */; + void *iov_base; /**< pointer to some data */ + size_t iov_len; /**< the length of data in bytes */ }; #define HAVE_STRUCT_IOVEC #endif /* HAVE_STRUCT_IOVEC */ #if defined(__sun) || defined(__SVR4) || defined(__svr4__) /* The `iov_len` is signed on Sun/Solaris. - * So define custom MDBX_val to avoid a lot of warings. */ -typedef struct MDBX_val { - void *iov_base /* pointer to some data */; - size_t iov_len /* the length of data in bytes */; -} MDBX_val; -#else -typedef struct iovec MDBX_val; + * So define custom MDBX_val to avoid a lot of warnings. */ +struct MDBX_val { + void *iov_base; /**< pointer to some data */ + size_t iov_len; /**< the length of data in bytes */ +}; +#ifndef __cplusplus +typedef struct MDBX_val MDBX_val; #endif +#else /* SunOS */ +typedef struct iovec MDBX_val; +#endif /* ! SunOS */ -/* The maximum size of a data item. - * MDBX only store a 32 bit value for node sizes. */ -#define MDBX_MAXDATASIZE INT32_MAX +enum MDBX_constants { + /** The hard limit for DBI handles */ + MDBX_MAX_DBI = UINT32_C(32765), -/**** DEBUG & LOGGING ********************************************************** - * Logging and runtime debug flags. - * - * NOTE: Most of debug feature enabled only when libmdbx builded with - * MDBX_DEBUG options. - */ + /** The maximum size of a data item. */ + MDBX_MAXDATASIZE = UINT32_C(0x7fff0000), -/* Log level (requires build libmdbx with MDBX_DEBUG) */ -#define MDBX_LOG_FATAL 0 /* critical conditions, i.e. assertion failures */ -#define MDBX_LOG_ERROR 1 /* error conditions */ -#define MDBX_LOG_WARN 2 /* warning conditions */ -#define MDBX_LOG_NOTICE 3 /* normal but significant condition */ -#define MDBX_LOG_VERBOSE 4 /* verbose informational */ -#define MDBX_LOG_DEBUG 5 /* debug-level messages */ -#define MDBX_LOG_TRACE 6 /* trace debug-level messages */ -#define MDBX_LOG_EXTRA 7 /* extra debug-level messages (dump pgno lists) */ + /** The minimal database page size in bytes. */ + MDBX_MIN_PAGESIZE = 256, -/* Runtime debug flags. + /** The maximal database page size in bytes. */ + MDBX_MAX_PAGESIZE = 65536, +}; + +/* THE FILES ******************************************************************* + * At the file system level, the environment corresponds to a pair of files. */ + +/** \brief The name of the lock file in the environment */ +#define MDBX_LOCKNAME "/mdbx.lck" +/** \brief The name of the data file in the environment */ +#define MDBX_DATANAME "/mdbx.dat" + +/** \brief The suffix of the lock file when \ref MDBX_NOSUBDIR is used */ +#define MDBX_LOCK_SUFFIX "-lck" + +/* DEBUG & LOGGING ************************************************************/ + +/** \addtogroup c_debug + * \note Most of debug feature enabled only when libmdbx builded with + * \ref MDBX_DEBUG build option. @{ */ + +/** Log level (requires build libmdbx with \ref MDBX_DEBUG option) */ +enum MDBX_log_level_t { + /** Critical conditions, i.e. assertion failures */ + MDBX_LOG_FATAL = 0, + + /** Enables logging for error conditions and \ref MDBX_LOG_FATAL */ + MDBX_LOG_ERROR = 1, + + /** Enables logging for warning conditions and \ref MDBX_LOG_ERROR ... + \ref MDBX_LOG_FATAL */ + MDBX_LOG_WARN = 2, + + /** Enables logging for normal but significant condition and + \ref MDBX_LOG_WARN ... \ref MDBX_LOG_FATAL */ + MDBX_LOG_NOTICE = 3, + + /** Enables logging for verbose informational and \ref MDBX_LOG_NOTICE ... + \ref MDBX_LOG_FATAL */ + MDBX_LOG_VERBOSE = 4, + + /** Enables logging for debug-level messages and \ref MDBX_LOG_VERBOSE ... + \ref MDBX_LOG_FATAL */ + MDBX_LOG_DEBUG = 5, + + /** Enables logging for trace debug-level messages and \ref MDBX_LOG_DEBUG ... + \ref MDBX_LOG_FATAL */ + MDBX_LOG_TRACE = 6, + + /** Enables extra debug-level messages (dump pgno lists) + and all other log-messages */ + MDBX_LOG_EXTRA = 7, + + /** for \ref mdbx_setup_debug() only: Don't change current settings */ + MDBX_LOG_DONTCHANGE = -1 +}; +#ifndef __cplusplus +typedef enum MDBX_log_level_t MDBX_log_level_t; +#endif + +/** \brief Runtime debug flags * - * MDBX_DBG_DUMP and MDBX_DBG_LEGACY_MULTIOPEN always have an effect, - * but MDBX_DBG_ASSERT, MDBX_DBG_AUDIT and MDBX_DBG_JITTER only if libmdbx - * builded with MDBX_DEBUG. */ + * \details `MDBX_DBG_DUMP` and `MDBX_DBG_LEGACY_MULTIOPEN` always have an + * effect, but `MDBX_DBG_ASSERT`, `MDBX_DBG_AUDIT` and `MDBX_DBG_JITTER` only if + * libmdbx builded with \ref MDBX_DEBUG. */ +enum MDBX_debug_flags_t { + /** Enable assertion checks. + * Requires build with \ref MDBX_DEBUG > 0 */ + MDBX_DBG_ASSERT = 1, -/* Enable assertion checks */ -#define MDBX_DBG_ASSERT 1 + /** Enable pages usage audit at commit transactions. + * Requires build with \ref MDBX_DEBUG > 0 */ + MDBX_DBG_AUDIT = 2, -/* Enable pages usage audit at commit transactions */ -#define MDBX_DBG_AUDIT 2 + /** Enable small random delays in critical points. + * Requires build with \ref MDBX_DEBUG > 0 */ + MDBX_DBG_JITTER = 4, -/* Enable small random delays in critical points */ -#define MDBX_DBG_JITTER 4 + /** Include or not meta-pages in coredump files. + * May affect performance in \ref MDBX_WRITEMAP mode */ + MDBX_DBG_DUMP = 8, -/* Include or not meta-pages in coredump files, - * MAY affect performance in MDBX_WRITEMAP mode */ -#define MDBX_DBG_DUMP 8 + /** Allow multi-opening environment(s) */ + MDBX_DBG_LEGACY_MULTIOPEN = 16, -/* Allow multi-opening environment(s) */ -#define MDBX_DBG_LEGACY_MULTIOPEN 16 + /** Allow read and write transactions overlapping for the same thread */ + MDBX_DBG_LEGACY_OVERLAP = 32, -/* Allow read and write transactions overlapping for the same thread */ -#define MDBX_DBG_LEGACY_OVERLAP 32 + /** for mdbx_setup_debug() only: Don't change current settings */ + MDBX_DBG_DONTCHANGE = -1 +}; +#ifndef __cplusplus +typedef enum MDBX_debug_flags_t MDBX_debug_flags_t; +#else +DEFINE_ENUM_FLAG_OPERATORS(MDBX_debug_flags_t) +#endif -/* A debug-logger callback function, +/** \brief A debug-logger callback function, * called before printing the message and aborting. + * \see mdbx_setup_debug() * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] msg The assertion message, not including newline. */ -typedef void MDBX_debug_func(int loglevel, const char *function, int line, - const char *msg, va_list args); + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] msg The assertion message, not including newline. */ +typedef void MDBX_debug_func(MDBX_log_level_t loglevel, const char *function, + int line, const char *msg, + va_list args) MDBX_CXX17_NOEXCEPT; -/* Don't change current settings */ -#define MDBX_LOG_DONTCHANGE (-1) -#define MDBX_DBG_DONTCHANGE (-1) +/** \brief The "don't change `logger`" value for mdbx_setup_debug() */ #define MDBX_LOGGER_DONTCHANGE ((MDBX_debug_func *)(intptr_t)-1) -/* Setup global log-level, debug options and debug logger. */ -LIBMDBX_API int mdbx_setup_debug(int loglevel, int flags, +/** \brief Setup global log-level, debug options and debug logger. + * \returns The previously `debug_flags` in the 0-15 bits + * and `log_level` in the 16-31 bits. */ +LIBMDBX_API int mdbx_setup_debug(MDBX_log_level_t log_level, + MDBX_debug_flags_t debug_flags, MDBX_debug_func *logger); -/* A callback function for most MDBX assert() failures, +/** \brief A callback function for most MDBX assert() failures, * called before printing the message and aborting. + * \see mdbx_env_set_assert() * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] msg The assertion message, not including newline. */ + * \param [in] env An environment handle returned by mdbx_env_create(). + * \param [in] msg The assertion message, not including newline. */ typedef void MDBX_assert_func(const MDBX_env *env, const char *msg, - const char *function, unsigned line); + const char *function, + unsigned line) MDBX_CXX17_NOEXCEPT; -/* Set or reset the assert() callback of the environment. +/** \brief Set or reset the assert() callback of the environment. * * Does nothing if libmdbx was built with MDBX_DEBUG=0 or with NDEBUG, - * and will return MDBX_ENOSYS in such case. + * and will return `MDBX_ENOSYS` in such case. * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] func An MDBX_assert_func function, or 0. + * \param [in] env An environment handle returned by mdbx_env_create(). + * \param [in] func An MDBX_assert_func function, or 0. * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func); -/* FIXME: Complete description */ +/** \brief Dump given MDBX_val to the buffer + * + * Dumps it as string if value is printable (all bytes in the range 0x20..0x7E), + * otherwise made hexadecimal dump. Requires at least 4 byte length buffer. + * + * \returns One of: + * - NULL if given buffer size less than 4 bytes; + * - pointer to constant string if given value NULL or empty; + * - otherwise pointer to given buffer. */ LIBMDBX_API const char *mdbx_dump_val(const MDBX_val *key, char *const buf, const size_t bufsize); -/**** THE FILES **************************************************************** - * At the file system level, the environment corresponds to a pair of files. */ +/** \brief Panics with message and causes abnormal process termination. */ +LIBMDBX_API void mdbx_panic(const char *fmt, ...) MDBX_PRINTF_ARGS(1, 2); + +/** @} end of logging & debug */ + +/** \brief Environment flags + * \ingroup c_opening + * \anchor env_flags + * \see mdbx_env_open() \see mdbx_env_set_flags() */ +enum MDBX_env_flags_t { + MDBX_ENV_DEFAULTS = 0, + + /** No environment directory. + * + * By default, MDBX creates its environment in a directory whose pathname is + * given in path, and creates its data and lock files under that directory. + * With this option, path is used as-is for the database main data file. + * The database lock file is the path with "-lck" appended. + * + * - with `MDBX_NOSUBDIR` = in a filesystem we have the pair of MDBX-files + * which names derived from given pathname by appending predefined suffixes. + * + * - without `MDBX_NOSUBDIR` = in a filesystem we have the MDBX-directory with + * given pathname, within that a pair of MDBX-files with predefined names. + * + * This flag affects only at new environment creating by \ref mdbx_env_open(), + * otherwise at opening an existing environment libmdbx will choice this + * automatically. */ + MDBX_NOSUBDIR = UINT32_C(0x4000), + + /** Read only mode. + * + * Open the environment in read-only mode. No write operations will be + * allowed. MDBX will still modify the lock file - except on read-only + * filesystems, where MDBX does not use locks. + * + * - with `MDBX_RDONLY` = open environment in read-only mode. + * MDBX supports pure read-only mode (i.e. without opening LCK-file) only + * when environment directory and/or both files are not writable (and the + * LCK-file may be missing). In such case allowing file(s) to be placed + * on a network read-only share. + * + * - without `MDBX_RDONLY` = open environment in read-write mode. + * + * This flag affects only at environment opening but can't be changed after. + */ + MDBX_RDONLY = UINT32_C(0x20000), + + /** Open environment in exclusive/monopolistic mode. + * + * `MDBX_EXCLUSIVE` flag can be used as a replacement for `MDB_NOLOCK`, + * which don't supported by MDBX. + * In this way, you can get the minimal overhead, but with the correct + * multi-process and multi-thread locking. + * + * - with `MDBX_EXCLUSIVE` = open environment in exclusive/monopolistic mode + * or return \ref MDBX_BUSY if environment already used by other process. + * The main feature of the exclusive mode is the ability to open the + * environment placed on a network share. + * + * - without `MDBX_EXCLUSIVE` = open environment in cooperative mode, + * i.e. for multi-process access/interaction/cooperation. + * The main requirements of the cooperative mode are: + * + * 1. data files MUST be placed in the LOCAL file system, + * but NOT on a network share. + * 2. environment MUST be opened only by LOCAL processes, + * but NOT over a network. + * 3. OS kernel (i.e. file system and memory mapping implementation) and + * all processes that open the given environment MUST be running + * in the physically single RAM with cache-coherency. The only + * exception for cache-consistency requirement is Linux on MIPS + * architecture, but this case has not been tested for a long time). + * + * This flag affects only at environment opening but can't be changed after. + */ + MDBX_EXCLUSIVE = UINT32_C(0x400000), + + /** Using database/environment which already opened by another process(es). + * + * The `MDBX_ACCEDE` flag is useful to avoid \ref MDBX_INCOMPATIBLE error + * while opening the database/environment which is already used by another + * process(es) with unknown mode/flags. In such cases, if there is a + * difference in the specified flags (\ref MDBX_NOMETASYNC, + * \ref MDBX_SAFE_NOSYNC, \ref MDBX_UTTERLY_NOSYNC, \ref MDBX_LIFORECLAIM, + * \ref MDBX_COALESCE and \ref MDBX_NORDAHEAD), instead of returning an error, + * the database will be opened in a compatibility with the already used mode. + * + * `MDBX_ACCEDE` has no effect if the current process is the only one either + * opening the DB in read-only mode or other process(es) uses the DB in + * read-only mode. */ + MDBX_ACCEDE = UINT32_C(0x40000000), + + /** Map data into memory with write permission. + * + * Use a writeable memory map unless \ref MDBX_RDONLY is set. This uses fewer + * mallocs and requires much less work for tracking database pages, but + * loses protection from application bugs like wild pointer writes and other + * bad updates into the database. This may be slightly faster for DBs that + * fit entirely in RAM, but is slower for DBs larger than RAM. Also adds the + * possibility for stray application writes thru pointers to silently + * corrupt the database. + * + * - with `MDBX_WRITEMAP` = all data will be mapped into memory in the + * read-write mode. This offers a significant performance benefit, since the + * data will be modified directly in mapped memory and then flushed to disk + * by single system call, without any memory management nor copying. + * + * - without `MDBX_WRITEMAP` = data will be mapped into memory in the + * read-only mode. This requires stocking all modified database pages in + * memory and then writing them to disk through file operations. + * + * \warning On the other hand, `MDBX_WRITEMAP` adds the possibility for stray + * application writes thru pointers to silently corrupt the database. + * + * \note The `MDBX_WRITEMAP` mode is incompatible with nested transactions, + * since this is unreasonable. I.e. nested transactions requires mallocation + * of database pages and more work for tracking ones, which neuters a + * performance boost caused by the `MDBX_WRITEMAP` mode. + * + * This flag affects only at environment opening but can't be changed after. + */ + MDBX_WRITEMAP = UINT32_C(0x80000), + + /** Tie reader locktable slots to read-only transactions + * instead of to threads. + * + * Don't use Thread-Local Storage, instead tie reader locktable slots to + * \ref MDBX_txn objects instead of to threads. So, \ref mdbx_txn_reset() + * keeps the slot reserved for the \ref MDBX_txn object. A thread may use + * parallel read-only transactions. And a read-only transaction may span + * threads if you synchronizes its use. + * + * Applications that multiplex many user threads over individual OS threads + * need this option. Such an application must also serialize the write + * transactions in an OS thread, since MDBX's write locking is unaware of + * the user threads. + * + * \note Regardless to `MDBX_NOTLS` flag a write transaction entirely should + * always be used in one thread from start to finish. MDBX checks this in a + * reasonable manner and return the \ref MDBX_THREAD_MISMATCH error in rules + * violation. + * + * This flag affects only at environment opening but can't be changed after. + */ + MDBX_NOTLS = UINT32_C(0x200000), + + /** Don't do readahead. + * + * Turn off readahead. Most operating systems perform readahead on read + * requests by default. This option turns it off if the OS supports it. + * Turning it off may help random read performance when the DB is larger + * than RAM and system RAM is full. + * + * By default libmdbx dynamically enables/disables readahead depending on + * the actual database size and currently available memory. On the other + * hand, such automation has some limitation, i.e. could be performed only + * when DB size changing but can't tracks and reacts changing a free RAM + * availability, since it changes independently and asynchronously. + * + * \note The mdbx_is_readahead_reasonable() function allows to quickly find + * out whether to use readahead or not based on the size of the data and the + * amount of available memory. + * + * This flag affects only at environment opening and can't be changed after. + */ + MDBX_NORDAHEAD = UINT32_C(0x800000), + + /** Don't initialize malloc'd memory before writing to datafile. + * + * Don't initialize malloc'd memory before writing to unused spaces in the + * data file. By default, memory for pages written to the data file is + * obtained using malloc. While these pages may be reused in subsequent + * transactions, freshly malloc'd pages will be initialized to zeroes before + * use. This avoids persisting leftover data from other code (that used the + * heap and subsequently freed the memory) into the data file. + * + * Note that many other system libraries may allocate and free memory from + * the heap for arbitrary uses. E.g., stdio may use the heap for file I/O + * buffers. This initialization step has a modest performance cost so some + * applications may want to disable it using this flag. This option can be a + * problem for applications which handle sensitive data like passwords, and + * it makes memory checkers like Valgrind noisy. This flag is not needed + * with \ref MDBX_WRITEMAP, which writes directly to the mmap instead of using + * malloc for pages. The initialization is also skipped if \ref MDBX_RESERVE + * is used; the caller is expected to overwrite all of the memory that was + * reserved in that case. + * + * This flag may be changed at any time using `mdbx_env_set_flags()`. */ + MDBX_NOMEMINIT = UINT32_C(0x1000000), + + /** Aims to coalesce a Garbage Collection items. + * + * With `MDBX_COALESCE` flag MDBX will aims to coalesce items while recycling + * a Garbage Collection. Technically, when possible short lists of pages + * will be combined into longer ones, but to fit on one database page. As a + * result, there will be fewer items in Garbage Collection and a page lists + * are longer, which slightly increases the likelihood of returning pages to + * Unallocated space and reducing the database file. + * + * This flag may be changed at any time using mdbx_env_set_flags(). */ + MDBX_COALESCE = UINT32_C(0x2000000), + + /** LIFO policy for recycling a Garbage Collection items. + * + * `MDBX_LIFORECLAIM` flag turns on LIFO policy for recycling a Garbage + * Collection items, instead of FIFO by default. On systems with a disk + * write-back cache, this can significantly increase write performance, up + * to several times in a best case scenario. + * + * LIFO recycling policy means that for reuse pages will be taken which became + * unused the lastest (i.e. just now or most recently). Therefore the loop of + * database pages circulation becomes as short as possible. In other words, + * the number of pages, that are overwritten in memory and on disk during a + * series of write transactions, will be as small as possible. Thus creates + * ideal conditions for the efficient operation of the disk write-back cache. + * + * \ref MDBX_LIFORECLAIM is compatible with all no-sync flags, but gives NO + * noticeable impact in combination with \ref MDBX_SAFE_NOSYNC or + * \ref MDBX_UTTERLY_NOSYNC. Because MDBX will reused pages only before the + * last "steady" MVCC-snapshot, i.e. the loop length of database pages + * circulation will be mostly defined by frequency of calling + * \ref mdbx_env_sync() rather than LIFO and FIFO difference. + * + * This flag may be changed at any time using mdbx_env_set_flags(). */ + MDBX_LIFORECLAIM = UINT32_C(0x4000000), + + /** Debugging option, fill/perturb released pages. */ + MDBX_PAGEPERTURB = UINT32_C(0x8000000), + + /* SYNC MODES****************************************************************/ + /** \defgroup sync_modes SYNC MODES + * + * \attention Using any combination of \ref MDBX_SAFE_NOSYNC, \ref + * MDBX_NOMETASYNC and especially \ref MDBX_UTTERLY_NOSYNC is always a deal to + * reduce durability for gain write performance. You must know exactly what + * you are doing and what risks you are taking! + * + * \note for LMDB users: \ref MDBX_SAFE_NOSYNC is NOT similar to LMDB_NOSYNC, + * but \ref MDBX_UTTERLY_NOSYNC is exactly match LMDB_NOSYNC. See details + * below. + * + * THE SCENE: + * - The DAT-file contains several MVCC-snapshots of B-tree at same time, + * each of those B-tree has its own root page. + * - Each of meta pages at the beginning of the DAT file contains a + * pointer to the root page of B-tree which is the result of the particular + * transaction, and a number of this transaction. + * - For data durability, MDBX must first write all MVCC-snapshot data + * pages and ensure that are written to the disk, then update a meta page + * with the new transaction number and a pointer to the corresponding new + * root page, and flush any buffers yet again. + * - Thus during commit a I/O buffers should be flushed to the disk twice; + * i.e. fdatasync(), FlushFileBuffers() or similar syscall should be + * called twice for each commit. This is very expensive for performance, + * but guaranteed durability even on unexpected system failure or power + * outage. Of course, provided that the operating system and the + * underlying hardware (e.g. disk) work correctly. + * + * TRADE-OFF: + * By skipping some stages described above, you can significantly benefit in + * speed, while partially or completely losing in the guarantee of data + * durability and/or consistency in the event of system or power failure. + * Moreover, if for any reason disk write order is not preserved, then at + * moment of a system crash, a meta-page with a pointer to the new B-tree may + * be written to disk, while the itself B-tree not yet. In that case, the + * database will be corrupted! + * + * \see MDBX_SYNC_DURABLE \see MDBX_NOMETASYNC \see MDBX_SAFE_NOSYNC + * \see MDBX_UTTERLY_NOSYNC + * + * @{ */ + + /** Default robust and durable sync mode. + * + * Metadata is written and flushed to disk after a data is written and + * flushed, which guarantees the integrity of the database in the event + * of a crash at any time. + * + * \attention Please do not use other modes until you have studied all the + * details and are sure. Otherwise, you may lose your users' data, as happens + * in [Miranda NG](https://www.miranda-ng.org/) messenger. */ + MDBX_SYNC_DURABLE = 0, + + /** Don't sync the meta-page after commit. + * + * Flush system buffers to disk only once per transaction commit, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-\ref MDBX_RDONLY commit or \ref mdbx_env_sync(). Depending on + * the platform and hardware, with \ref MDBX_NOMETASYNC you may get a doubling + * of write performance. + * + * This trade-off maintains database integrity, but a system crash may + * undo the last committed transaction. I.e. it preserves the ACI + * (atomicity, consistency, isolation) but not D (durability) database + * property. + * + * `MDBX_NOMETASYNC` flag may be changed at any time using + * \ref mdbx_env_set_flags() or by passing to \ref mdbx_txn_begin() for + * particular write transaction. \see sync_modes */ + MDBX_NOMETASYNC = UINT32_C(0x40000), + + /** Don't sync anything but keep previous steady commits. + * + * Like \ref MDBX_UTTERLY_NOSYNC the `MDBX_SAFE_NOSYNC` flag similarly disable + * flush system buffers to disk when committing a transaction. But there is a + * huge difference in how are recycled the MVCC snapshots corresponding to + * previous "steady" transactions (see below). + * + * With \ref MDBX_WRITEMAP the `MDBX_SAFE_NOSYNC` instructs MDBX to use + * asynchronous mmap-flushes to disk. Asynchronous mmap-flushes means that + * actually all writes will scheduled and performed by operation system on it + * own manner, i.e. unordered. MDBX itself just notify operating system that + * it would be nice to write data to disk, but no more. + * + * Depending on the platform and hardware, with `MDBX_SAFE_NOSYNC` you may get + * a multiple increase of write performance, even 10 times or more. + * + * In contrast to \ref MDBX_UTTERLY_NOSYNC mode, with `MDBX_SAFE_NOSYNC` flag + * MDBX will keeps untouched pages within B-tree of the last transaction + * "steady" which was synced to disk completely. This has big implications for + * both data durability and (unfortunately) performance: + * - a system crash can't corrupt the database, but you will lose the last + * transactions; because MDBX will rollback to last steady commit since it + * kept explicitly. + * - the last steady transaction makes an effect similar to "long-lived" read + * transaction (see above in the \ref restrictions section) since prevents + * reuse of pages freed by newer write transactions, thus the any data + * changes will be placed in newly allocated pages. + * - to avoid rapid database growth, the system will sync data and issue + * a steady commit-point to resume reuse pages, each time there is + * insufficient space and before increasing the size of the file on disk. + * + * In other words, with `MDBX_SAFE_NOSYNC` flag MDBX insures you from the + * whole database corruption, at the cost increasing database size and/or + * number of disk IOPs. So, `MDBX_SAFE_NOSYNC` flag could be used with + * \ref mdbx_env_sync() as alternatively for batch committing or nested + * transaction (in some cases). As well, auto-sync feature exposed by + * \ref mdbx_env_set_syncbytes() and \ref mdbx_env_set_syncperiod() functions + * could be very useful with `MDBX_SAFE_NOSYNC` flag. + * + * The number and volume of of disk IOPs with MDBX_SAFE_NOSYNC flag will + * exactly the as without any no-sync flags. However, you should expect a + * larger process's [work set](https://bit.ly/2kA2tFX) and significantly worse + * a [locality of reference](https://bit.ly/2mbYq2J), due to the more + * intensive allocation of previously unused pages and increase the size of + * the database. + * + * `MDBX_SAFE_NOSYNC` flag may be changed at any time using + * \ref mdbx_env_set_flags() or by passing to \ref mdbx_txn_begin() for + * particular write transaction. */ + MDBX_SAFE_NOSYNC = UINT32_C(0x10000), + + /** \deprecated Please use \ref MDBX_SAFE_NOSYNC instead of `MDBX_MAPASYNC`. + * + * Since version 0.9.x the `MDBX_MAPASYNC` is deprecated and has the same + * effect as \ref MDBX_SAFE_NOSYNC with \ref MDBX_WRITEMAP. This just API + * simplification is for convenience and clarity. */ + MDBX_MAPASYNC = MDBX_SAFE_NOSYNC, + + /** Don't sync anything and wipe previous steady commits. + * + * Don't flush system buffers to disk when committing a transaction. This + * optimization means a system crash can corrupt the database, if buffers are + * not yet flushed to disk. Depending on the platform and hardware, with + * `MDBX_UTTERLY_NOSYNC` you may get a multiple increase of write performance, + * even 100 times or more. + * + * If the filesystem preserves write order (which is rare and never provided + * unless explicitly noted) and the \ref MDBX_WRITEMAP and \ref + * MDBX_LIFORECLAIM flags are not used, then a system crash can't corrupt the + * database, but you can lose the last transactions, if at least one buffer is + * not yet flushed to disk. The risk is governed by how often the system + * flushes dirty buffers to disk and how often \ref mdbx_env_sync() is called. + * So, transactions exhibit ACI (atomicity, consistency, isolation) properties + * and only lose `D` (durability). I.e. database integrity is maintained, but + * a system crash may undo the final transactions. + * + * Otherwise, if the filesystem not preserves write order (which is + * typically) or \ref MDBX_WRITEMAP or \ref MDBX_LIFORECLAIM flags are used, + * you should expect the corrupted database after a system crash. + * + * So, most important thing about `MDBX_UTTERLY_NOSYNC`: + * - a system crash immediately after commit the write transaction + * high likely lead to database corruption. + * - successful completion of mdbx_env_sync(force = true) after one or + * more commited transactions guarantees consistency and durability. + * - BUT by committing two or more transactions you back database into + * a weak state, in which a system crash may lead to database corruption! + * In case single transaction after mdbx_env_sync, you may lose transaction + * itself, but not a whole database. + * + * Nevertheless, `MDBX_UTTERLY_NOSYNC` provides "weak" durability in case + * of an application crash (but no durability on system failure), and + * therefore may be very useful in scenarios where data durability is + * not required over a system failure (e.g for short-lived data), or if you + * can take such risk. + * + * `MDBX_UTTERLY_NOSYNC` flag may be changed at any time using + * \ref mdbx_env_set_flags(), but don't has effect if passed to + * \ref mdbx_txn_begin() for particular write transaction. \see sync_modes */ + MDBX_UTTERLY_NOSYNC = MDBX_SAFE_NOSYNC | UINT32_C(0x100000), + + /** @} end of SYNC MODES */ +}; +#ifndef __cplusplus +/** \ingroup c_opening */ +typedef enum MDBX_env_flags_t MDBX_env_flags_t; +#else +DEFINE_ENUM_FLAG_OPERATORS(MDBX_env_flags_t) +#endif -/* The name of the lock file in the environment */ -#define MDBX_LOCKNAME "/mdbx.lck" -/* The name of the data file in the environment */ -#define MDBX_DATANAME "/mdbx.dat" +/** Transaction flags + * \ingroup c_transactions + * \anchor txn_flags + * \see mdbx_txn_begin() \see mdbx_txn_flags() */ +enum MDBX_txn_flags_t { + /** Start read-write transaction. + * + * Only one write transaction may be active at a time. Writes are fully + * serialized, which guarantees that writers can never deadlock. */ + MDBX_TXN_READWRITE = 0, + + /** Start read-only transaction. + * + * There can be multiple read-only transactions simultaneously that do not + * block each other and a write transactions. */ + MDBX_TXN_RDONLY = MDBX_RDONLY, + +/** Prepare but not start read-only transaction. + * + * Transaction will not be started immediately, but created transaction handle + * will be ready for use with \ref mdbx_txn_renew(). This flag allows to + * preallocate memory and assign a reader slot, thus avoiding these operations + * at the next start of the transaction. */ +#if defined(__cplusplus) && !defined(__cpp_constexpr) && !defined(DOXYGEN) + MDBX_TXN_RDONLY_PREPARE = uint32_t(MDBX_RDONLY) | uint32_t(MDBX_NOMEMINIT), +#else + MDBX_TXN_RDONLY_PREPARE = MDBX_RDONLY | MDBX_NOMEMINIT, +#endif -/* The suffix of the lock file when MDBX_NOSUBDIR is used */ -#define MDBX_LOCK_SUFFIX "-lck" + /** Do not block when starting a write transaction. */ + MDBX_TXN_TRY = UINT32_C(0x10000000), -/**** ENVIRONMENT FLAGS *******************************************************/ - -/* MDBX_NOSUBDIR = no environment directory. - * - * By default, MDBX creates its environment in a directory whose pathname is - * given in path, and creates its data and lock files under that directory. - * With this option, path is used as-is for the database main data file. - * The database lock file is the path with "-lck" appended. - * - * - with MDBX_NOSUBDIR = in a filesystem we have the pair of MDBX-files which - * names derived from given pathname by appending predefined suffixes. - * - * - without MDBX_NOSUBDIR = in a filesystem we have the MDBX-directory with - * given pathname, within that a pair of MDBX-files with predefined names. - * - * This flag affects only at new environment creating by mdbx_env_open(), - * otherwise at opening an existing environment libmdbx will choice this - * automatically. */ -#define MDBX_NOSUBDIR 0x4000u - -/* MDBX_RDONLY = read only mode. - * - * Open the environment in read-only mode. No write operations will be allowed. - * MDBX will still modify the lock file - except on read-only filesystems, - * where MDBX does not use locks. - * - * - with MDBX_RDONLY = open environment in read-only mode. - * MDBX supports pure read-only mode (i.e. without opening LCK-file) only - * when environment directory and/or both files are not writable (and the - * LCK-file may be missing). In such case allowing file(s) to be placed - * on a network read-only share. - * - * - without MDBX_RDONLY = open environment in read-write mode. - * - * This flag affects only at environment opening but can't be changed after. */ -#define MDBX_RDONLY 0x20000u - -/* MDBX_EXCLUSIVE = open environment in exclusive/monopolistic mode. - * - * MDBX_EXCLUSIVE flag can be used as a replacement for MDB_NOLOCK, which don't - * supported by MDBX. In this way, you can get the minimal overhead, but with - * the correct multi-process and mutli-thread locking. - * - * - with MDBX_EXCLUSIVE = open environment in exclusive/monopolistic mode - * or return MDBX_BUSY if environment already used by other process. - * The main feature of the exclusive mode is the ability to open the - * environment placed on a network share. - * - * - without MDBX_EXCLUSIVE = open environment in cooperative mode, - * i.e. for multi-process access/interaction/cooperation. - * The main requirements of the cooperative mode are: - * 1. data files MUST be placed in the LOCAL file system, - * but NOT on a network share. - * 2. environment MUST be opened only by LOCAL processes, - * but NOT over a network. - * 3. OS kernel (i.e. file system and memory mapping implementation) and - * all processes that open the given environment MUST be running - * in the physically single RAM with cache-coherency. The only - * exception for cache-consistency requirement is Linux on MIPS - * architecture, but this case has not been tested for a long time). - - * This flag affects only at environment opening but can't be changed after. */ -#define MDBX_EXCLUSIVE 0x400000u - -/* MDBX_ACCEDE = using database which already opened by another process(es). - * - * The MDBX_ACCEDE flag avoid MDBX_INCOMPATIBLE error while opening If the - * database is already used by another process(es) and environment mode/flags - * isn't compatible. In such cases, when using the MDBX_ACCEDE flag, instead of - * the specified incompatible options, the mode in which the database is already - * opened by other processes will be used, including MDBX_LIFORECLAIM, - * MDBX_COALESCE and MDBX_NORDAHEAD. The MDBX_ACCEDE flag is useful to open a - * database that already used by another process(es) and used mode/flags isn't - * known. - * - * MDBX_ACCEDE has no effect if the current process is the only one either - * opening the DB in read-only mode or other process(es) uses the DB in - * read-only mode. */ -#define MDBX_ACCEDE 0x40000000u - -/* MDBX_WRITEMAP = map data into memory with write permission. - * - * Use a writeable memory map unless MDBX_RDONLY is set. This uses fewer mallocs - * and requires much less work for tracking database pages, but loses protection - * from application bugs like wild pointer writes and other bad updates into the - * database. This may be slightly faster for DBs that fit entirely in RAM, but - * is slower for DBs larger than RAM. Also adds the possibility for stray - * application writes thru pointers to silently corrupt the database. - * Incompatible with nested transactions. - * - * NOTE: The MDBX_WRITEMAP mode is incompatible with nested transactions, since - * this is unreasonable. I.e. nested transactions requires mallocation of - * database pages and more work for tracking ones, which neuters a - * performance boost caused by the MDBX_WRITEMAP mode. - * - * - with MDBX_WRITEMAP = all data will be mapped into memory in the read-write - * mode. This offers a significant performance benefit, since the data will - * be modified directly in mapped memory and then flushed to disk by - * single system call, without any memory management nor copying. - * (!) On the other hand, MDBX_WRITEMAP adds the possibility for stray - * application writes thru pointers to silently corrupt the database. - * Moreover, MDBX_WRITEMAP disallows nested write transactions. - * - * - without MDBX_WRITEMAP = data will be mapped into memory in the read-only - * mode. This requires stocking all modified database pages in memory and - * then writing them to disk through file operations. - * - * This flag affects only at environment opening but can't be changed after. */ -#define MDBX_WRITEMAP 0x80000u - -/* MDBX_NOTLS = tie reader locktable slots to read-only transactions instead - * of to threads. - * - * Don't use Thread-Local Storage, instead tie reader locktable slots to - * MDBX_txn objects instead of to threads. So, mdbx_txn_reset() keeps the slot - * reserved for the MDBX_txn object. A thread may use parallel read-only - * transactions. And a read-only transaction may span threads if you - * synchronizes its use. - * - * Applications that multiplex many user threads over individual OS threads need - * this option. Such an application must also serialize the write transactions - * in an OS thread, since MDBX's write locking is unaware of the user threads. - * - * NOTE: Regardless to MDBX_NOTLS flag a write transaction entirely should - * always be used in one thread from start to finish. MDBX checks this in a - * reasonable manner and return the MDBX_THREAD_MISMATCH error in rules - * violation. - * - * This flag affects only at environment opening but can't be changed after. */ -#define MDBX_NOTLS 0x200000u - -/* MDBX_NORDAHEAD = don't do readahead. - * - * Turn off readahead. Most operating systems perform readahead on read requests - * by default. This option turns it off if the OS supports it. Turning it off - * may help random read performance when the DB is larger than RAM and system - * RAM is full. - * - * By default libmdbx dynamically enables/disables readahead depending on the - * actual database size and currently available memory. On the other hand, such - * automation has some limitation, i.e. could be performed only when DB size - * changing but can't tracks and reacts changing a free RAM availability, since - * it changes independently and asynchronously. - * - * NOTE: The mdbx_is_readahead_reasonable() function allows to quickly find out - * whether to use readahead or not based on the size of the data and the - * amount of available memory. - * - * This flag affects only at environment opening and can't be changed after. */ -#define MDBX_NORDAHEAD 0x800000u - -/* MDBX_NOMEMINIT = don't initialize malloc'd memory before writing to datafile. - * - * Don't initialize malloc'd memory before writing to unused spaces in the data - * file. By default, memory for pages written to the data file is obtained using - * malloc. While these pages may be reused in subsequent transactions, freshly - * malloc'd pages will be initialized to zeroes before use. This avoids - * persisting leftover data from other code (that used the heap and subsequently - * freed the memory) into the data file. - * - * Note that many other system libraries may allocate and free memory from the - * heap for arbitrary uses. E.g., stdio may use the heap for file I/O buffers. - * This initialization step has a modest performance cost so some applications - * may want to disable it using this flag. This option can be a problem for - * applications which handle sensitive data like passwords, and it makes memory - * checkers like Valgrind noisy. This flag is not needed with MDBX_WRITEMAP, - * which writes directly to the mmap instead of using malloc for pages. The - * initialization is also skipped if MDBX_RESERVE is used; the caller is - * expected to overwrite all of the memory that was reserved in that case. - * - * This flag may be changed at any time using mdbx_env_set_flags(). */ -#define MDBX_NOMEMINIT 0x1000000u - -/* MDBX_COALESCE = aims to coalesce a Garbage Collection items. - * - * With MDBX_COALESCE flag MDBX will aims to coalesce items while recycling - * a Garbage Collection. Technically, when possible short lists of pages will - * be combined into longer ones, but to fit on one database page. As a result, - * there will be fewer items in Garbage Collection and a page lists are longer, - * which slightly increases the likelihood of returning pages to Unallocated - * space and reducing the database file. - * - * This flag may be changed at any time using mdbx_env_set_flags(). */ -#define MDBX_COALESCE 0x2000000u - -/* MDBX_LIFORECLAIM = LIFO policy for recycling a Garbage Collection items. - * - * MDBX_LIFORECLAIM flag turns on LIFO policy for recycling a Garbage - * Collection items, instead of FIFO by default. On systems with a disk - * write-back cache, this can significantly increase write performance, up to - * several times in a best case scenario. - * - * LIFO recycling policy means that for reuse pages will be taken which became - * unused the lastest (i.e. just now or most recently). Therefore the loop of - * database pages circulation becomes as short as possible. In other words, the - * number of pages, that are overwritten in memory and on disk during a series - * of write transactions, will be as small as possible. Thus creates ideal - * conditions for the efficient operation of the disk write-back cache. - * - * MDBX_LIFORECLAIM is compatible with all no-sync flags (i.e. MDBX_NOMETASYNC, - * MDBX_SAFE_NOSYNC, MDBX_UTTERLY_NOSYNC, MDBX_MAPASYNC), but gives no - * noticeable impact in combination with MDBX_SAFE_NOSYNC. Because MDBX will - * reused pages only before the last "steady" MVCC-snapshot, i.e. the loop - * length of database pages circulation will be mostly defined by frequency of - * calling mdbx_env_sync() rather than LIFO and FIFO difference. - * - * This flag may be changed at any time using mdbx_env_set_flags(). */ -#define MDBX_LIFORECLAIM 0x4000000u - -/* Debugging option, fill/perturb released pages. */ -#define MDBX_PAGEPERTURB 0x8000000u - -/**** SYNC MODES *************************************************************** - * (!!!) Using any combination of MDBX_SAFE_NOSYNC, MDBX_NOMETASYNC, - * MDBX_MAPASYNC and especially MDBX_UTTERLY_NOSYNC is always a deal to reduce - * durability for gain write performance. You must know exactly what you are - * doing and what risks you are taking! - * - * NOTE for LMDB users: MDBX_SAFE_NOSYNC is NOT similar to LMDB_NOSYNC, but - * MDBX_UTTERLY_NOSYNC is exactly match LMDB_NOSYNC. - * See details below. - * - * THE SCENE: - * - The DAT-file contains several MVCC-snapshots of B-tree at same time, - * each of those B-tree has its own root page. - * - Each of meta pages at the beginning of the DAT file contains a pointer - * to the root page of B-tree which is the result of the particular - * transaction, and a number of this transaction. - * - For data durability, MDBX must first write all MVCC-snapshot data pages - * and ensure that are written to the disk, then update a meta page with - * the new transaction number and a pointer to the corresponding new root - * page, and flush any buffers yet again. - * - Thus during commit a I/O buffers should be flushed to the disk twice; - * i.e. fdatasync(), FlushFileBuffers() or similar syscall should be called - * twice for each commit. This is very expensive for performance, but - * guaranteed durability even on unexpected system failure or power outage. - * Of course, provided that the operating system and the underlying hardware - * (e.g. disk) work correctly. - * - * TRADE-OFF: By skipping some stages described above, you can significantly - * benefit in speed, while partially or completely losing in the guarantee of - * data durability and/or consistency in the event of system or power failure. - * Moreover, if for any reason disk write order is not preserved, then at moment - * of a system crash, a meta-page with a pointer to the new B-tree may be - * written to disk, while the itself B-tree not yet. In that case, the database - * will be corrupted! - * - * - * MDBX_NOMETASYNC = don't sync the meta-page after commit. - * - * Flush system buffers to disk only once per transaction, omit the - * metadata flush. Defer that until the system flushes files to disk, - * or next non-MDBX_RDONLY commit or mdbx_env_sync(). Depending on the - * platform and hardware, with MDBX_NOMETASYNC you may get a doubling of - * write performance. - * - * This trade-off maintains database integrity, but a system crash may - * undo the last committed transaction. I.e. it preserves the ACI - * (atomicity, consistency, isolation) but not D (durability) database - * property. - * - * MDBX_NOMETASYNC flag may be changed at any time using - * mdbx_env_set_flags() or by passing to mdbx_txn_begin() for particular - * write transaction. - * - * - * MDBX_UTTERLY_NOSYNC = don't sync anything and wipe previous steady commits. - * - * Don't flush system buffers to disk when committing a transaction. This - * optimization means a system crash can corrupt the database, if buffers - * are not yet flushed to disk. Depending on the platform and hardware, - * with MDBX_UTTERLY_NOSYNC you may get a multiple increase of write - * performance, even 100 times or more. - * - * If the filesystem preserves write order (which is rare and never - * provided unless explicitly noted) and the MDBX_WRITEMAP and - * MDBX_LIFORECLAIM flags are not used, then a system crash can't corrupt - * the database, but you can lose the last transactions, if at least one - * buffer is not yet flushed to disk. The risk is governed by how often the - * system flushes dirty buffers to disk and how often mdbx_env_sync() is - * called. So, transactions exhibit ACI (atomicity, consistency, isolation) - * properties and only lose D (durability). I.e. database integrity is - * maintained, but a system crash may undo the final transactions. - * - * Otherwise, if the filesystem not preserves write order (which is - * typically) or MDBX_WRITEMAP or MDBX_LIFORECLAIM flags are used, you - * should expect the corrupted database after a system crash. - * - * So, most important thing about MDBX_UTTERLY_NOSYNC: - * - a system crash immediately after commit the write transaction - * high likely lead to database corruption. - * - successful completion of mdbx_env_sync(force = true) after one or - * more commited transactions guarantees consystency and durability. - * - BUT by committing two or more transactions you back database into a - * weak state, in which a system crash may lead to database corruption! - * In case single transaction after mdbx_env_sync, you may lose - * transaction itself, but not a whole database. - * - * Nevertheless, MDBX_UTTERLY_NOSYNC provides "weak" durability in case of - * an application crash (but no durability on system failure), and - * therefore may be very useful in scenarios where data durability is not - * required over a system failure (e.g for short-lived data), or if you can - * take such risk. - * - * MDBX_UTTERLY_NOSYNC flag may be changed at any time using - * mdbx_env_set_flags(), but don't has effect if passed to mdbx_txn_begin() - * for particular write transaction. - * - * - * MDBX_SAFE_NOSYNC = don't sync anything but keep previous steady commits. - * - * Like MDBX_UTTERLY_NOSYNC the MDBX_SAFE_NOSYNC flag similarly disable - * flush system buffers to disk when committing a transaction. But there - * is a huge difference in how are recycled the MVCC snapshots - * corresponding to previous "steady" transactions (see below). - * - * Depending on the platform and hardware, with MDBX_SAFE_NOSYNC you may - * get a multiple increase of write performance, even 10 times or more. - * NOTE that (MDBX_SAFE_NOSYNC | MDBX_WRITEMAP) leaves the system with no - * hint for when to write transactions to disk. Therefore the - * (MDBX_MAPASYNC | MDBX_WRITEMAP) may be preferable, but without - * MDBX_SAFE_NOSYNC because the (MDBX_MAPASYNC | MDBX_SAFE_NOSYNC) actually - * gives MDBX_UTTERLY_NOSYNC. - * - * In contrast to MDBX_UTTERLY_NOSYNC mode, with MDBX_SAFE_NOSYNC flag MDBX - * will keeps untouched pages within B-tree of the last transaction - * "steady" which was synced to disk completely. This has big implications - * for both data durability and (unfortunately) performance: - * - a system crash can't corrupt the database, but you will lose the - * last transactions; because MDBX will rollback to last steady commit - * since it kept explicitly. - * - the last steady transaction makes an effect similar to "long-lived" - * read transaction (see above in the "RESTRICTIONS & CAVEATS" section) - * since prevents reuse of pages freed by newer write transactions, - * thus the any data changes will be placed in newly allocated pages. - * - to avoid rapid database growth, the system will sync data and issue - * a steady commit-point to resume reuse pages, each time there is - * insufficient space and before increasing the size of the file on - * disk. - * - * In other words, with MDBX_SAFE_NOSYNC flag MDBX insures you from the - * whole database corruption, at the cost increasing database size and/or - * number of disk IOPS. So, MDBX_SAFE_NOSYNC flag could be used with - * mdbx_env_synv() as alternatively for batch committing or nested - * transaction (in some cases). As well, auto-sync feature exposed by - * mdbx_env_set_syncbytes() and mdbx_env_set_syncperiod() functions could - * be very usefull with MDBX_SAFE_NOSYNC flag. - * - * The number and volume of of disk IOPS with MDBX_SAFE_NOSYNC flag will - * exactly the as without any no-sync flags. However, you should expect a - * larger process's work set (https://bit.ly/2kA2tFX) and significantly - * worse a locality of reference (https://bit.ly/2mbYq2J), due to the more - * intensive allocation of previously unused pages and increase the size of - * the database. - * - * MDBX_SAFE_NOSYNC flag may be changed at any time using - * mdbx_env_set_flags() or by passing to mdbx_txn_begin() for particular - * write transaction. - * - * - * MDBX_MAPASYNC = use asynchronous msync when MDBX_WRITEMAP is used. - * - * MDBX_MAPASYNC meaningful and give effect only in conjunction - * with MDBX_WRITEMAP or MDBX_SAFE_NOSYNC: - * - with MDBX_SAFE_NOSYNC actually gives MDBX_UTTERLY_NOSYNC, which - * wipe previous steady commits for reuse pages as described above. - * - with MDBX_WRITEMAP but without MDBX_SAFE_NOSYNC instructs MDBX to use - * asynchronous mmap-flushes to disk as described below. - * - with both MDBX_WRITEMAP and MDBX_SAFE_NOSYNC you get the both - * effects. - * - * Asynchronous mmap-flushes means that actually all writes will scheduled - * and performed by operation system on it own manner, i.e. unordered. - * MDBX itself just notify operating system that it would be nice to write - * data to disk, but no more. - * - * With MDBX_MAPASYNC flag, but without MDBX_UTTERLY_NOSYNC (i.e. without - * OR'ing with MDBX_SAFE_NOSYNC) MDBX will keeps untouched pages within - * B-tree of the last transaction "steady" which was synced to disk - * completely. So, this makes exactly the same "long-lived" impact and the - * same consequences as described above for MDBX_SAFE_NOSYNC flag. - * - * Depending on the platform and hardware, with combination of - * MDBX_WRITEMAP and MDBX_MAPASYNC you may get a multiple increase of write - * performance, even 25 times or more. MDBX_MAPASYNC flag may be changed at - * any time using mdbx_env_set_flags() or by passing to mdbx_txn_begin() - * for particular write transaction. - */ + /** Exactly the same as \ref MDBX_NOMETASYNC, + * but for this transaction only */ + MDBX_TXN_NOMETASYNC = MDBX_NOMETASYNC, -/* Don't sync meta-page after commit, - * see description in the "SYNC MODES" section above. */ -#define MDBX_NOMETASYNC 0x40000u - -/* Don't sync anything but keep previous steady commits, - * see description in the "SYNC MODES" section above. - * - * (!) don't combine this flag with MDBX_MAPASYNC - * since you will got MDBX_UTTERLY_NOSYNC in that way (see below) */ -#define MDBX_SAFE_NOSYNC 0x10000u - -/* Use asynchronous msync when MDBX_WRITEMAP is used, - * see description in the "SYNC MODES" section above. - * - * (!) don't combine this flag with MDBX_SAFE_NOSYNC - * since you will got MDBX_UTTERLY_NOSYNC in that way (see below) */ -#define MDBX_MAPASYNC 0x100000u - -/* Don't sync anything and wipe previous steady commits, - * see description in the "SYNC MODES" section above. */ -#define MDBX_UTTERLY_NOSYNC (MDBX_SAFE_NOSYNC | MDBX_MAPASYNC) - -/**** DATABASE FLAGS **********************************************************/ -/* Use reverse string keys */ -#define MDBX_REVERSEKEY 0x02u - -/* Use sorted duplicates */ -#define MDBX_DUPSORT 0x04u - -/* Numeric keys in native byte order, either uint32_t or uint64_t. - * The keys must all be of the same size and must be aligned while passing as - * arguments. */ -#define MDBX_INTEGERKEY 0x08u - -/* With MDBX_DUPSORT, sorted dup items have fixed size */ -#define MDBX_DUPFIXED 0x10u - -/* With MDBX_DUPSORT, dups are MDBX_INTEGERKEY-style integers. - * The data values must all be of the same size and must be aligned while - * passing as arguments. */ -#define MDBX_INTEGERDUP 0x20u - -/* With MDBX_DUPSORT, use reverse string dups */ -#define MDBX_REVERSEDUP 0x40u - -/* Create DB if not already existing */ -#define MDBX_CREATE 0x40000u - -/**** DATA UPDATE FLAGS *******************************************************/ -/* For put: Don't write if the key already exists. */ -#define MDBX_NOOVERWRITE 0x10u -/* Only for MDBX_DUPSORT - * For put: don't write if the key and data pair already exist. - * For mdbx_cursor_del: remove all duplicate data items. */ -#define MDBX_NODUPDATA 0x20u -/* For mdbx_cursor_put: overwrite the current key/data pair - * MDBX allows this flag for mdbx_put() for explicit overwrite/update without - * insertion. */ -#define MDBX_CURRENT 0x40u -/* For put: Just reserve space for data, don't copy it. Return a - * pointer to the reserved space. */ -#define MDBX_RESERVE 0x10000u -/* Data is being appended, don't split full pages. */ -#define MDBX_APPEND 0x20000u -/* Duplicate data is being appended, don't split full pages. */ -#define MDBX_APPENDDUP 0x40000u -/* Store multiple data items in one call. Only for MDBX_DUPFIXED. */ -#define MDBX_MULTIPLE 0x80000u - -/**** TRANSACTION FLAGS *******************************************************/ -/* Do not block when starting a write transaction */ -#define MDBX_TRYTXN 0x10000000u - -/**** ENVIRONMENT COPY FLAGS **************************************************/ -/* Compacting: Omit free space from copy, and renumber all pages sequentially */ -#define MDBX_CP_COMPACT 1u -#define MDBX_CP_FORCE_RESIZEABLE 2u - -/**** CURSOR OPERATIONS ******************************************************** - * - * This is the set of all operations for retrieving data - * using a cursor. */ -typedef enum MDBX_cursor_op { - MDBX_FIRST, /* Position at first key/data item */ - MDBX_FIRST_DUP, /* MDBX_DUPSORT-only: Position at first data item - * of current key. */ - MDBX_GET_BOTH, /* MDBX_DUPSORT-only: Position at key/data pair. */ - MDBX_GET_BOTH_RANGE, /* MDBX_DUPSORT-only: Position at given key and at first - * data greater than or equal to specified data. */ - MDBX_GET_CURRENT, /* Return key/data at current cursor position */ - MDBX_GET_MULTIPLE, /* MDBX_DUPFIXED-only: Return up to a page of duplicate - * data items from current cursor position. - * Move cursor to prepare for MDBX_NEXT_MULTIPLE. */ - MDBX_LAST, /* Position at last key/data item */ - MDBX_LAST_DUP, /* MDBX_DUPSORT-only: Position at last data item - * of current key. */ - MDBX_NEXT, /* Position at next data item */ - MDBX_NEXT_DUP, /* MDBX_DUPSORT-only: Position at next data item - * of current key. */ - MDBX_NEXT_MULTIPLE, /* MDBX_DUPFIXED-only: Return up to a page of - * duplicate data items from next cursor position. - * Move cursor to prepare for MDBX_NEXT_MULTIPLE. */ - MDBX_NEXT_NODUP, /* Position at first data item of next key */ - MDBX_PREV, /* Position at previous data item */ - MDBX_PREV_DUP, /* MDBX_DUPSORT-only: Position at previous data item - * of current key. */ - MDBX_PREV_NODUP, /* Position at last data item of previous key */ - MDBX_SET, /* Position at specified key */ - MDBX_SET_KEY, /* Position at specified key, return both key and data */ - MDBX_SET_RANGE, /* Position at first key greater than or equal to - * specified key. */ - MDBX_PREV_MULTIPLE /* MDBX_DUPFIXED-only: Position at previous page and - * return up to a page of duplicate data items. */ -} MDBX_cursor_op; - -/**** ERRORS & RETURN CODES **************************************************** - * BerkeleyDB uses -30800 to -30999, we'll go under them */ - -/* Successful result */ -#define MDBX_SUCCESS 0 -#define MDBX_RESULT_FALSE MDBX_SUCCESS -/* Successful result with special meaning or a flag */ -#define MDBX_RESULT_TRUE (-1) - -/* key/data pair already exists */ -#define MDBX_KEYEXIST (-30799) - -/* key/data pair not found (EOF) */ -#define MDBX_NOTFOUND (-30798) - -/* Requested page not found - this usually indicates corruption */ -#define MDBX_PAGE_NOTFOUND (-30797) - -/* Database is corrupted (page was wrong type and so on) */ -#define MDBX_CORRUPTED (-30796) - -/* Environment had fatal error (i.e. update of meta page failed and so on) */ -#define MDBX_PANIC (-30795) - -/* DB file version mismatch with libmdbx */ -#define MDBX_VERSION_MISMATCH (-30794) - -/* File is not a valid MDBX file */ -#define MDBX_INVALID (-30793) - -/* Environment mapsize reached */ -#define MDBX_MAP_FULL (-30792) - -/* Environment maxdbs reached */ -#define MDBX_DBS_FULL (-30791) - -/* Environment maxreaders reached */ -#define MDBX_READERS_FULL (-30790) - -/* Transaction has too many dirty pages, i.e transaction too big */ -#define MDBX_TXN_FULL (-30788) - -/* Cursor stack too deep - this usually indicates corruption, - * i.e branch-pages loop */ -#define MDBX_CURSOR_FULL (-30787) - -/* Page has not enough space - internal error */ -#define MDBX_PAGE_FULL (-30786) - -/* Database engine was unable to extend mapping, e.g. since address space - * is unavailable or busy. This can mean: - * - Database size extended by other process beyond to environment mapsize - * and engine was unable to extend mapping while starting read transaction. - * Environment should be reopened to continue. - * - Engine was unable to extend mapping during write transaction - * or explicit call of mdbx_env_set_geometry(). */ -#define MDBX_UNABLE_EXTEND_MAPSIZE (-30785) - -/* MDBX_MAP_RESIZED is deprecated. - * Please review your code to use MDBX_UNABLE_EXTEND_MAPSIZE instead. */ -static __inline int __deprecated MDBX_MAP_RESIZED() { - return MDBX_UNABLE_EXTEND_MAPSIZE; -} -#define MDBX_MAP_RESIZED MDBX_MAP_RESIZED() + /** Exactly the same as \ref MDBX_SAFE_NOSYNC, + * but for this transaction only */ + MDBX_TXN_NOSYNC = MDBX_SAFE_NOSYNC +}; +#ifndef __cplusplus +typedef enum MDBX_txn_flags_t MDBX_txn_flags_t; +#else +DEFINE_ENUM_FLAG_OPERATORS(MDBX_txn_flags_t) +#endif + +/** \brief Database flags + * \ingroup c_dbi + * \anchor db_flags + * \see mdbx_dbi_open() */ +enum MDBX_db_flags_t { + MDBX_DB_DEFAULTS = 0, + + /** Use reverse string keys */ + MDBX_REVERSEKEY = UINT32_C(0x02), + + /** Use sorted duplicates, i.e. allow multi-values */ + MDBX_DUPSORT = UINT32_C(0x04), + + /** Numeric keys in native byte order either uint32_t or uint64_t. The keys + * must all be of the same size and must be aligned while passing as + * arguments. */ + MDBX_INTEGERKEY = UINT32_C(0x08), + + /** With \ref MDBX_DUPSORT; sorted dup items have fixed size */ + MDBX_DUPFIXED = UINT32_C(0x10), + + /** With \ref MDBX_DUPSORT; dups are \ref MDBX_INTEGERKEY -style integers. The + * data values must all be of the same size and must be aligned while passing + * as arguments. */ + MDBX_INTEGERDUP = UINT32_C(0x20), + + /** With \ref MDBX_DUPSORT; use reverse string comparison */ + MDBX_REVERSEDUP = UINT32_C(0x40), + + /** Create DB if not already existing */ + MDBX_CREATE = UINT32_C(0x40000), + + /** Opens an existing sub-database created with unknown flags. + * + * The `MDBX_DB_ACCEDE` flag is intend to open a existing sub-database which + * was created with unknown flags (\ref MDBX_REVERSEKEY, \ref MDBX_DUPSORT, + * \ref MDBX_INTEGERKEY, \ref MDBX_DUPFIXED, \ref MDBX_INTEGERDUP and + * \ref MDBX_REVERSEDUP). + * + * In such cases, instead of returning the \ref MDBX_INCOMPATIBLE error, the + * sub-database will be opened with flags which it was created, and then an + * application could determine the actual flags by \ref mdbx_dbi_flags(). */ + MDBX_DB_ACCEDE = MDBX_ACCEDE +}; +#ifndef __cplusplus +/** \ingroup c_dbi */ +typedef enum MDBX_db_flags_t MDBX_db_flags_t; +#else +DEFINE_ENUM_FLAG_OPERATORS(MDBX_db_flags_t) +#endif + +/** \brief Data changing flags + * \ingroup c_crud + * \see c_crud_hint + * \see mdbx_put() \see mdbx_cursor_put() \see mdbx_replace() */ +enum MDBX_put_flags_t { + /** Upsertion by default (without any other flags) */ + MDBX_UPSERT = 0, + + /** For insertion: Don't write if the key already exists. */ + MDBX_NOOVERWRITE = UINT32_C(0x10), + + /** Has effect only for \ref MDBX_DUPSORT databases. + * For upsertion: don't write if the key-value pair already exist. + * For deletion: remove all values for key. */ + MDBX_NODUPDATA = UINT32_C(0x20), + + /** For upsertion: overwrite the current key/data pair. + * MDBX allows this flag for \ref mdbx_put() for explicit overwrite/update + * without insertion. + * For deletion: remove only single entry at the current cursor position. */ + MDBX_CURRENT = UINT32_C(0x40), + + /** Has effect only for \ref MDBX_DUPSORT databases. + * For deletion: remove all multi-values (aka duplicates) for given key. + * For upsertion: replace all multi-values for given key with a new one. */ + MDBX_ALLDUPS = UINT32_C(0x80), + + /** For upsertion: Just reserve space for data, don't copy it. + * Return a pointer to the reserved space. */ + MDBX_RESERVE = UINT32_C(0x10000), + + /** Data is being appended. + * Don't split full pages, continue on a new instead. */ + MDBX_APPEND = UINT32_C(0x20000), + + /** Has effect only for \ref MDBX_DUPSORT databases. + * Duplicate data is being appended. + * Don't split full pages, continue on a new instead. */ + MDBX_APPENDDUP = UINT32_C(0x40000), + + /** Only for \ref MDBX_DUPFIXED. + * Store multiple data items in one call. */ + MDBX_MULTIPLE = UINT32_C(0x80000) +}; +#ifndef __cplusplus +/** \ingroup c_crud */ +typedef enum MDBX_put_flags_t MDBX_put_flags_t; +#else +DEFINE_ENUM_FLAG_OPERATORS(MDBX_put_flags_t) +#endif + +/** \brief Environment copy flags + * \ingroup c_extra + * \see mdbx_env_copy() \see mdbx_env_copy2fd() */ +enum MDBX_copy_flags_t { + MDBX_CP_DEFAULTS = 0, + + /** Copy with compactification: Omit free space from copy and renumber all + * pages sequentially */ + MDBX_CP_COMPACT = 1u, + + /** Force to make resizeable copy, i.e. dynamic size instead of fixed */ + MDBX_CP_FORCE_DYNAMIC_SIZE = 2u +}; +#ifndef __cplusplus +/** \ingroup c_extra */ +typedef enum MDBX_copy_flags_t MDBX_copy_flags_t; +#else +DEFINE_ENUM_FLAG_OPERATORS(MDBX_copy_flags_t) +#endif + +/** \brief Cursor operations + * \ingroup c_cursors + * This is the set of all operations for retrieving data using a cursor. + * \see mdbx_cursor_set() */ +enum MDBX_cursor_op { + /** Position at first key/data item */ + MDBX_FIRST, + + /** \ref MDBX_DUPSORT -only: Position at first data item of current key. */ + MDBX_FIRST_DUP, + + /** \ref MDBX_DUPSORT -only: Position at key/data pair. */ + MDBX_GET_BOTH, + + /** \ref MDBX_DUPSORT -only: Position at given key and at first data greater + * than or equal to specified data. */ + MDBX_GET_BOTH_RANGE, + + /** Return key/data at current cursor position */ + MDBX_GET_CURRENT, + + /** \ref MDBX_DUPFIXED -only: Return up to a page of duplicate data items + * from current cursor position. Move cursor to prepare + * for \ref MDBX_NEXT_MULTIPLE. */ + MDBX_GET_MULTIPLE, + + /** Position at last key/data item */ + MDBX_LAST, + + /** \ref MDBX_DUPSORT -only: Position at last data item of current key. */ + MDBX_LAST_DUP, + + /** Position at next data item */ + MDBX_NEXT, + + /** \ref MDBX_DUPSORT -only: Position at next data item of current key. */ + MDBX_NEXT_DUP, + + /** \ref MDBX_DUPFIXED -only: Return up to a page of duplicate data items + * from next cursor position. Move cursor to prepare + * for `MDBX_NEXT_MULTIPLE`. */ + MDBX_NEXT_MULTIPLE, + + /** Position at first data item of next key */ + MDBX_NEXT_NODUP, + + /** Position at previous data item */ + MDBX_PREV, + + /** \ref MDBX_DUPSORT -only: Position at previous data item of current key. */ + MDBX_PREV_DUP, + + /** Position at last data item of previous key */ + MDBX_PREV_NODUP, + + /** Position at specified key */ + MDBX_SET, + + /** Position at specified key, return both key and data */ + MDBX_SET_KEY, + + /** Position at first key greater than or equal to specified key. */ + MDBX_SET_RANGE, + + /** \ref MDBX_DUPFIXED -only: Position at previous page and return up to + * a page of duplicate data items. */ + MDBX_PREV_MULTIPLE +}; +#ifndef __cplusplus +/** \ingroup c_cursors */ +typedef enum MDBX_cursor_op MDBX_cursor_op; +#endif + +/** \brief Errors and return codes + * \ingroup c_err + * + * BerkeleyDB uses -30800 to -30999, we'll go under them + * \see mdbx_strerror() \see mdbx_strerror_r() \see mdbx_liberr2str() */ +enum MDBX_error_t { + /** Successful result */ + MDBX_SUCCESS = 0, + + /** Alias for \ref MDBX_SUCCESS */ + MDBX_RESULT_FALSE = MDBX_SUCCESS, + + /** Successful result with special meaning or a flag */ + MDBX_RESULT_TRUE = -1, + + /** key/data pair already exists */ + MDBX_KEYEXIST = -30799, + + /** The first LMDB-compatible defined error code */ + MDBX_FIRST_LMDB_ERRCODE = MDBX_KEYEXIST, + + /** key/data pair not found (EOF) */ + MDBX_NOTFOUND = -30798, + + /** Requested page not found - this usually indicates corruption */ + MDBX_PAGE_NOTFOUND = -30797, + + /** Database is corrupted (page was wrong type and so on) */ + MDBX_CORRUPTED = -30796, + + /** Environment had fatal error, + * i.e. update of meta page failed and so on. */ + MDBX_PANIC = -30795, + + /** DB file version mismatch with libmdbx */ + MDBX_VERSION_MISMATCH = -30794, + + /** File is not a valid MDBX file */ + MDBX_INVALID = -30793, + + /** Environment mapsize reached */ + MDBX_MAP_FULL = -30792, + + /** Environment maxdbs reached */ + MDBX_DBS_FULL = -30791, + + /** Environment maxreaders reached */ + MDBX_READERS_FULL = -30790, + + /** Transaction has too many dirty pages, i.e transaction too big */ + MDBX_TXN_FULL = -30788, + + /** Cursor stack too deep - this usually indicates corruption, + * i.e branch-pages loop */ + MDBX_CURSOR_FULL = -30787, + + /** Page has not enough space - internal error */ + MDBX_PAGE_FULL = -30786, -/* Environment or database is not compatible with the requested operation - * or the specified flags. This can mean: - * - The operation expects an MDBX_DUPSORT / MDBX_DUPFIXED database. - * - Opening a named DB when the unnamed DB has MDBX_DUPSORT/MDBX_INTEGERKEY. - * - Accessing a data record as a database, or vice versa. - * - The database was dropped and recreated with different flags. */ -#define MDBX_INCOMPATIBLE (-30784) + /** Database engine was unable to extend mapping, e.g. since address space + * is unavailable or busy. This can mean: + * - Database size extended by other process beyond to environment mapsize + * and engine was unable to extend mapping while starting read + * transaction. Environment should be reopened to continue. + * - Engine was unable to extend mapping during write transaction + * or explicit call of \ref mdbx_env_set_geometry(). */ + MDBX_UNABLE_EXTEND_MAPSIZE = -30785, -/* Invalid reuse of reader locktable slot, - * e.g. read-transaction already run for current thread */ -#define MDBX_BAD_RSLOT (-30783) + /** Environment or database is not compatible with the requested operation + * or the specified flags. This can mean: + * - The operation expects an \ref MDBX_DUPSORT / \ref MDBX_DUPFIXED + * database. + * - Opening a named DB when the unnamed DB has \ref MDBX_DUPSORT / + * \ref MDBX_INTEGERKEY. + * - Accessing a data record as a database, or vice versa. + * - The database was dropped and recreated with different flags. */ + MDBX_INCOMPATIBLE = -30784, -/* Transaction is not valid for requested operation, - * e.g. had errored and be must aborted, has a child, or is invalid */ -#define MDBX_BAD_TXN (-30782) + /** Invalid reuse of reader locktable slot, + * e.g. read-transaction already run for current thread */ + MDBX_BAD_RSLOT = -30783, -/* Invalid size or alignment of key or data for target database, - * either invalid subDB name */ -#define MDBX_BAD_VALSIZE (-30781) + /** Transaction is not valid for requested operation, + * e.g. had errored and be must aborted, has a child, or is invalid */ + MDBX_BAD_TXN = -30782, -/* The specified DBI-handle is invalid - * or changed by another thread/transaction */ -#define MDBX_BAD_DBI (-30780) + /** Invalid size or alignment of key or data for target database, + * either invalid subDB name */ + MDBX_BAD_VALSIZE = -30781, -/* Unexpected internal error, transaction should be aborted */ -#define MDBX_PROBLEM (-30779) + /** The specified DBI-handle is invalid + * or changed by another thread/transaction */ + MDBX_BAD_DBI = -30780, -/* The last LMDB-compatible defined error code */ -#define MDBX_LAST_LMDB_ERRCODE MDBX_PROBLEM + /** Unexpected internal error, transaction should be aborted */ + MDBX_PROBLEM = -30779, -/* Another write transaction is running or environment is already used while - * opening with MDBX_EXCLUSIVE flag */ -#define MDBX_BUSY (-30778) + /** The last LMDB-compatible defined error code */ + MDBX_LAST_LMDB_ERRCODE = MDBX_PROBLEM, -/* The specified key has more than one associated value */ -#define MDBX_EMULTIVAL (-30421) + /** Another write transaction is running or environment is already used while + * opening with \ref MDBX_EXCLUSIVE flag */ + MDBX_BUSY = -30778, -/* Bad signature of a runtime object(s), this can mean: - * - memory corruption or double-free; - * - ABI version mismatch (rare case); */ -#define MDBX_EBADSIGN (-30420) + /** The first of MDBX-added error codes */ + MDBX_FIRST_ADDED_ERRCODE = MDBX_BUSY, -/* Database should be recovered, but this could NOT be done for now - * since it opened in read-only mode */ -#define MDBX_WANNA_RECOVERY (-30419) + /** The specified key has more than one associated value */ + MDBX_EMULTIVAL = -30421, -/* The given key value is mismatched to the current cursor position */ -#define MDBX_EKEYMISMATCH (-30418) + /** Bad signature of a runtime object(s), this can mean: + * - memory corruption or double-free; + * - ABI version mismatch (rare case); */ + MDBX_EBADSIGN = -30420, -/* Database is too large for current system, - * e.g. could NOT be mapped into RAM. */ -#define MDBX_TOO_LARGE (-30417) + /** Database should be recovered, but this could NOT be done for now + * since it opened in read-only mode */ + MDBX_WANNA_RECOVERY = -30419, -/* A thread has attempted to use a not owned object, - * e.g. a transaction that started by another thread. */ -#define MDBX_THREAD_MISMATCH (-30416) + /** The given key value is mismatched to the current cursor position */ + MDBX_EKEYMISMATCH = -30418, -/* Overlapping read and write transactions for the current thread */ -#define MDBX_TXN_OVERLAPPING (-30415) + /** Database is too large for current system, + * e.g. could NOT be mapped into RAM. */ + MDBX_TOO_LARGE = -30417, -/**** FUNCTIONS & RELATED STRUCTURES ******************************************/ + /** A thread has attempted to use a not owned object, + * e.g. a transaction that started by another thread. */ + MDBX_THREAD_MISMATCH = -30416, -/* Return a string describing a given error code. + /** Overlapping read and write transactions for the current thread */ + MDBX_TXN_OVERLAPPING = -30415, + + /* The last of MDBX-added error codes */ + MDBX_LAST_ADDED_ERRCODE = MDBX_TXN_OVERLAPPING, + +#if defined(_WIN32) || defined(_WIN64) + MDBX_ENODATA = ERROR_HANDLE_EOF, + MDBX_EINVAL = ERROR_INVALID_PARAMETER, + MDBX_EACCESS = ERROR_ACCESS_DENIED, + MDBX_ENOMEM = ERROR_OUTOFMEMORY, + MDBX_EROFS = ERROR_FILE_READ_ONLY, + MDBX_ENOSYS = ERROR_NOT_SUPPORTED, + MDBX_EIO = ERROR_WRITE_FAULT, + MDBX_EPERM = ERROR_INVALID_FUNCTION, + MDBX_EINTR = ERROR_CANCELLED, + MDBX_ENOFILE = ERROR_FILE_NOT_FOUND, + MDBX_EREMOTE = ERROR_REMOTE_STORAGE_MEDIA_ERROR +#else /* Windows */ +#ifdef ENODATA + MDBX_ENODATA = ENODATA, +#else + MDBX_ENODATA = -1, +#endif /* ENODATA */ + MDBX_EINVAL = EINVAL, + MDBX_EACCESS = EACCES, + MDBX_ENOMEM = ENOMEM, + MDBX_EROFS = EROFS, + MDBX_ENOSYS = ENOSYS, + MDBX_EIO = EIO, + MDBX_EPERM = EPERM, + MDBX_EINTR = EINTR, + MDBX_ENOFILE = ENOENT, + MDBX_EREMOTE = ENOTBLK +#endif /* !Windows */ +}; +#ifndef __cplusplus +/** \ingroup c_err */ +typedef enum MDBX_error_t MDBX_error_t; +#endif + +/** MDBX_MAP_RESIZED + * \ingroup c_err + * \deprecated Please review your code to use MDBX_UNABLE_EXTEND_MAPSIZE + * instead. */ +MDBX_DEPRECATED static __inline int MDBX_MAP_RESIZED_is_deprecated() { + return MDBX_UNABLE_EXTEND_MAPSIZE; +} +#define MDBX_MAP_RESIZED MDBX_MAP_RESIZED_is_deprecated() + +/** \brief Return a string describing a given error code. + * \ingroup c_err * - * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) `strerror()` * function. If the error code is greater than or equal to 0, then the string - * returned by the system function strerror(3) is returned. If the error code + * returned by the system function `strerror()` is returned. If the error code * is less than 0, an error string corresponding to the MDBX library error is * returned. See errors for a list of MDBX-specific error codes. * - * mdbx_strerror() - is NOT thread-safe because may share common internal - * buffer for system maessages. The returned string must - * NOT be modified by the application, but MAY be modified - * by a subsequent call to mdbx_strerror(), strerror() and - * other related functions. - * - * mdbx_strerror_r() - is thread-safe since uses user-supplied buffer where - * appropriate. The returned string must NOT be modified - * by the application, since it may be pointer to internal - * constatn string. However, there is no restriction if the - * returned string points to the supplied buffer. + * `mdbx_strerror()` is NOT thread-safe because may share common internal buffer + * for system messages. The returned string must NOT be modified by the + * application, but MAY be modified by a subsequent call to + * \ref mdbx_strerror(), `strerror()` and other related functions. + * \see mdbx_strerror_r() * - * [in] err The error code. + * \param [in] errnum The error code. * - * Returns "error message" The description of the error. */ + * \returns "error message" The description of the error. */ LIBMDBX_API const char *mdbx_strerror(int errnum); + +/** \brief Return a string describing a given error code. + * \ingroup c_err + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) `strerror()` + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function `strerror()` is returned. If the error code + * is less than 0, an error string corresponding to the MDBX library error is + * returned. See errors for a list of MDBX-specific error codes. + * + * `mdbx_strerror_r()` is thread-safe since uses user-supplied buffer where + * appropriate. The returned string must NOT be modified by the application, + * since it may be pointer to internal constant string. However, there is no + * restriction if the returned string points to the supplied buffer. + * \see mdbx_strerror() + * + * mdbx_liberr2str() returns string describing only MDBX error numbers but NULL + * for non-MDBX error codes. This function is thread-safe since return pointer + * to constant non-localized strings. + * + * \param [in] errnum The error code. + * \param [in,out] buf Buffer to store the error message. + * \param [in] buflen The size of buffer to store the message. + * + * \returns "error message" The description of the error. */ LIBMDBX_API const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API const char *mdbx_liberr2str(int errnum); -#if defined(_WIN32) || defined(_WIN64) -/* Bit of Windows' madness. The similar functions but returns Windows - * error-messages in the OEM-encoding for console utilities. */ +#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) +/** Bit of Windows' madness. The similar to \ref mdbx_strerror() but returns + * Windows error-messages in the OEM-encoding for console utilities. + * \ingroup c_err + * \see mdbx_strerror_r_ANSI2OEM() */ LIBMDBX_API const char *mdbx_strerror_ANSI2OEM(int errnum); + +/** Bit of Windows' madness. The similar to \ref mdbx_strerror_r() but returns + * Windows error-messages in the OEM-encoding for console utilities. + * \ingroup c_err + * \see mdbx_strerror_ANSI2OEM() */ LIBMDBX_API const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen); #endif /* Bit of Windows' madness */ -/* Create an MDBX environment instance. +/** \brief Create an MDBX environment instance. + * \ingroup c_opening * - * This function allocates memory for a MDBX_env structure. To release - * the allocated memory and discard the handle, call mdbx_env_close(). - * Before the handle may be used, it must be opened using mdbx_env_open(). + * This function allocates memory for a \ref MDBX_env structure. To release + * the allocated memory and discard the handle, call \ref mdbx_env_close(). + * Before the handle may be used, it must be opened using \ref mdbx_env_open(). * * Various other options may also need to be set before opening the handle, - * e.g. mdbx_env_set_geometry(), mdbx_env_set_maxreaders(), - * mdbx_env_set_maxdbs(), depending on usage requirements. + * e.g. \ref mdbx_env_set_geometry(), \ref mdbx_env_set_maxreaders(), + * \ref mdbx_env_set_maxdbs(), depending on usage requirements. * - * [out] env The address where the new handle will be stored. + * \param [out] penv The address where the new handle will be stored. * - * Returns a non-zero error value on failure and 0 on success. */ + * \returns a non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_create(MDBX_env **penv); -/* Open an environment instance. +/** \brief Open an environment instance. + * \ingroup c_opening + * + * Indifferently this function will fails or not, the \ref mdbx_env_close() must + * be called later to discard the \ref MDBX_env handle and release associated + * resources. * - * Indifferently this function will fails or not, the mdbx_env_close() must be - * called later to discard the MDBX_env handle and release associated resources. + * \param [in] env An environment handle returned + * by \ref mdbx_env_create() * - * [in] env An environment handle returned by mdbx_env_create() - * [in] pathname The directory in which the database files reside. - * This directory must already exist and be writable. - * [in] flags Special options for this environment. This parameter - * must be set to 0 or by bitwise OR'ing together one - * or more of the values described above in the - * "ENVIRONMENT FLAGS" and "SYNC MODES" sections. + * \param [in] pathname The directory in which the database files reside. + * This directory must already exist and be writable. + * + * \param [in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one + * or more of the values described above in the + * \ref env_flags and \ref sync_modes sections. * * Flags set by mdbx_env_set_flags() are also used: - * - MDBX_NOSUBDIR, MDBX_RDONLY, MDBX_EXCLUSIVE, MDBX_WRITEMAP, MDBX_NOTLS, - * MDBX_NORDAHEAD, MDBX_NOMEMINIT, MDBX_COALESCE, MDBX_LIFORECLAIM. - * See "ENVIRONMENT FLAGS" section above. + * - \ref MDBX_NOSUBDIR, \ref MDBX_RDONLY, \ref MDBX_EXCLUSIVE, + * \ref MDBX_WRITEMAP, \ref MDBX_NOTLS, \ref MDBX_NORDAHEAD, + * \ref MDBX_NOMEMINIT, \ref MDBX_COALESCE, \ref MDBX_LIFORECLAIM. + * See \ref env_flags section. * - * - MDBX_NOMETASYNC, MDBX_SAFE_NOSYNC, MDBX_UTTERLY_NOSYNC, MDBX_MAPASYNC. - * See "SYNC MODES" section above. + * - \ref MDBX_NOMETASYNC, \ref MDBX_SAFE_NOSYNC, \ref MDBX_UTTERLY_NOSYNC. + * See \ref sync_modes section. * - * NOTE: MDB_NOLOCK flag don't supported by MDBX, - * try use MDBX_EXCLUSIVE as a replacement. + * \note `MDB_NOLOCK` flag don't supported by MDBX, + * try use \ref MDBX_EXCLUSIVE as a replacement. * - * NOTE: MDBX don't allow to mix processes with different MDBX_SAFE_NOSYNC, - * MDBX_NOMETASYNC, MDBX_MAPASYNC flags on the same environment. - * In such case MDBX_INCOMPATIBLE will be returned. + * \note MDBX don't allow to mix processes with different \ref MDBX_SAFE_NOSYNC + * flags on the same environment. + * In such case \ref MDBX_INCOMPATIBLE will be returned. * * If the database is already exist and parameters specified early by - * mdbx_env_set_geometry() are incompatible (i.e. for instance, different page - * size) then mdbx_env_open() will return MDBX_INCOMPATIBLE error. - * - * [in] mode The UNIX permissions to set on created files. Zero value means - * to open existing, but do not create. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_VERSION_MISMATCH = the version of the MDBX library doesn't match the - * version that created the database environment. - * - MDBX_INVALID = the environment file headers are corrupted. - * - MDBX_ENOENT = the directory specified by the path parameter - * doesn't exist. - * - MDBX_EACCES = the user didn't have permission to access - * the environment files. - * - MDBX_EAGAIN = the environment was locked by another process. - * - MDBX_BUSY = MDBX_EXCLUSIVE flag was specified and the - * environment is in use by another process, - * or the current process tries to open environment - * more than once. - * - MDBX_INCOMPATIBLE = Environment is already opened by another process, - * but with different set of MDBX_SAFE_NOSYNC, - * MDBX_NOMETASYNC, MDBX_MAPASYNC flags. - * Or if the database is already exist and - * parameters specified early by - * mdbx_env_set_geometry() are incompatible (i.e. - * for instance, different page size). - * - MDBX_WANNA_RECOVERY = MDBX_RDONLY flag was specified but read-write - * access is required to rollback inconsistent state - * after a system crash. - * - MDBX_TOO_LARGE = Database is too large for this process, i.e. - * 32-bit process tries to open >4Gb database. */ + * \ref mdbx_env_set_geometry() are incompatible (i.e. for instance, different + * page size) then \ref mdbx_env_open() will return \ref MDBX_INCOMPATIBLE + * error. + * + * \param [in] mode The UNIX permissions to set on created files. + * Zero value means to open existing, but do not create. + * + * \return A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_VERSION_MISMATCH The version of the MDBX library doesn't match + * the version that created the database environment. + * \retval MDBX_INVALID The environment file headers are corrupted. + * \retval MDBX_ENOENT The directory specified by the path parameter + * doesn't exist. + * \retval MDBX_EACCES The user didn't have permission to access + * the environment files. + * \retval MDBX_EAGAIN The environment was locked by another process. + * \retval MDBX_BUSY The \ref MDBX_EXCLUSIVE flag was specified and the + * environment is in use by another process, + * or the current process tries to open environment + * more than once. + * \retval MDBX_INCOMPATIBLE Environment is already opened by another process, + * but with different set of \ref MDBX_SAFE_NOSYNC, + * \ref MDBX_UTTERLY_NOSYNC flags. + * Or if the database is already exist and parameters + * specified early by \ref mdbx_env_set_geometry() + * are incompatible (i.e. different pagesize, etc). + * + * \retval MDBX_WANNA_RECOVERY The \ref MDBX_RDONLY flag was specified but + * read-write access is required to rollback + * inconsistent state after a system crash. + * + * \retval MDBX_TOO_LARGE Database is too large for this process, + * i.e. 32-bit process tries to open >4Gb database. + */ LIBMDBX_API int mdbx_env_open(MDBX_env *env, const char *pathname, - unsigned flags, mode_t mode); + MDBX_env_flags_t flags, mdbx_mode_t mode); -/* Copy an MDBX environment to the specified path, with options. +/** \brief Copy an MDBX environment to the specified path, with options. + * \ingroup c_extra * * This function may be used to make a backup of an existing environment. * No lockfile is created, since it gets recreated at need. - * NOTE: This call can trigger significant file size growth if run in + * \note This call can trigger significant file size growth if run in * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under "Caveats" section. - * - * [in] env An environment handle returned by mdbx_env_create(). It must - * have already been opened successfully. - * [in] dest The pathname of a file in which the copy will reside. This file - * must not be already exist, but parent directory must be writable. - * [in] flags Special options for this operation. This parameter must be set - * to 0 or by bitwise OR'ing together one or more of the values - * described here: - * - * - MDBX_CP_COMPACT + * transaction. See long-lived transactions under \ref restrictions section. + * + * \param [in] env An environment handle returned by mdbx_env_create(). + * It must have already been opened successfully. + * \param [in] dest The pathname of a file in which the copy will reside. + * This file must not be already exist, but parent directory + * must be writable. + * \param [in] flags Special options for this operation. This parameter must + * be set to 0 or by bitwise OR'ing together one or more + * of the values described here: + * + * - \ref MDBX_CP_COMPACT * Perform compaction while copying: omit free pages and sequentially * renumber all pages in output. This option consumes little bit more * CPU for processing, but may running quickly than the default, on * account skipping free pages. * - * Returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, unsigned flags); + * - \ref MDBX_CP_FORCE_DYNAMIC_SIZE + * Force to make resizeable copy, i.e. dynamic size instead of fixed. + * + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, + MDBX_copy_flags_t flags); -/* Copy an MDBX environment to the specified file descriptor, - * with options. +/** \brief Copy an environment to the specified file descriptor, with + * options. \ingroup c_extra * * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. See - * mdbx_env_copy() for further details. + * No lockfile is created, since it gets recreated at need. + * \see mdbx_env_copy() * - * NOTE: This call can trigger significant file size growth if run in + * \note This call can trigger significant file size growth if run in * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under "Caveats" section. + * transaction. See long-lived transactions under \ref restrictions + * section. * - * NOTE: Fails if the environment has suffered a page leak and the destination + * \note Fails if the environment has suffered a page leak and the destination * file descriptor is associated with a pipe, socket, or FIFO. * - * [in] env An environment handle returned by mdbx_env_create(). It must - * have already been opened successfully. - * [in] fd The filedescriptor to write the copy to. It must have already - * been opened for Write access. - * [in] flags Special options for this operation. See mdbx_env_copy() for - * options. + * \param [in] env An environment handle returned by mdbx_env_create(). + * It must have already been opened successfully. + * \param [in] fd The file descriptor to write the copy to. It must have + * already been opened for Write access. + * \param [in] flags Special options for this operation. \see mdbx_env_copy() * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, - unsigned flags); - -/* Statistics for a database in the environment */ -typedef struct MDBX_stat { - uint32_t ms_psize; /* Size of a database page. - * This is the same for all databases. */ - uint32_t ms_depth; /* Depth (height) of the B-tree */ - uint64_t ms_branch_pages; /* Number of internal (non-leaf) pages */ - uint64_t ms_leaf_pages; /* Number of leaf pages */ - uint64_t ms_overflow_pages; /* Number of overflow pages */ - uint64_t ms_entries; /* Number of data items */ - uint64_t ms_mod_txnid; /* Transaction ID of commited last modification */ -} MDBX_stat; - -/* Return statistics about the MDBX environment. + MDBX_copy_flags_t flags); + +/** \brief Statistics for a database in the environment + * \ingroup c_statinfo + * \see mdbx_env_stat_ex() \see mdbx_dbi_stat() */ +struct MDBX_stat { + uint32_t ms_psize; /**< Size of a database page. This is the same for all + databases. */ + uint32_t ms_depth; /**< Depth (height) of the B-tree */ + uint64_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + uint64_t ms_leaf_pages; /**< Number of leaf pages */ + uint64_t ms_overflow_pages; /**< Number of overflow pages */ + uint64_t ms_entries; /**< Number of data items */ + uint64_t ms_mod_txnid; /**< Transaction ID of commited last modification */ +}; +#ifndef __cplusplus +/** \ingroup c_statinfo */ +typedef struct MDBX_stat MDBX_stat; +#endif + +/** \brief Return statistics about the MDBX environment. + * \ingroup c_statinfo * * At least one of env or txn argument must be non-null. If txn is passed * non-null then stat will be filled accordingly to the given transaction. - * Otherwise, if txn is null, then stat will be populated by a snapshot from the - * last committed write transaction, and at next time, other information can be - * returned. + * Otherwise, if txn is null, then stat will be populated by a snapshot from + * the last committed write transaction, and at next time, other information + * can be returned. * - * Legacy mdbx_env_stat() correspond to calling mdbx_env_stat_ex() with the null - * txn argument. + * Legacy mdbx_env_stat() correspond to calling \ref mdbx_env_stat_ex() with the + * null `txn` argument. * - * [in] env An environment handle returned by mdbx_env_create() - * [in] txn A transaction handle returned by mdbx_txn_begin() - * [out] stat The address of an MDBX_stat structure where the statistics - * will be copied + * \param [in] env An environment handle returned by \ref mdbx_env_create() + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() + * \param [out] stat The address of an \ref MDBX_stat structure where + * the statistics will be copied + * \param [in] bytes The size of \ref MDBX_stat. * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_stat *stat, size_t bytes); -__deprecated LIBMDBX_API int mdbx_env_stat(MDBX_env *env, MDBX_stat *stat, - size_t bytes); - -/* Information about the environment */ -typedef struct MDBX_envinfo { +/** \brief Return statistics about the MDBX environment. + * \ingroup c_statinfo + * \deprecated Please use mdbx_env_stat_ex() instead. */ +MDBX_DEPRECATED LIBMDBX_API int mdbx_env_stat(MDBX_env *env, MDBX_stat *stat, + size_t bytes); + +/** \brief Information about the environment + * \ingroup c_statinfo + * \see mdbx_env_info_ex() */ +struct MDBX_envinfo { struct { - uint64_t lower; /* lower limit for datafile size */ - uint64_t upper; /* upper limit for datafile size */ - uint64_t current; /* current datafile size */ - uint64_t shrink; /* shrink threshold for datafile */ - uint64_t grow; /* growth step for datafile */ + uint64_t lower; /**< Lower limit for datafile size */ + uint64_t upper; /**< Upper limit for datafile size */ + uint64_t current; /**< Current datafile size */ + uint64_t shrink; /**< Shrink threshold for datafile */ + uint64_t grow; /**< Growth step for datafile */ } mi_geo; - uint64_t mi_mapsize; /* Size of the data memory map */ - uint64_t mi_last_pgno; /* ID of the last used page */ - uint64_t mi_recent_txnid; /* ID of the last committed transaction */ - uint64_t mi_latter_reader_txnid; /* ID of the last reader transaction */ - uint64_t mi_self_latter_reader_txnid; /* ID of the last reader transaction of - caller process */ + uint64_t mi_mapsize; /**< Size of the data memory map */ + uint64_t mi_last_pgno; /**< Number of the last used page */ + uint64_t mi_recent_txnid; /**< ID of the last committed transaction */ + uint64_t mi_latter_reader_txnid; /**< ID of the last reader transaction */ + uint64_t mi_self_latter_reader_txnid; /**< ID of the last reader transaction + of caller process */ uint64_t mi_meta0_txnid, mi_meta0_sign; uint64_t mi_meta1_txnid, mi_meta1_sign; uint64_t mi_meta2_txnid, mi_meta2_sign; - uint32_t mi_maxreaders; /* max reader slots in the environment */ - uint32_t mi_numreaders; /* max reader slots used in the environment */ - uint32_t mi_dxb_pagesize; /* database pagesize */ - uint32_t mi_sys_pagesize; /* system pagesize */ - + uint32_t mi_maxreaders; /**< Total reader slots in the environment */ + uint32_t mi_numreaders; /**< Max reader slots used in the environment */ + uint32_t mi_dxb_pagesize; /**< Database pagesize */ + uint32_t mi_sys_pagesize; /**< System pagesize */ + + /** \brief A mostly unique ID that is regenerated on each boot. + + As such it can be used to identify the local machine's current boot. MDBX + uses such when open the database to determine whether rollback required to + the last steady sync point or not. I.e. if current bootid is differ from the + value within a database then the system was rebooted and all changes since + last steady sync must be reverted for data integrity. Zeros mean that no + relevant information is available from the system. */ struct { - /* A mostly unique ID that is regenerated on each boot. As such it can be - used to identify the local machine's current boot. MDBX uses such when - open the database to determine whether rollback required to the last - steady sync point or not. I.e. if current bootid is differ from the value - within a database then the system was rebooted and all changes since last - steady sync must be reverted for data integrity. Zeros mean that no - relevant information is available from the system. */ struct { - uint64_t l, h; + uint64_t x, y; } current, meta0, meta1, meta2; } mi_bootid; - uint64_t mi_unsync_volume; /* bytes not explicitly synchronized to disk */ - uint64_t mi_autosync_threshold; /* current auto-sync threshold, see - mdbx_env_set_syncbytes(). */ - uint32_t mi_since_sync_seconds16dot16; /* time since the last steady sync in - 1/65536 of second */ - uint32_t mi_autosync_period_seconds16dot16 /* current auto-sync period in - 1/65536 of second, see - mdbx_env_set_syncperiod(). */ - ; - uint32_t mi_since_reader_check_seconds16dot16; /* time since the last readers - check in 1/65536 of second, - see mdbx_reader_check(). */ - uint32_t mi_mode; /* current environment mode, the same as - mdbx_env_get_flags() returns. */ -} MDBX_envinfo; - -/* Return information about the MDBX environment. + /** Bytes not explicitly synchronized to disk */ + uint64_t mi_unsync_volume; + /** Current auto-sync threshold, see \ref mdbx_env_set_syncbytes(). */ + uint64_t mi_autosync_threshold; + /** Time since the last steady sync in 1/65536 of second */ + uint32_t mi_since_sync_seconds16dot16; + /** Current auto-sync period in 1/65536 of second, + * see \ref mdbx_env_set_syncperiod(). */ + uint32_t mi_autosync_period_seconds16dot16; + /** Time since the last readers check in 1/65536 of second, + * see \ref mdbx_reader_check(). */ + uint32_t mi_since_reader_check_seconds16dot16; + /** Current environment mode. + * The same as \ref mdbx_env_get_flags() returns. */ + uint32_t mi_mode; +}; +#ifndef __cplusplus +/** \ingroup c_statinfo */ +typedef struct MDBX_envinfo MDBX_envinfo; +#endif + +/** \brief Return information about the MDBX environment. + * \ingroup c_statinfo * * At least one of env or txn argument must be non-null. If txn is passed * non-null then stat will be filled accordingly to the given transaction. - * Otherwise, if txn is null, then stat will be populated by a snapshot from the - * last committed write transaction, and at next time, other information can be - * returned. + * Otherwise, if txn is null, then stat will be populated by a snapshot from + * the last committed write transaction, and at next time, other information + * can be returned. * - * Legacy mdbx_env_info() correspond to calling mdbx_env_info_ex() with the null - * txn argument. - - * [in] env An environment handle returned by mdbx_env_create() - * [in] txn A transaction handle returned by mdbx_txn_begin() - * [out] stat The address of an MDBX_envinfo structure - * where the information will be copied + * Legacy \ref mdbx_env_info() correspond to calling \ref mdbx_env_info_ex() + * with the null `txn` argument. * - * Returns A non-zero error value on failure and 0 on success. */ + * \param [in] env An environment handle returned by \ref mdbx_env_create() + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() + * \param [out] info The address of an \ref MDBX_envinfo structure + * where the information will be copied + * \param [in] bytes The size of \ref MDBX_envinfo. + * + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *info, size_t bytes); -__deprecated LIBMDBX_API int mdbx_env_info(MDBX_env *env, MDBX_envinfo *info, - size_t bytes); - -/* Flush the environment data buffers to disk. - * - * Unless the environment was opened with no-sync flags (MDBX_NOMETASYNC, - * MDBX_SAFE_NOSYNC, MDBX_UTTERLY_NOSYNC and MDBX_MAPASYNC), then data is always - * written an flushed to disk when mdbx_txn_commit() is called. Otherwise - * mdbx_env_sync() may be called to manually write and flush unsynced data to - * disk. - * - * Besides, mdbx_env_sync_ex() with argument force=false may be used to +/** \brief Return information about the MDBX environment. + * \ingroup c_statinfo + * \deprecated Please use mdbx_env_info_ex() instead. */ +MDBX_DEPRECATED LIBMDBX_API int mdbx_env_info(MDBX_env *env, MDBX_envinfo *info, + size_t bytes); + +/** \brief Flush the environment data buffers to disk. + * \ingroup c_extra + * + * Unless the environment was opened with no-sync flags (\ref MDBX_NOMETASYNC, + * \ref MDBX_SAFE_NOSYNC and \ref MDBX_UTTERLY_NOSYNC), then + * data is always written an flushed to disk when \ref mdbx_txn_commit() is + * called. Otherwise \ref mdbx_env_sync() may be called to manually write and + * flush unsynced data to disk. + * + * Besides, \ref mdbx_env_sync_ex() with argument `force=false` may be used to * provide polling mode for lazy/asynchronous sync in conjunction with - * mdbx_env_set_syncbytes() and/or mdbx_env_set_syncperiod(). - * - * The mdbx_env_sync() is shortcut to calling mdbx_env_sync_ex() with - * try force=true and nonblock=false arguments. - * - * The mdbx_env_sync_poll() is shortcut to calling mdbx_env_sync_ex() with - * the force=false and nonblock=true arguments. - * - * NOTE: This call is not valid if the environment was opened with MDBX_RDONLY. - * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] force If non-zero, force a flush. Otherwise, if force is zero, then - * will run in polling mode, i.e. it will check the thresholds - * that were set mdbx_env_set_syncbytes() and/or - * mdbx_env_set_syncperiod() and perform flush If at least one - * of the thresholds is reached. - * [in] nonblock Don't wait if write transaction is running by other thread. - * - * Returns A non-zero error value on failure and MDBX_RESULT_TRUE or 0 on - * success. The MDBX_RESULT_TRUE means no data pending for flush to disk, - * and 0 otherwise. Some possible errors are: - * - MDBX_EACCES = the environment is read-only. - * - MDBX_BUSY = the environment is used by other thread and nonblock=true. - * - MDBX_EINVAL = an invalid parameter was specified. - * - MDBX_EIO = an error occurred during synchronization. */ -LIBMDBX_API int mdbx_env_sync_ex(MDBX_env *env, int force, int nonblock); + * \ref mdbx_env_set_syncbytes() and/or \ref mdbx_env_set_syncperiod(). + * + * \note This call is not valid if the environment was opened with MDBX_RDONLY. + * + * \param [in] env An environment handle returned by \ref mdbx_env_create() + * \param [in] force If non-zero, force a flush. Otherwise, If force is + * zero, then will run in polling mode, + * i.e. it will check the thresholds that were + * set \ref mdbx_env_set_syncbytes() + * and/or \ref mdbx_env_set_syncperiod() and perform flush + * if at least one of the thresholds is reached. + * + * \param [in] nonblock Don't wait if write transaction + * is running by other thread. + * + * \returns A non-zero error value on failure and \ref MDBX_RESULT_TRUE or 0 on + * success. The \ref MDBX_RESULT_TRUE means no data pending for flush + * to disk, and 0 otherwise. Some possible errors are: + * + * \retval MDBX_EACCES the environment is read-only. + * \retval MDBX_BUSY the environment is used by other thread + * and `nonblock=true`. + * \retval MDBX_EINVAL an invalid parameter was specified. + * \retval MDBX_EIO an error occurred during synchronization. */ +LIBMDBX_API int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock); + +/** \brief The shortcut to calling \ref mdbx_env_sync_ex() with + * the `force=true` and `nonblock=false` arguments. + * \ingroup c_extra */ LIBMDBX_API int mdbx_env_sync(MDBX_env *env); + +/** \brief The shortcut to calling \ref mdbx_env_sync_ex() with + * the `force=false` and `nonblock=true` arguments. + * \ingroup c_extra */ LIBMDBX_API int mdbx_env_sync_poll(MDBX_env *env); -/* Sets threshold to force flush the data buffers to disk, even of - * MDBX_SAFE_NOSYNC, MDBX_NOMETASYNC and MDBX_MAPASYNC flags in the environment. +/** \brief Sets threshold to force flush the data buffers to disk, even any of + * \ref MDBX_SAFE_NOSYNC flag in the environment. + * \ingroup c_settings + * * The threshold value affects all processes which operates with given * environment until the last process close environment or a new value will be * settled. * - * Data is always written to disk when mdbx_txn_commit() is called, but the - * operating system may keep it buffered. MDBX always flushes the OS buffers - * upon commit as well, unless the environment was opened with MDBX_SAFE_NOSYNC, - * MDBX_MAPASYNC or in part MDBX_NOMETASYNC. + * Data is always written to disk when \ref mdbx_txn_commit() is called, but + * the operating system may keep it buffered. MDBX always flushes the OS buffers + * upon commit as well, unless the environment was opened with + * \ref MDBX_SAFE_NOSYNC, \ref MDBX_UTTERLY_NOSYNC + * or in part \ref MDBX_NOMETASYNC. * * The default is 0, than mean no any threshold checked, and no additional * flush will be made. * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] threshold The size in bytes of summary changes when a synchronous - * flush would be made. + * \param [in] env An environment handle returned by mdbx_env_create(). + * \param [in] threshold The size in bytes of summary changes when + * a synchronous flush would be made. * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold); -/* Sets relative period since the last unsteay commit to force flush the data - * buffers to disk, even of MDBX_SAFE_NOSYNC, MDBX_NOMETASYNC and MDBX_MAPASYNC - * flags in the environment. The relative period value affects all processes - * which operates with given environment until the last process close - * environment or a new value will be settled. +/** \brief Sets relative period since the last unsteady commit to force flush + * the data buffers to disk, even of \ref MDBX_SAFE_NOSYNC flag in the + * environment. + * + * \ingroup c_settings * - * Data is always written to disk when mdbx_txn_commit() is called, but the + * The relative period value affects all processes which operates with given + * environment until the last process close environment or a new value will be + * settled. + * + * Data is always written to disk when \ref mdbx_txn_commit() is called, but the * operating system may keep it buffered. MDBX always flushes the OS buffers - * upon commit as well, unless the environment was opened with MDBX_SAFE_NOSYNC, - * MDBX_MAPASYNC or in part MDBX_NOMETASYNC. + * upon commit as well, unless the environment was opened with + * \ref MDBX_SAFE_NOSYNC or in part \ref MDBX_NOMETASYNC. * * Settled period don't checked asynchronously, but only by the - * mdbx_txn_commit() and mdbx_env_sync() functions. Therefore, in cases where - * transactions are committed infrequently and/or irregularly, polling by - * mdbx_env_sync() may be a reasonable solution to timeout enforcement. + * \ref mdbx_txn_commit() and \ref mdbx_env_sync() functions. Therefore, in + * cases where transactions are committed infrequently and/or irregularly, + * polling by \ref mdbx_env_sync() may be a reasonable solution to timeout + * enforcement. * * The default is 0, than mean no any timeout checked, and no additional * flush will be made. * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] seconds_16dot16 The period in 1/65536 of second when a synchronous - * flush would be made since the last unsteay commit. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] seconds_16dot16 The period in 1/65536 of second when + * a synchronous flush would be made since + * the last unsteady commit. * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16); -/* Close the environment and release the memory map. +/** \brief Close the environment and release the memory map. + * \ingroup c_opening * * Only a single thread may call this function. All transactions, databases, * and cursors must already be closed before calling this function. Attempts - * to use any such handles after calling this function will cause a SIGSEGV. + * to use any such handles after calling this function will cause a `SIGSEGV`. * The environment handle will be freed and must not be used again after this * call. * - * Legacy mdbx_env_close() correspond to calling mdbx_env_close_ex() with the - * argument dont_sync=false. - * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] dont_sync A dont'sync flag, if non-zero the last checkpoint (meta-page - * update) will be kept "as is" and may be still "weak" in the - * NOSYNC/MAPASYNC modes. Such "weak" checkpoint will be - * ignored on opening next time, and transactions since the - * last non-weak checkpoint (meta-page update) will rolledback - * for consistency guarantee. - * - * Returns A non-zero error value on failure and 0 on success. - * Some possible errors are: - * - MDBX_BUSY = The write transaction is running by other thread, in such - * case MDBX_env instance has NOT be destroyed not released! - * NOTE: if any OTHER error code was returned then given - * MDBX_env instance has been destroyed and released. - * - MDBX_EBADSIGN = Environment handle already closed (i.e. mdbx_env_close() - * was already called or not valid (i.e. was not created - * by mdbx_env_create()). - * - MDBX_PANIC = If mdbx_env_close_ex() was called in the child process - * after fork(). In this case MDBX_PANIC is a expecte, - * i.e. MDBX_env instance was freed in proper manner. - * - MDBX_EIO = an error occurred during synchronization. */ -LIBMDBX_API int mdbx_env_close_ex(MDBX_env *env, int dont_sync); + * \param [in] env An environment handle returned by + * \ref mdbx_env_create(). + * + * \param [in] dont_sync A dont'sync flag, if non-zero the last checkpoint + * will be kept "as is" and may be still "weak" in the + * \ref MDBX_SAFE_NOSYNC or \ref MDBX_UTTERLY_NOSYNC + * modes. Such "weak" checkpoint will be ignored on + * opening next time, and transactions since the last + * non-weak checkpoint (meta-page update) will rolledback + * for consistency guarantee. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_BUSY The write transaction is running by other thread, + * in such case \ref MDBX_env instance has NOT be destroyed + * not released! + * \note If any OTHER error code was returned then + * given MDBX_env instance has been destroyed and released. + * + * \retval MDBX_EBADSIGN Environment handle already closed or not valid, + * i.e. \ref mdbx_env_close() was already called for the + * `env` or was not created by \ref mdbx_env_create(). + * + * \retval MDBX_PANIC If \ref mdbx_env_close_ex() was called in the child + * process after `fork()`. In this case \ref MDBX_PANIC + * is expected, i.e. \ref MDBX_env instance was freed in + * proper manner. + * + * \retval MDBX_EIO An error occurred during synchronization. */ +LIBMDBX_API int mdbx_env_close_ex(MDBX_env *env, bool dont_sync); + +/** \brief The shortcut to calling \ref mdbx_env_close_ex() with + * the `dont_sync=false` argument. + * \ingroup c_opening */ LIBMDBX_API int mdbx_env_close(MDBX_env *env); -/* Set environment flags. +/** \brief Set environment flags. + * \ingroup c_settings * * This may be used to set some flags in addition to those from * mdbx_env_open(), or to unset these flags. + * \see mdbx_env_get_flags() * - * NOTE: In contrast to LMDB, the MDBX serialize threads via mutex while + * \note In contrast to LMDB, the MDBX serialize threads via mutex while * changing the flags. Therefore this function will be blocked while a write - * transaction running by other thread, or MDBX_BUSY will be returned if + * transaction running by other thread, or \ref MDBX_BUSY will be returned if * function called within a write transaction. * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] flags The flags to change, bitwise OR'ed together. - * [in] onoff A non-zero value sets the flags, zero clears them. + * \param [in] env An environment handle returned + * by \ref mdbx_env_create(). + * \param [in] flags The \ref env_flags to change, bitwise OR'ed together. + * \param [in] onoff A non-zero value sets the flags, zero clears them. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ -LIBMDBX_API int mdbx_env_set_flags(MDBX_env *env, unsigned flags, int onoff); + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. */ +LIBMDBX_API int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, + bool onoff); -/* Get environment flags. +/** \brief Get environment flags. + * \ingroup c_statinfo + * \see mdbx_env_set_flags() * - * [in] env An environment handle returned by mdbx_env_create(). - * [out] flags The address of an integer to store the flags. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [out] flags The address of an integer to store the flags. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_env_get_flags(const MDBX_env *env, unsigned *flags); -/* Return the path that was used in mdbx_env_open(). +/** \brief Return the path that was used in mdbx_env_open(). + * \ingroup c_statinfo * - * [in] env An environment handle returned by mdbx_env_create() - * [out] dest Address of a string pointer to contain the path. - * This is the actual string in the environment, not a copy. - * It should not be altered in any way. + * \param [in] env An environment handle returned by \ref mdbx_env_create() + * \param [out] dest Address of a string pointer to contain the path. + * This is the actual string in the environment, not a + * copy. It should not be altered in any way. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_env_get_path(const MDBX_env *env, const char **dest); -/* Return the file descriptor for the given environment. +/** \brief Return the file descriptor for the given environment. + * \ingroup c_statinfo * - * NOTE: All MDBX file descriptors have FD_CLOEXEC and - * could't be used after exec() and or fork(). + * \note All MDBX file descriptors have `FD_CLOEXEC` and + * couldn't be used after exec() and or `fork()`. * - * [in] env An environment handle returned by mdbx_env_create(). - * [out] fd Address of a int to contain the descriptor. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [out] fd Address of a int to contain the descriptor. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *fd); -/* Set all size-related parameters of environment, including page size and the - * min/max size of the memory map. +/** \brief Set all size-related parameters of environment, including page size + * and the min/max size of the memory map. \ingroup c_settings * * In contrast to LMDB, the MDBX provide automatic size management of an * database according the given parameters, including shrinking and resizing * on the fly. From user point of view all of these just working. Nevertheless, - * it is reasonable to know some details in order to make optimal decisions when - * choosing parameters. + * it is reasonable to know some details in order to make optimal decisions + * when choosing parameters. * - * Both mdbx_env_info_ex() and legacy mdbx_env_info() are inapplicable to - * read-only opened environment. + * Both \ref mdbx_env_info_ex() and legacy \ref mdbx_env_info() are inapplicable + * to read-only opened environment. * - * Both mdbx_env_info_ex() and legacy mdbx_env_info() could be called either - * before or after mdbx_env_open(), either within the write transaction running - * by current thread or not: + * Both \ref mdbx_env_info_ex() and legacy \ref mdbx_env_info() could be called + * either before or after \ref mdbx_env_open(), either within the write + * transaction running by current thread or not: * - * - In case mdbx_env_info_ex() or legacy mdbx_env_info() was called BEFORE - * mdbx_env_open(), i.e. for closed environment, then the specified - * parameters will be used for new database creation, or will be appliend - * during openeing if database exists and no other process using it. + * - In case \ref mdbx_env_info_ex() or legacy \ref mdbx_env_info() was called + * BEFORE \ref mdbx_env_open(), i.e. for closed environment, then the + * specified parameters will be used for new database creation, or will be + * applied during opening if database exists and no other process using it. * - * If the database is already exist, opened with MDBX_EXCLUSIVE or not used - * by any other process, and parameters specified by mdbx_env_set_geometry() - * are incompatible (i.e. for instance, different page size) then - * mdbx_env_open() will return MDBX_INCOMPATIBLE error. + * If the database is already exist, opened with \ref MDBX_EXCLUSIVE or not + * used by any other process, and parameters specified by + * \ref mdbx_env_set_geometry() are incompatible (i.e. for instance, + * different page size) then \ref mdbx_env_open() will return + * \ref MDBX_INCOMPATIBLE error. * * In another way, if database will opened read-only or will used by other - * process during calling mdbx_env_open() that specified parameters will - * silently discarded (open the database with MDBX_EXCLUSIVE flag to avoid - * this). - * - * - In case mdbx_env_info_ex() or legacy mdbx_env_info() was called after - * mdbx_env_open() WITHIN the write transaction running by current thread, - * then specified parameters will be appliad as a part of write transaction, - * i.e. will not be visible to any others processes until the current write - * transaction has been committed by the current process. However, if - * transaction will be aborted, then the database file will be reverted to - * the previous size not immediately, but when a next transaction will be - * committed or when the database will be opened next time. - * - * - In case mdbx_env_info_ex() or legacy mdbx_env_info() was called after - * mdbx_env_open() but OUTSIDE a write transaction, then MDBX will execute - * internal pseudo-transaction to apply new parameters (but only if anything - * has been changed), and changes be visible to any others processes - * immediatelly after succesfull competeion of function. + * process during calling \ref mdbx_env_open() that specified parameters will + * silently discarded (open the database with \ref MDBX_EXCLUSIVE flag + * to avoid this). + * + * - In case \ref mdbx_env_info_ex() or legacy \ref mdbx_env_info() was called + * after \ref mdbx_env_open() WITHIN the write transaction running by current + * thread, then specified parameters will be applied as a part of write + * transaction, i.e. will not be visible to any others processes until the + * current write transaction has been committed by the current process. + * However, if transaction will be aborted, then the database file will be + * reverted to the previous size not immediately, but when a next transaction + * will be committed or when the database will be opened next time. + * + * - In case \ref mdbx_env_info_ex() or legacy \ref mdbx_env_info() was called + * after \ref mdbx_env_open() but OUTSIDE a write transaction, then MDBX will + * execute internal pseudo-transaction to apply new parameters (but only if + * anything has been changed), and changes be visible to any others processes + * immediately after succesful completion of function. * * Essentially a concept of "automatic size management" is simple and useful: * - There are the lower and upper bound of the database file size; @@ -2050,7 +2263,7 @@ LIBMDBX_API int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *fd); * size. Besides, the upper bound defines the linear address space * reservation in each process that opens the database. Therefore changing * the upper bound is costly and may be required reopening environment in - * case of MDBX_UNABLE_EXTEND_MAPSIZE errors, and so on. Therefore, this + * case of \ref MDBX_UNABLE_EXTEND_MAPSIZE errors, and so on. Therefore, this * value should be chosen reasonable as large as possible, to accommodate * future growth of the database. * - The growth step must be greater than zero to allow the database to grow, @@ -2060,8 +2273,8 @@ LIBMDBX_API int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *fd); * to shrink but also reasonable not too small (to avoid extra overhead) and * not less than growth step to avoid up-and-down flouncing. * - The current size (i.e. size_now argument) is an auxiliary parameter for - * simulation legacy mdbx_env_set_mapsize() and as workaround Windows issues - * (see below). + * simulation legacy \ref mdbx_env_set_mapsize() and as workaround Windows + * issues (see below). * * Unfortunately, Windows has is a several issues * with resizing of memory-mapped file: @@ -2075,7 +2288,7 @@ LIBMDBX_API int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *fd); * * MDBX bypasses all Windows issues, but at a cost: * - Ability to resize database on the fly requires an additional lock - * and release SlimReadWriteLock during each read-only transaction. + * and release `SlimReadWriteLock during` each read-only transaction. * - During resize all in-process threads should be paused and then resumed. * - Shrinking of database file is performed only when it used by single * process, i.e. when a database closes by the last process or opened @@ -2085,482 +2298,685 @@ LIBMDBX_API int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *fd); * remapping further. * * For create a new database with particular parameters, including the page - * size, mdbx_env_set_geometry() should be called after mdbx_env_create() and - * before mdbx_env_open(). Once the database is created, the page size cannot be - * changed. If you do not specify all or some of the parameters, the - * corresponding default values will be used. For instance, the default for - * database size is 10485760 bytes. + * size, \ref mdbx_env_set_geometry() should be called after + * \ref mdbx_env_create() and before mdbx_env_open(). Once the database is + * created, the page size cannot be changed. If you do not specify all or some + * of the parameters, the corresponding default values will be used. For + * instance, the default for database size is 10485760 bytes. * * If the mapsize is increased by another process, MDBX silently and * transparently adopt these changes at next transaction start. However, - * mdbx_txn_begin() will return MDBX_UNABLE_EXTEND_MAPSIZE if new mapping size - * could not be applied for current process (for instance if address space - * is busy). Therefore, in the case of MDBX_UNABLE_EXTEND_MAPSIZE error you - * need close and reopen the environment to resolve error. + * \ref mdbx_txn_begin() will return \ref MDBX_UNABLE_EXTEND_MAPSIZE if new + * mapping size could not be applied for current process (for instance if + * address space is busy). Therefore, in the case of + * \ref MDBX_UNABLE_EXTEND_MAPSIZE error you need close and reopen the + * environment to resolve error. * - * NOTE: Actual values may be different than your have specified because of + * \note Actual values may be different than your have specified because of * rounding to specified database page size, the system page size and/or the * size of the system virtual memory management unit. You can get actual values - * by mdbx_env_sync_ex() or see by using the tool "mdbx_chk" with the "-v" + * by \ref mdbx_env_sync_ex() or see by using the tool `mdbx_chk` with the `-v` * option. * - * Legacy mdbx_env_set_mapsize() correspond to calling mdbx_env_set_geometry() - * with the arguments size_lower, size_now, size_upper equal to the size - * and -1 (i.e. default) for all other parameters. + * Legacy \ref mdbx_env_set_mapsize() correspond to calling + * \ref mdbx_env_set_geometry() with the arguments `size_lower`, `size_now`, + * `size_upper` equal to the `size` and `-1` (i.e. default) for all other + * parameters. * - * [in] env An environment handle returned by mdbx_env_create() + * \param [in] env An environment handle returned + * by \ref mdbx_env_create() * - * [in] size_lower The lower bound of database sive in bytes. - * Zero value means "minimal acceptable", - * and negative means "keep current or use default". + * \param [in] size_lower The lower bound of database size in bytes. + * Zero value means "minimal acceptable", + * and negative means "keep current or use default". * - * [in] size_now The size in bytes to setup the database size for now. - * Zero value means "minimal acceptable", - * and negative means "keep current or use default". - * So, it is recommended always pass -1 in this argument - * except some special cases. + * \param [in] size_now The size in bytes to setup the database size for + * now. Zero value means "minimal acceptable", and + * negative means "keep current or use default". So, + * it is recommended always pass -1 in this argument + * except some special cases. * - * [in] size_upper The upper bound of database sive in bytes. + * \param [in] size_upper The upper bound of database size in bytes. * Zero value means "minimal acceptable", * and negative means "keep current or use default". * It is recommended to avoid change upper bound while * database is used by other processes or threaded * (i.e. just pass -1 in this argument except absolutely - * necessity). Otherwise you must be ready for - * MDBX_UNABLE_EXTEND_MAPSIZE error(s), unexpected pauses - * during remapping and/or system errors like "addtress - * busy", and so on. In other words, there is no way to - * handle a growth of the upper bound robustly because - * there may be a lack of appropriate system resources - * (which are extremely volatile in a multi-process - * multi-threaded environment). - * - * [in] growth_step The growth step in bytes, must be greater than zero - * to allow the database to grow. - * Negative value means "keep current or use default". - * - * [in] shrink_threshold The shrink threshold in bytes, must be greater than - * zero to allow the database to shrink. - * Negative value means "keep current or use default". - * - * [in] pagesize The database page size for new database creation - * or -1 otherwise. Must be power of 2 in the range - * between MDBX_MIN_PAGESIZE and MDBX_MAX_PAGESIZE. - * Zero value means "minimal acceptable", - * and negative means "keep current or use default". - * - * Returns A non-zero error value on failure and 0 on success, - * some possible errors are: - * - MDBX_EINVAL = An invalid parameter was specified, - * or the environment has an active write transaction. - * - MDBX_EPERM = specific for Windows: Shrinking was disabled before and - * now it wanna be enabled, but there are reading threads - * that don't use the additional SRWL (that is required to - * avoid Windows issues). - * - MDBX_EACCESS = The environment opened in read-only. - * - MDBX_MAP_FULL = Specified size smaller than the space already - * consumed by the environment. - * - MDBX_TOO_LARGE = Specified size is too large, i.e. too many pages for - * given size, or a 32-bit process requests too much bytes - * for the 32-bit address space. */ + * necessary). Otherwise you must be ready for + * \ref MDBX_UNABLE_EXTEND_MAPSIZE error(s), unexpected + * pauses during remapping and/or system errors like + * "address busy", and so on. In other words, there + * is no way to handle a growth of the upper bound + * robustly because there may be a lack of appropriate + * system resources (which are extremely volatile in + * a multi-process multi-threaded environment). + * + * \param [in] growth_step The growth step in bytes, must be greater than + * zero to allow the database to grow. Negative value + * means "keep current or use default". + * + * \param [in] shrink_threshold The shrink threshold in bytes, must be greater + * than zero to allow the database to shrink. + * Negative value means "keep current + * or use default". + * + * \param [in] pagesize The database page size for new database + * creation or -1 otherwise. Must be power of 2 + * in the range between \ref MDBX_MIN_PAGESIZE and + * \ref MDBX_MAX_PAGESIZE. Zero value means + * "minimal acceptable", and negative means + * "keep current or use default". + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified, + * or the environment has an active write transaction. + * \retval MDBX_EPERM Specific for Windows: Shrinking was disabled before + * and now it wanna be enabled, but there are reading + * threads that don't use the additional `SRWL` (that + * is required to avoid Windows issues). + * \retval MDBX_EACCESS The environment opened in read-only. + * \retval MDBX_MAP_FULL Specified size smaller than the space already + * consumed by the environment. + * \retval MDBX_TOO_LARGE Specified size is too large, i.e. too many pages for + * given size, or a 32-bit process requests too much + * bytes for the 32-bit address space. */ LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper, intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize); -__deprecated LIBMDBX_API int mdbx_env_set_mapsize(MDBX_env *env, size_t size); -/* Find out whether to use readahead or not, based on the given database size - * and the amount of available memory. - * - * [in] volume The expected database size in bytes. - * [in] redundancy Additional reserve or overload in case of negative value. - * - * Returns: - * - MDBX_RESULT_TRUE = readahead is reasonable. - * - MDBX_RESULT_FALSE = readahead is NOT reasonable, i.e. MDBX_NORDAHEAD - * is useful to open environment by mdbx_env_open(). - * - Otherwise the error code. */ +/** \deprecated Please use \ref mdbx_env_set_geometry() instead. + * \ingroup c_settings */ +MDBX_DEPRECATED LIBMDBX_API int mdbx_env_set_mapsize(MDBX_env *env, + size_t size); + +/** \brief Find out whether to use readahead or not, based on the given database + * size and the amount of available memory. \ingroup c_extra + * + * \param [in] volume The expected database size in bytes. + * \param [in] redundancy Additional reserve or overload in case of negative + * value. + * + * \returns A \ref MDBX_RESULT_TRUE or \ref MDBX_RESULT_FALSE value, + * otherwise the error code: + * \retval MDBX_RESULT_TRUE Readahead is reasonable. + * \retval MDBX_RESULT_FALSE Readahead is NOT reasonable, + * i.e. \ref MDBX_NORDAHEAD is useful to + * open environment by \ref mdbx_env_open(). + * \retval Otherwise the error code. */ LIBMDBX_API int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy); -/* The minimal database page size in bytes. */ -#define MDBX_MIN_PAGESIZE 256 -__inline intptr_t mdbx_limits_pgsize_min(void) { return MDBX_MIN_PAGESIZE; } - -/* The maximal database page size in bytes. */ -#define MDBX_MAX_PAGESIZE 65536 -__inline intptr_t mdbx_limits_pgsize_max(void) { return MDBX_MAX_PAGESIZE; } - -/* Returns minimal database size in bytes for given page size, - * or -1 if pagesize is invalid. */ -LIBMDBX_API intptr_t mdbx_limits_dbsize_min(intptr_t pagesize); - -/* Returns maximal database size in bytes for given page size, - * or -1 if pagesize is invalid. */ -LIBMDBX_API intptr_t mdbx_limits_dbsize_max(intptr_t pagesize); - -/* Returns maximal key and data size in bytes for given page size - * and database flags (see mdbx_dbi_open_ex() description), - * or -1 if pagesize is invalid. */ -LIBMDBX_API intptr_t mdbx_limits_keysize_max(intptr_t pagesize, unsigned flags); -LIBMDBX_API intptr_t mdbx_limits_valsize_max(intptr_t pagesize, unsigned flags); +/** \brief Returns the minimal database page size in bytes. + * \ingroup c_statinfo */ +MDBX_NOTHROW_CONST_FUNCTION __inline intptr_t mdbx_limits_pgsize_min(void) { + return MDBX_MIN_PAGESIZE; +} -/* Returns maximal write transaction size (i.e. limit for summary volume of - * dirty pages) in bytes for given page size, or -1 if pagesize is invalid. */ -LIBMDBX_API intptr_t mdbx_limits_txnsize_max(intptr_t pagesize); +/** \brief Returns the maximal database page size in bytes. + * \ingroup c_statinfo */ +MDBX_NOTHROW_CONST_FUNCTION __inline intptr_t mdbx_limits_pgsize_max(void) { + return MDBX_MAX_PAGESIZE; +} -/* Set the maximum number of threads/reader slots for the environment. +/** \brief Returns minimal database size in bytes for given page size, + * or -1 if pagesize is invalid. + * \ingroup c_statinfo */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t +mdbx_limits_dbsize_min(intptr_t pagesize); + +/** \brief Returns maximal database size in bytes for given page size, + * or -1 if pagesize is invalid. + * \ingroup c_statinfo */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t +mdbx_limits_dbsize_max(intptr_t pagesize); + +/** \brief Returns maximal key size in bytes for given page size + * and database flags, or -1 if pagesize is invalid. + * \ingroup c_statinfo + * \see db_flags */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t +mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags); + +/** \brief Returns maximal data size in bytes for given page size + * and database flags, or -1 if pagesize is invalid. + * \ingroup c_statinfo + * \see db_flags */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t +mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags); + +/** \brief Returns maximal write transaction size (i.e. limit for summary volume + * of dirty pages) in bytes for given page size, or -1 if pagesize is invalid. + * \ingroup c_statinfo */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t +mdbx_limits_txnsize_max(intptr_t pagesize); + +/** \brief Set the maximum number of threads/reader slots for the environment. + * \ingroup c_settings * * This defines the number of slots in the lock table that is used to track * readers in the the environment. The default is 119 for 4K system page size. * Starting a read-only transaction normally ties a lock table slot to the * current thread until the environment closes or the thread exits. If - * MDBX_NOTLS is in use, mdbx_txn_begin() instead ties the slot to the - * MDBX_txn object until it or the MDBX_env object is destroyed. - * This function may only be called after mdbx_env_create() and before - * mdbx_env_open(). - * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] readers The maximum number of reader lock table slots. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. - * - MDBX_EPERM = the environment is already open. */ + * \ref MDBX_NOTLS is in use, \ref mdbx_txn_begin() instead ties the slot to the + * \ref MDBX_txn object until it or the \ref MDBX_env object is destroyed. + * This function may only be called after \ref mdbx_env_create() and before + * \ref mdbx_env_open(). + * \see mdbx_env_get_maxreaders() + * + * \param [in] env An environment handle returned + * by \ref mdbx_env_create(). + * \param [in] readers The maximum number of reader lock table slots. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. + * \retval MDBX_EPERM The environment is already open. */ LIBMDBX_API int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers); -/* Get the maximum number of threads/reader slots for the environment. +/** \brief Get the maximum number of threads/reader slots for the environment. + * \ingroup c_statinfo + * \see mdbx_env_set_maxreaders() * - * [in] env An environment handle returned by mdbx_env_create(). - * [out] readers Address of an integer to store the number of readers. + * \param [in] env An environment handle returned + * by \ref mdbx_env_create(). + * \param [out] readers Address of an integer to store the number of readers. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers); -/* Set the maximum number of named databases for the environment. +/** \brief Set the maximum number of named databases for the environment. + * \ingroup c_settings * * This function is only needed if multiple databases will be used in the * environment. Simpler applications that use the environment as a single * unnamed database can ignore this option. - * This function may only be called after mdbx_env_create() and before - * mdbx_env_open(). + * This function may only be called after \ref mdbx_env_create() and before + * \ref mdbx_env_open(). * * Currently a moderate number of slots are cheap but a huge number gets - * expensive: 7-120 words per transaction, and every mdbx_dbi_open() + * expensive: 7-120 words per transaction, and every \ref mdbx_dbi_open() * does a linear search of the opened slots. + * \see mdbx_env_get_maxdbs() * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] dbs The maximum number of databases. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] dbs The maximum number of databases. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. - * - MDBX_EPERM = the environment is already open. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. + * \retval MDBX_EPERM The environment is already open. */ LIBMDBX_API int mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs); -/* Get the maximum size of keys and data we can write. +/** \brief Get the maximum number of named databases for the environment. + * \ingroup c_statinfo + * \see mdbx_env_set_maxdbs() * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] flags Database options (MDBX_DUPSORT, MDBX_INTEGERKEY ans so on), - * see mdbx_dbi_open_ex() description. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [out] dbs Address to store the maximum number of databases. * - * Returns The maximum size of a key we can write, - * or -1 if something is wrong. */ -LIBMDBX_API int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, unsigned flags); -LIBMDBX_API int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, unsigned flags); -__deprecated LIBMDBX_API int mdbx_env_get_maxkeysize(const MDBX_env *env); + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EINVAL An invalid parameter was specified. */ +LIBMDBX_API int mdbx_env_get_maxdbs(MDBX_env *env, MDBX_dbi *dbs); -/* Set application information associated with the MDBX_env. +/** \brief Get the maximum size of keys can write. + * \ingroup c_statinfo * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] ctx An arbitrary pointer for whatever the application needs. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] flags Database options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY + * and so on). \see db_flags + * + * \returns The maximum size of a key can write, + * or -1 if something is wrong. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int +mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags); + +/** \brief Get the maximum size of data we can write. + * \ingroup c_statinfo + * + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] flags Database options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY + * and so on). \see db_flags + * + * \returns The maximum size of a data can write, + * or -1 if something is wrong. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int +mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags); + +/** \deprecated Please use \ref mdbx_env_get_maxkeysize_ex() + * and/or \ref mdbx_env_get_maxvalsize_ex() + * \ingroup c_statinfo */ +MDBX_NOTHROW_PURE_FUNCTION MDBX_DEPRECATED LIBMDBX_API int +mdbx_env_get_maxkeysize(const MDBX_env *env); + +/** \brief Set application information associated with the \ref MDBX_env. + * \ingroup c_settings + * \see mdbx_env_get_userctx() * - * Returns A non-zero error value on failure and 0 on success. */ + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] ctx An arbitrary pointer for whatever the application needs. + * + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_env_set_userctx(MDBX_env *env, void *ctx); -/* Get the application information associated with the MDBX_env. - * - * [in] env An environment handle returned by mdbx_env_create() - * Returns The pointer set by mdbx_env_set_userctx(). */ -LIBMDBX_API void *mdbx_env_get_userctx(const MDBX_env *env); - -/* Create a transaction for use with the environment. - * - * The transaction handle may be discarded using mdbx_txn_abort() - * or mdbx_txn_commit(). - * - * NOTE: A transaction and its cursors must only be used by a single thread, - * and a thread may only have a single transaction at a time. If MDBX_NOTLS is - * in use, this does not apply to read-only transactions. - * - * NOTE: Cursors may not span transactions. - * - * [in] env An environment handle returned by mdbx_env_create() - * [in] parent If this parameter is non-NULL, the new transaction will be - * a nested transaction, with the transaction indicated by parent - * as its parent. Transactions may be nested to any level. - * A parent transaction and its cursors may not issue any other - * operations than mdbx_txn_commit and mdbx_txn_abort while it - * has active child transactions. - * [in] flags Special options for this transaction. This parameter - * must be set to 0 or by bitwise OR'ing together one or more - * of the values described here. - * - * - MDBX_RDONLY - * This transaction will not perform any write operations. - * - * - MDBX_TRYTXN - * Do not block when starting a write transaction. - * - * - MDBX_SAFE_NOSYNC, MDBX_NOMETASYNC or MDBX_MAPASYNC - * Do not sync data to disk corresponding to MDBX_NOMETASYNC - * or MDBX_SAFE_NOSYNC description (see abobe). - * - * [out] txn Address where the new MDBX_txn handle will be stored - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_PANIC = a fatal error occurred earlier and the environment - * must be shut down. - * - MDBX_UNABLE_EXTEND_MAPSIZE - * = another process wrote data beyond this MDBX_env's - * mapsize and this environment's map must be resized - * as well. See mdbx_env_set_mapsize(). - * - MDBX_READERS_FULL = a read-only transaction was requested and the reader - * lock table is full. See mdbx_env_set_maxreaders(). - * - MDBX_ENOMEM = out of memory. - * - MDBX_BUSY = the write transaction is already started by the - * current thread. */ -LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags, - MDBX_txn **txn); - -/* Information about the transaction */ -typedef struct MDBX_txn_info { - uint64_t txn_id; /* The ID of the transaction. For a READ-ONLY transaction, - this corresponds to the snapshot being read. */ - - uint64_t - txn_reader_lag; /* For READ-ONLY transaction: the lag from a recent - MVCC-snapshot, i.e. the number of committed - transaction since read transaction started. - For WRITE transaction (provided if scan_rlt=true): the - lag of the oldest reader from current transaction (i.e. - atleast 1 if any reader running). */ - - uint64_t txn_space_used; /* Used space by this transaction, i.e. corresponding - to the last used database page. */ - - uint64_t txn_space_limit_soft; /* Current size of database file. */ - - uint64_t - txn_space_limit_hard; /* Upper bound for size the database file, - i.e. the value "size_upper" argument of the - approriate call of mdbx_env_set_geometry(). */ - - uint64_t txn_space_retired; /* For READ-ONLY transaction: The total size of - the database pages that were retired by - committed write transactions after the reader's - MVCC-snapshot, i.e. the space which would be - freed after the Reader releases the - MVCC-snapshot for reuse by completion read - transaction. - For WRITE transaction: The summarized size of - the database pages that were retired for now - due Copy-On-Write during this transaction. */ - - uint64_t - txn_space_leftover; /* For READ-ONLY transaction: the space available for - writer(s) and that must be exhausted for reason to - call the OOM-killer for this read transaction. - For WRITE transaction: the space inside transaction - that left to MDBX_TXN_FULL error. */ - - uint64_t txn_space_dirty; /* For READ-ONLY transaction (provided if - scan_rlt=true): The space that actually become - available for reuse when only this transaction - will be finished. - For WRITE transaction: The summarized size of the - dirty database pages that generated during this - transaction. */ -} MDBX_txn_info; - -/* Return information about the MDBX transaction. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [out] stat The address of an MDBX_txn_info structure - * where the information will be copied. - * [in] scan_rlt The boolean flag controls the scan of the read lock table to - * provide complete information. Such scan is relatively - * expensive and you can avoid it if corresponding fields are - * not needed (see description of MDBX_txn_info above). - * - * Returns A non-zero error value on failure and 0 on success. */ +/** \brief Get the application information associated with the MDBX_env. + * \ingroup c_statinfo + * \see mdbx_env_set_userctx() + * + * \param [in] env An environment handle returned by \ref mdbx_env_create() + * \returns The pointer set by \ref mdbx_env_set_userctx() + * or `NULL` if something wrong. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * +mdbx_env_get_userctx(const MDBX_env *env); + +/** \brief Create a transaction for use with the environment. + * \ingroup c_transactions + * + * The transaction handle may be discarded using \ref mdbx_txn_abort() + * or \ref mdbx_txn_commit(). + * \see mdbx_txn_begin_ex() + * + * \note A transaction and its cursors must only be used by a single thread, + * and a thread may only have a single transaction at a time. If \ref MDBX_NOTLS + * is in use, this does not apply to read-only transactions. + * + * \note Cursors may not span transactions. + * + * \param [in] env An environment handle returned by \ref mdbx_env_create() + * \param [in] parent If this parameter is non-NULL, the new transaction will + * be a nested transaction, with the transaction indicated + * by parent as its parent. Transactions may be nested + * to any level. A parent transaction and its cursors may + * not issue any other operations than mdbx_txn_commit and + * \ref mdbx_txn_abort() while it has active child + * transactions. + * \param [in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one + * or more of the values described here: + * - \ref MDBX_RDONLY This transaction will not perform + * any write operations. + * + * - \ref MDBX_TXN_TRY Do not block when starting + * a write transaction. + * + * - \ref MDBX_SAFE_NOSYNC, \ref MDBX_NOMETASYNC. + * Do not sync data to disk corresponding + * to \ref MDBX_NOMETASYNC or \ref MDBX_SAFE_NOSYNC + * description. \see sync_modes + * + * \param [out] txn Address where the new MDBX_txn handle will be stored. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_PANIC A fatal error occurred earlier and the + * environment must be shut down. + * \retval MDBX_UNABLE_EXTEND_MAPSIZE Another process wrote data beyond + * this MDBX_env's mapsize and this + * environment map must be resized as well. + * See \ref mdbx_env_set_mapsize(). + * \retval MDBX_READERS_FULL A read-only transaction was requested and + * the reader lock table is full. + * See \ref mdbx_env_set_maxreaders(). + * \retval MDBX_ENOMEM Out of memory. + * \retval MDBX_BUSY The write transaction is already started by the + * current thread. */ +LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, + MDBX_txn_flags_t flags, MDBX_txn **txn); + +/** \brief Create a transaction with a user provided context pointer + * for use with the environment. + * \ingroup c_transactions + * + * The transaction handle may be discarded using \ref mdbx_txn_abort() + * or \ref mdbx_txn_commit(). + * \see mdbx_txn_begin() + * + * \note A transaction and its cursors must only be used by a single thread, + * and a thread may only have a single transaction at a time. If \ref MDBX_NOTLS + * is in use, this does not apply to read-only transactions. + * + * \note Cursors may not span transactions. + * + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * + * \param [in] parent If this parameter is non-NULL, the new transaction will + * be a nested transaction, with the transaction indicated + * by parent as its parent. Transactions may be nested + * to any level. A parent transaction and its cursors may + * not issue any other operations than mdbx_txn_commit and + * \ref mdbx_txn_abort() while it has active child + * transactions. + * + * \param [in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one + * or more of the values described here: + * - \ref MDBX_RDONLY This transaction will not perform + * any write operations. + * + * - \ref MDBX_TXN_TRY Do not block when starting + * a write transaction. + * + * - \ref MDBX_SAFE_NOSYNC, \ref MDBX_NOMETASYNC. + * Do not sync data to disk corresponding + * to \ref MDBX_NOMETASYNC or \ref MDBX_SAFE_NOSYNC + * description. \see sync_modes + * + * \param [out] txn Address where the new MDBX_txn handle will be stored. + * + * \param [in] context A pointer to application context to be associated with + * created transaction and could be retrieved by + * \ref mdbx_txn_get_userctx() until transaction finished. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_PANIC A fatal error occurred earlier and the + * environment must be shut down. + * \retval MDBX_UNABLE_EXTEND_MAPSIZE Another process wrote data beyond + * this MDBX_env's mapsize and this + * environment map must be resized as well. + * See \ref mdbx_env_set_mapsize(). + * \retval MDBX_READERS_FULL A read-only transaction was requested and + * the reader lock table is full. + * See \ref mdbx_env_set_maxreaders(). + * \retval MDBX_ENOMEM Out of memory. + * \retval MDBX_BUSY The write transaction is already started by the + * current thread. */ +LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, + MDBX_txn_flags_t flags, MDBX_txn **txn, + void *context); + +/** \brief Set application information associated with the \ref MDBX_txn. + * \ingroup c_transactions + * \see mdbx_txn_get_userctx() + * + * \param [in] txn An transaction handle returned by \ref mdbx_txn_begin_ex() + * or \ref mdbx_txn_begin(). + * \param [in] ctx An arbitrary pointer for whatever the application needs. + * + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx); + +/** \brief Get the application information associated with the MDBX_txn. + * \ingroup c_transactions + * \see mdbx_txn_set_userctx() + * + * \param [in] txn An transaction handle returned by \ref mdbx_txn_begin_ex() + * or \ref mdbx_txn_begin(). + * \returns The pointer which was passed via the `context` parameter + * of `mdbx_txn_begin_ex()` or set by \ref mdbx_txn_set_userctx(), + * or `NULL` if something wrong. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * +mdbx_txn_get_userctx(const MDBX_txn *txn); + +/** \brief Information about the transaction + * \ingroup c_statinfo + * \see mdbx_txn_info */ +struct MDBX_txn_info { + /** The ID of the transaction. For a READ-ONLY transaction, this corresponds + to the snapshot being read. */ + uint64_t txn_id; + + /** For READ-ONLY transaction: the lag from a recent MVCC-snapshot, i.e. the + number of committed transaction since read transaction started. For WRITE + transaction (provided if `scan_rlt=true`): the lag of the oldest reader + from current transaction (i.e. at least 1 if any reader running). */ + uint64_t txn_reader_lag; + + /** Used space by this transaction, i.e. corresponding to the last used + * database page. */ + uint64_t txn_space_used; + + /** Current size of database file. */ + uint64_t txn_space_limit_soft; + + /** Upper bound for size the database file, i.e. the value `size_upper` + argument of the appropriate call of \ref mdbx_env_set_geometry(). */ + uint64_t txn_space_limit_hard; + + /** For READ-ONLY transaction: The total size of the database pages that were + retired by committed write transactions after the reader's MVCC-snapshot, + i.e. the space which would be freed after the Reader releases the + MVCC-snapshot for reuse by completion read transaction. + For WRITE transaction: The summarized size of the database pages that were + retired for now due Copy-On-Write during this transaction. */ + uint64_t txn_space_retired; + + /** For READ-ONLY transaction: the space available for writer(s) and that + must be exhausted for reason to call the Handle-Slow-Readers callback for + this read transaction. For WRITE transaction: the space inside transaction + that left to `MDBX_TXN_FULL` error. */ + uint64_t txn_space_leftover; + + /** For READ-ONLY transaction (provided if `scan_rlt=true`): The space that + actually become available for reuse when only this transaction will be + finished. + For WRITE transaction: The summarized size of the dirty database + pages that generated during this transaction. */ + uint64_t txn_space_dirty; +}; +#ifndef __cplusplus +/** \ingroup c_statinfo */ +typedef struct MDBX_txn_info MDBX_txn_info; +#endif + +/** \brief Return information about the MDBX transaction. + * \ingroup c_statinfo + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() + * \param [out] info The address of an \ref MDBX_txn_info structure + * where the information will be copied. + * \param [in] scan_rlt The boolean flag controls the scan of the read lock + * table to provide complete information. Such scan + * is relatively expensive and you can avoid it + * if corresponding fields are not needed. + * See description of \ref MDBX_txn_info. + * + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, - int scan_rlt); + bool scan_rlt); -/* Returns the transaction's MDBX_env. +/** \brief Returns the transaction's MDBX_env. + * \ingroup c_transactions * - * [in] txn A transaction handle returned by mdbx_txn_begin() */ -LIBMDBX_API MDBX_env *mdbx_txn_env(const MDBX_txn *txn); + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_env * +mdbx_txn_env(const MDBX_txn *txn); -/* Return the transaction's flags. +/** \brief Return the transaction's flags. + * \ingroup c_transactions * * This returns the flags associated with this transaction. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). * - * Returns A transaction flags, valid if input is an valid transaction, - * otherwise -1. */ -LIBMDBX_API int mdbx_txn_flags(const MDBX_txn *txn); + * \returns A transaction flags, valid if input is an valid transaction, + * otherwise -1. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_txn_flags(const MDBX_txn *txn); -/* Return the transaction's ID. +/** \brief Return the transaction's ID. + * \ingroup c_statinfo * - * This returns the identifier associated with this transaction. For a read-only - * transaction, this corresponds to the snapshot being read; concurrent readers - * will frequently have the same transaction ID. + * This returns the identifier associated with this transaction. For a + * read-only transaction, this corresponds to the snapshot being read; + * concurrent readers will frequently have the same transaction ID. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). * - * Returns A transaction ID, valid if input is an active transaction, - * otherwise 0. */ -LIBMDBX_API uint64_t mdbx_txn_id(const MDBX_txn *txn); + * \returns A transaction ID, valid if input is an active transaction, + * otherwise 0. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t +mdbx_txn_id(const MDBX_txn *txn); -/* Commit all the operations of a transaction into the database. +/** \brief Commit all the operations of a transaction into the database. + * \ingroup c_transactions * * If the current thread is not eligible to manage the transaction then - * the MDBX_THREAD_MISMATCH error will returned. Otherwise the transaction + * the \ref MDBX_THREAD_MISMATCH error will returned. Otherwise the transaction * will be committed and its handle is freed. If the transaction cannot * be committed, it will be aborted with the corresponding error returned. - * Thus, a result other than MDBX_THREAD_MISMATCH means that the transaction - * is terminated: + * + * Thus, a result other than \ref MDBX_THREAD_MISMATCH means that the + * transaction is terminated: * - Resources are released; * - Transaction handle is invalid; * - Cursor(s) associated with transaction must not be used, except with - * mdbx_cursor_renew() and mdbx_cursor_close(). - * Such cursor(s) must be closed explicitly by mdbx_cursor_close() before - * or after transaction commit, either can be reused with mdbx_cursor_renew() - * until it will be explicitly closed by mdbx_cursor_close(). - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_RESULT_TRUE = transaction was aborted since it should be aborted - * due to previous errors. - * - MDBX_PANIC = a fatal error occurred earlier and the environment - * must be shut down. - * - MDBX_BAD_TXN = transaction is already fihished or never began. - * - MDBX_EBADSIGN = transaction object has invalid signature, - * e.g. transaction was already terminated - * or memory was corrupted. - * - MDBX_THREAD_MISMATCH = given transaction is not owned by current thread. - * - MDBX_EINVAL = transaction handle is NULL. - * - MDBX_ENOSPC = no more disk space. - * - MDBX_EIO = a system-level I/O error occurred while writing. - * - MDBX_ENOMEM = out of memory. */ + * mdbx_cursor_renew() and \ref mdbx_cursor_close(). + * Such cursor(s) must be closed explicitly by \ref mdbx_cursor_close() + * before or after transaction commit, either can be reused with + * \ref mdbx_cursor_renew() until it will be explicitly closed by + * \ref mdbx_cursor_close(). + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_RESULT_TRUE Transaction was aborted since it should + * be aborted due to previous errors. + * \retval MDBX_PANIC A fatal error occurred earlier + * and the environment must be shut down. + * \retval MDBX_BAD_TXN Transaction is already finished or never began. + * \retval MDBX_EBADSIGN Transaction object has invalid signature, + * e.g. transaction was already terminated + * or memory was corrupted. + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL Transaction handle is NULL. + * \retval MDBX_ENOSPC No more disk space. + * \retval MDBX_EIO A system-level I/O error occurred. + * \retval MDBX_ENOMEM Out of memory. */ LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn); -/* Abandon all the operations of the transaction instead of saving them. +/** \brief Abandon all the operations of the transaction instead of saving them. + * \ingroup c_transactions * * The transaction handle is freed. It and its cursors must not be used again - * after this call, except with mdbx_cursor_renew() and mdbx_cursor_close(). + * after this call, except with \ref mdbx_cursor_renew() and + * \ref mdbx_cursor_close(). * * If the current thread is not eligible to manage the transaction then - * the MDBX_THREAD_MISMATCH error will returned. Otherwise the transaction + * the \ref MDBX_THREAD_MISMATCH error will returned. Otherwise the transaction * will be aborted and its handle is freed. Thus, a result other than - * MDBX_THREAD_MISMATCH means that the transaction is terminated: + * \ref MDBX_THREAD_MISMATCH means that the transaction is terminated: * - Resources are released; * - Transaction handle is invalid; * - Cursor(s) associated with transaction must not be used, except with - * mdbx_cursor_renew() and mdbx_cursor_close(). - * Such cursor(s) must be closed explicitly by mdbx_cursor_close() before - * or after transaction abort, either can be reused with mdbx_cursor_renew() - * until it will be explicitly closed by mdbx_cursor_close(). - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_PANIC = a fatal error occurred earlier and the environment - * must be shut down. - * - MDBX_BAD_TXN = transaction is already fihished or never began. - * - MDBX_EBADSIGN = transaction object has invalid signature, - * e.g. transaction was already terminated - * or memory was corrupted. - * - MDBX_THREAD_MISMATCH = given transaction is not owned by current thread. - * - MDBX_EINVAL = transaction handle is NULL. */ + * \ref mdbx_cursor_renew() and \ref mdbx_cursor_close(). + * Such cursor(s) must be closed explicitly by \ref mdbx_cursor_close() + * before or after transaction abort, either can be reused with + * \ref mdbx_cursor_renew() until it will be explicitly closed by + * \ref mdbx_cursor_close(). + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_PANIC A fatal error occurred earlier and + * the environment must be shut down. + * \retval MDBX_BAD_TXN Transaction is already finished or never began. + * \retval MDBX_EBADSIGN Transaction object has invalid signature, + * e.g. transaction was already terminated + * or memory was corrupted. + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL Transaction handle is NULL. */ LIBMDBX_API int mdbx_txn_abort(MDBX_txn *txn); -/* Reset a read-only transaction. +/** \brief Marks transaction as broken. + * \ingroup c_transactions + * + * Function keeps the transaction handle and corresponding locks, but it + * is not possible to perform any operations in a broken transaction. + * Broken transaction must then be aborted explicitly later. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * + * \see mdbx_txn_abort() \see mdbx_txn_reset() \see mdbx_txn_commit() + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_break(MDBX_txn *txn); + +/** \brief Reset a read-only transaction. + * \ingroup c_transactions * - * Abort the read-only transaction like mdbx_txn_abort(), but keep the - * transaction handle. Therefore mdbx_txn_renew() may reuse the handle. This - * saves allocation overhead if the process will start a new read-only - * transaction soon, and also locking overhead if MDBX_NOTLS is in use. The - * reader table lock is released, but the table slot stays tied to its thread or - * MDBX_txn. Use mdbx_txn_abort() to discard a reset handle, and to free its - * lock table slot if MDBX_NOTLS is in use. + * Abort the read-only transaction like \ref mdbx_txn_abort(), but keep the + * transaction handle. Therefore \ref mdbx_txn_renew() may reuse the handle. + * This saves allocation overhead if the process will start a new read-only + * transaction soon, and also locking overhead if \ref MDBX_NOTLS is in use. The + * reader table lock is released, but the table slot stays tied to its thread + * or \ref MDBX_txn. Use \ref mdbx_txn_abort() to discard a reset handle, and to + * free its lock table slot if \ref MDBX_NOTLS is in use. * - * Cursors opened within the transaction must not be used again after this call, - * except with mdbx_cursor_renew() and mdbx_cursor_close(). + * Cursors opened within the transaction must not be used again after this + * call, except with \ref mdbx_cursor_renew() and \ref mdbx_cursor_close(). * * Reader locks generally don't interfere with writers, but they keep old * versions of database pages allocated. Thus they prevent the old pages from * being reused when writers commit new data, and so under heavy load the * database size may grow much more rapidly than otherwise. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_PANIC = a fatal error occurred earlier and the environment - * must be shut down. - * - MDBX_BAD_TXN = transaction is already fihished or never began. - * - MDBX_EBADSIGN = transaction object has invalid signature, - * e.g. transaction was already terminated - * or memory was corrupted. - * - MDBX_THREAD_MISMATCH = given transaction is not owned by current thread. - * - MDBX_EINVAL = transaction handle is NULL. */ + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_PANIC A fatal error occurred earlier and + * the environment must be shut down. + * \retval MDBX_BAD_TXN Transaction is already finished or never began. + * \retval MDBX_EBADSIGN Transaction object has invalid signature, + * e.g. transaction was already terminated + * or memory was corrupted. + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL Transaction handle is NULL. */ LIBMDBX_API int mdbx_txn_reset(MDBX_txn *txn); -/* Renew a read-only transaction. +/** \brief Renew a read-only transaction. + * \ingroup c_transactions * * This acquires a new reader lock for a transaction handle that had been - * released by mdbx_txn_reset(). It must be called before a reset transaction - * may be used again. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_PANIC = a fatal error occurred earlier and the environment - * must be shut down. - * - MDBX_BAD_TXN = transaction is already fihished or never began. - * - MDBX_EBADSIGN = transaction object has invalid signature, - * e.g. transaction was already terminated - * or memory was corrupted. - * - MDBX_THREAD_MISMATCH = transaction is running by other thread. - * - MDBX_EINVAL = transaction handle is NULL. */ + * released by \ref mdbx_txn_reset(). It must be called before a reset + * transaction may be used again. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_PANIC A fatal error occurred earlier and + * the environment must be shut down. + * \retval MDBX_BAD_TXN Transaction is already finished or never began. + * \retval MDBX_EBADSIGN Transaction object has invalid signature, + * e.g. transaction was already terminated + * or memory was corrupted. + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL Transaction handle is NULL. */ LIBMDBX_API int mdbx_txn_renew(MDBX_txn *txn); -/* The fours integers markers (aka "canary") associated with the environment. +/** \brief The fours integers markers (aka "canary") associated with the + * environment. \ingroup c_crud \see mdbx_canary_set() \see mdbx_canary_get() * - * The `x`, `y` and `z` values could be set by mdbx_canary_put(), while the 'v' - * will be always set to the transaction number. Updated values becomes visible - * outside the current transaction only after it was committed. Current values - * could be retrieved by mdbx_canary_get(). */ -typedef struct mdbx_canary { + * The `x`, `y` and `z` values could be set by \ref mdbx_canary_put(), while the + * 'v' will be always set to the transaction number. Updated values becomes + * visible outside the current transaction only after it was committed. Current + * values could be retrieved by \ref mdbx_canary_get(). */ +struct MDBX_canary { uint64_t x, y, z, v; -} mdbx_canary; +}; +#ifndef __cplusplus +/** \ingroup c_crud */ +typedef struct MDBX_canary MDBX_canary; +#endif -/* Set integers markers (aka "canary") associated with the environment. +/** \brief Set integers markers (aka "canary") associated with the environment. + * \ingroup c_crud + * \see mdbx_canary_get() * - * [in] txn A transaction handle returned by mdbx_txn_begin() - * [in] canary A optional pointer to mdbx_canary structure for `x`, `y` - * and `z` values from. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() + * \param [in] canary A optional pointer to \ref MDBX_canary structure for `x`, + * `y` and `z` values from. * - If canary is NOT NULL then the `x`, `y` and `z` values will be * updated from given canary argument, but the 'v' be always set * to the current transaction number if at least one `x`, `y` or @@ -2571,38 +2987,46 @@ typedef struct mdbx_canary { * to the current transaction number without changes `x`, `y` nor * `z`. * - * Returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary); + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary); -/* Returns fours integers markers (aka "canary") associated with the +/** \brief Returns fours integers markers (aka "canary") associated with the * environment. + * \ingroup c_crud + * \see mdbx_canary_set() * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] canary The address of an mdbx_canary structure where the information - * will be copied. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] canary The address of an MDBX_canary structure where the + * information will be copied. * - * Returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_canary_get(const MDBX_txn *txn, mdbx_canary *canary); + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary); -/* A callback function used to compare two keys in a database */ -typedef int(MDBX_cmp_func)(const MDBX_val *a, const MDBX_val *b); +/** \brief A callback function used to compare two keys in a database + * \ingroup c_crud + * \see mdbx_cmp() \see mdbx_get_keycmp() + * \see mdbx_get_datacmp \see mdbx_dcmp() */ +typedef int(MDBX_cmp_func)(const MDBX_val *a, + const MDBX_val *b) MDBX_CXX17_NOEXCEPT; -/* Open a database in the environment. +/** \brief Open or Create a database in the environment. + * \ingroup c_dbi * * A database handle denotes the name and parameters of a database, * independently of whether such a database exists. The database handle may be - * discarded by calling mdbx_dbi_close(). The old database handle is returned if - * the database was already open. The handle may only be closed once. + * discarded by calling \ref mdbx_dbi_close(). The old database handle is + * returned if the database was already open. The handle may only be closed + * once. * - * (!) A notable difference between MDBX and LMDB is that MDBX make handles + * \note A notable difference between MDBX and LMDB is that MDBX make handles * opened for existing databases immediately available for other transactions, * regardless this transaction will be aborted or reset. The REASON for this is - * to avoiding the requirement for multiple opening a same handles in concurrent - * read transactions, and tracking of such open but hidden handles until the - * completion of read transactions which opened them. + * to avoiding the requirement for multiple opening a same handles in + * concurrent read transactions, and tracking of such open but hidden handles + * until the completion of read transactions which opened them. * - * Nevertheless, the handle for the NEWLY CREATED database will be invisible for - * other transactions until the this write transaction is successfully + * Nevertheless, the handle for the NEWLY CREATED database will be invisible + * for other transactions until the this write transaction is successfully * committed. If the write transaction is aborted the handle will be closed * automatically. After a successful commit the such handle will reside in the * shared environment, and may be used by other transactions. @@ -2610,332 +3034,438 @@ typedef int(MDBX_cmp_func)(const MDBX_val *a, const MDBX_val *b); * In contrast to LMDB, the MDBX allow this function to be called from multiple * concurrent transactions or threads in the same process. * - * To use named database (with name != NULL), mdbx_env_set_maxdbs() + * To use named database (with name != NULL), \ref mdbx_env_set_maxdbs() * must be called before opening the environment. Table names are * keys in the internal unnamed database, and may be read but not written. * - * [in] txn transaction handle returned by mdbx_txn_begin(). - * [in] name The name of the database to open. If only a single - * database is needed in the environment, this value may be NULL. - * [in] flags Special options for this database. This parameter must be set - * to 0 or by bitwise OR'ing together one or more of the values - * described here: - * - MDBX_REVERSEKEY + * \param [in] txn transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] name The name of the database to open. If only a single + * database is needed in the environment, + * this value may be NULL. + * \param [in] flags Special options for this database. This parameter must + * be set to 0 or by bitwise OR'ing together one or more + * of the values described here: + * - \ref MDBX_REVERSEKEY * Keys are strings to be compared in reverse order, from the end * of the strings to the beginning. By default, Keys are treated as * strings and compared from beginning to end. - * - MDBX_DUPSORT + * - \ref MDBX_INTEGERKEY + * Keys are binary integers in native byte order, either uint32_t or + * uint64_t, and will be sorted as such. The keys must all be of the + * same size and must be aligned while passing as arguments. + * - \ref MDBX_DUPSORT * Duplicate keys may be used in the database. Or, from another point of * view, keys may have multiple data items, stored in sorted order. By * default keys must be unique and may have only a single data item. - * - MDBX_INTEGERKEY - * Keys are binary integers in native byte order, either uin32_t or - * uint64_t, and will be sorted as such. The keys must all be of the - * same size and must be aligned while passing as arguments. - * - MDBX_DUPFIXED - * This flag may only be used in combination with MDBX_DUPSORT. This + * - \ref MDBX_DUPFIXED + * This flag may only be used in combination with \ref MDBX_DUPSORT. This * option tells the library that the data items for this database are * all the same size, which allows further optimizations in storage and - * retrieval. When all data items are the same size, the MDBX_GET_MULTIPLE, - * MDBX_NEXT_MULTIPLE and MDBX_PREV_MULTIPLE cursor operations may be used - * to retrieve multiple items at once. - * - MDBX_INTEGERDUP + * retrieval. When all data items are the same size, the + * \ref MDBX_GET_MULTIPLE, \ref MDBX_NEXT_MULTIPLE and + * \ref MDBX_PREV_MULTIPLE cursor operations may be used to retrieve + * multiple items at once. + * - \ref MDBX_INTEGERDUP * This option specifies that duplicate data items are binary integers, - * similar to MDBX_INTEGERKEY keys. The data values must all be of the + * similar to \ref MDBX_INTEGERKEY keys. The data values must all be of the * same size and must be aligned while passing as arguments. - * - MDBX_REVERSEDUP + * - \ref MDBX_REVERSEDUP * This option specifies that duplicate data items should be compared as * strings in reverse order (the comparison is performed in the direction * from the last byte to the first). - * - MDBX_CREATE + * - \ref MDBX_CREATE * Create the named database if it doesn't exist. This option is not * allowed in a read-only transaction or a read-only environment. * - * [out] dbi Address where the new MDBX_dbi handle will be stored. + * \param [out] dbi Address where the new \ref MDBX_dbi handle + * will be stored. * - * For mdbx_dbi_open_ex() additional arguments allow you to set custom + * For \ref mdbx_dbi_open_ex() additional arguments allow you to set custom * comparison functions for keys and values (for multimaps). * However, I recommend not using custom comparison functions, but instead * converting the keys to one of the forms that are suitable for built-in - * comparators (for instance take look to the mdbx_key_from_xxx() - * functions). The reasons to not using custom comparators are: + * comparators (for instance take look to the \ref value2key). + * The reasons to not using custom comparators are: * - The order of records could not be validated without your code. - * So mdbx_chk utility will reports "wrong order" errors - * and the '-i' option is required to ignore ones. + * So `mdbx_chk` utility will reports "wrong order" errors + * and the `-i` option is required to ignore ones. * - A records could not be ordered or sorted without your code. - * So mdbx_load utility should be used with '-a' option to preserve + * So mdbx_load utility should be used with `-a` option to preserve * input data order. * - * [in] keycmp Optional custom key comparison function for a database. - * [in] datacmp Optional custom data comparison function for a database, takes - * effect only if database was opened with the MDB_DUPSORT flag. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_NOTFOUND = the specified database doesn't exist in the - * environment and MDBX_CREATE was not specified. - * - MDBX_DBS_FULL = too many databases have been opened. - * See mdbx_env_set_maxdbs(). - * - MDBX_INCOMPATIBLE = Database is incompatible with given flags, + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_NOTFOUND The specified database doesn't exist in the + * environment and \ref MDBX_CREATE was not specified. + * \retval MDBX_DBS_FULL Too many databases have been opened. + * \see mdbx_env_set_maxdbs() + * \retval MDBX_INCOMPATIBLE Database is incompatible with given flags, * i.e. the passed flags is different with which the * database was created, or the database was already - * opened with a different comparison function(s). */ -LIBMDBX_API int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, - unsigned flags, MDBX_dbi *dbi, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); -LIBMDBX_API int mdbx_dbi_open(MDBX_txn *txn, const char *name, unsigned flags, - MDBX_dbi *dbi); - -/* Key-making (value-to-key) functions to avoid custom comparators. - * - * The mdbx_key_from_jsonInteger() build key which are comparable with - * keys created by mdbx_key_from_double(). So this allow mix int64 and IEEE754 - * double values in one index for JSON-numbers with restriction for integer - * numbers range corresponding to RFC-7159 (i.e. [-(2**53)+1, (2**53)-1]. - * See bottom of page 6 at https://tools.ietf.org/html/rfc7159 */ -LIBMDBX_API uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer); -LIBMDBX_API uint64_t mdbx_key_from_double(const double ieee754_64bit); -LIBMDBX_API uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit); -LIBMDBX_API uint32_t mdbx_key_from_float(const float ieee754_32bit); -LIBMDBX_API uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit); -__inline uint64_t mdbx_key_from_int64(const int64_t i64) { + * opened with a different comparison function(s). + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. */ +LIBMDBX_API int mdbx_dbi_open(MDBX_txn *txn, const char *name, + MDBX_db_flags_t flags, MDBX_dbi *dbi); + +/** \deprecated Please avoid using custom comparators + * and use mdbx_dbi_open() instead. + * \ingroup c_dbi + * + * \param [in] txn transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] name The name of the database to open. If only a single + * database is needed in the environment, + * this value may be NULL. + * \param [in] flags Special options for this database. + * \param [in] keycmp Optional custom key comparison function for a database. + * \param [in] datacmp Optional custom data comparison function for a database. + * \param [out] dbi Address where the new MDBX_dbi handle will be stored. + * \returns A non-zero error value on failure and 0 on success. */ +MDBX_DEPRECATED LIBMDBX_API int +mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, + MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); + +/** \defgroup value2key Value-to-Key functions to avoid custom comparators + * \see key2value + * @{ + * + * The \ref mdbx_key_from_jsonInteger() build a keys which are comparable with + * keys created by \ref mdbx_key_from_double(). So this allows mixing `int64_t` + * and IEEE754 double values in one index for JSON-numbers with restriction for + * integer numbers range corresponding to RFC-7159, i.e. \f$[-2^{53}+1, + * 2^{53}-1]\f$. See bottom of page 6 at https://tools.ietf.org/html/rfc7159 */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t +mdbx_key_from_jsonInteger(const int64_t json_integer); + +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t +mdbx_key_from_double(const double ieee754_64bit); + +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t +mdbx_key_from_ptrdouble(const double *const ieee754_64bit); + +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint32_t +mdbx_key_from_float(const float ieee754_32bit); + +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint32_t +mdbx_key_from_ptrfloat(const float *const ieee754_32bit); + +MDBX_NOTHROW_CONST_FUNCTION __inline uint64_t +mdbx_key_from_int64(const int64_t i64) { return UINT64_C(0x8000000000000000) + i64; } -__inline uint32_t mdbx_key_from_int32(const int32_t i32) { + +MDBX_NOTHROW_CONST_FUNCTION __inline uint32_t +mdbx_key_from_int32(const int32_t i32) { return UINT32_C(0x80000000) + i32; } +/** @} */ -/* Key-reverse (key-to-value) functions to avoid custom comparators. */ -LIBMDBX_API int64_t mdbx_jsonInteger_from_key(const MDBX_val); -LIBMDBX_API double mdbx_double_from_key(const MDBX_val); -LIBMDBX_API float mdbx_float_from_key(const MDBX_val); -LIBMDBX_API int32_t mdbx_int32_from_key(const MDBX_val); -LIBMDBX_API int64_t mdbx_int64_from_key(const MDBX_val); +/** \defgroup key2value Key-to-Value functions to avoid custom comparators + * \see value2key + * @{ */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t +mdbx_jsonInteger_from_key(const MDBX_val); -/* Retrieve statistics for a database. +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API double +mdbx_double_from_key(const MDBX_val); + +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API float +mdbx_float_from_key(const MDBX_val); + +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int32_t +mdbx_int32_from_key(const MDBX_val); + +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t +mdbx_int64_from_key(const MDBX_val); +/** @} */ + +/** \brief Retrieve statistics for a database. + * \ingroup c_statinfo * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [out] stat The address of an MDBX_stat structure where the statistics - * will be copied. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [out] stat The address of an \ref MDBX_stat structure where + * the statistics will be copied. + * \param [in] bytes The size of \ref MDBX_stat. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *stat, size_t bytes); -/* Retrieve depth (bitmask) information of nested dupsort (multi-value) B+trees - * for given database. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [out] mask The address of an uint32_t value where the bitmask - * will be stored. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. - * - MDBX_RESULT_TRUE = the dbi isn't a dupsort (multi-value) database. */ +/** \brief Retrieve depth (bitmask) information of nested dupsort (multi-value) + * B+trees for given database. + * \ingroup c_statinfo + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [out] mask The address of an uint32_t value where the bitmask + * will be stored. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL An invalid parameter was specified. + * \retval MDBX_RESULT_TRUE The dbi isn't a dupsort (multi-value) database. */ LIBMDBX_API int mdbx_dbi_dupsort_depthmask(MDBX_txn *txn, MDBX_dbi dbi, uint32_t *mask); -/* Retrieve the DB flags and status for a database handle. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [out] flags Address where the flags will be returned. - * [out] state Address where the state will be returned. +/** \brief DBI state bits returted by \ref mdbx_dbi_flags_ex() + * \ingroup c_statinfo + * \see mdbx_dbi_flags_ex() */ +enum MDBX_dbi_state_t { + /** DB was written in this txn */ + MDBX_DBI_DIRTY = 0x01, + /** Named-DB record is older than txnID */ + MDBX_DBI_STALE = 0x02, + /** Named-DB handle opened in this txn */ + MDBX_DBI_FRESH = 0x04, + /** Named-DB handle created in this txn */ + MDBX_DBI_CREAT = 0x08, +}; +#ifndef __cplusplus +/** \ingroup c_statinfo */ +typedef enum MDBX_dbi_state_t MDBX_dbi_state_t; +#else +DEFINE_ENUM_FLAG_OPERATORS(MDBX_dbi_state_t) +#endif + +/** \brief Retrieve the DB flags and status for a database handle. + * \ingroup c_statinfo * - * Legacy mdbx_dbi_flags() correspond to calling mdbx_dbi_flags_ex() with - * discarding result from the last argument. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [out] flags Address where the flags will be returned. + * \param [out] state Address where the state will be returned. * - * Returns A non-zero error value on failure and 0 on success. */ -#define MDBX_DBI_DIRTY 0x01 /* DB was written in this txn */ -#define MDBX_DBI_STALE 0x02 /* Named-DB record is older than txnID */ -#define MDBX_DBI_FRESH 0x04 /* Named-DB handle opened in this txn */ -#define MDBX_DBI_CREAT 0x08 /* Named-DB handle created in this txn */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_dbi_flags_ex(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, unsigned *state); +/** \brief The shortcut to calling \ref mdbx_dbi_flags_ex() with `state=NULL` + * for discarding it result. \ingroup c_statinfo */ LIBMDBX_API int mdbx_dbi_flags(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags); -/* Close a database handle. Normally unnecessary. +/** \brief Close a database handle. Normally unnecessary. + * \ingroup c_dbi * - * NOTE: Use with care. - * This call is synchronized via mutex with mdbx_dbi_close(), but NOT with + * Closing a database handle is not necessary, but lets \ref mdbx_dbi_open() + * reuse the handle value. Usually it's better to set a bigger + * \ref mdbx_env_set_maxdbs(), unless that value would be large. + * + * \note Use with care. + * This call is synchronized via mutex with \ref mdbx_dbi_close(), but NOT with * other transactions running by other threads. The "next" version of libmdbx - * (MithrilDB) will solve this issue. + * (\ref MithrilDB) will solve this issue. * * Handles should only be closed if no other threads are going to reference * the database handle or one of its cursors any further. Do not close a handle * if an existing transaction has modified its database. Doing so can cause - * misbehavior from database corruption to errors like MDBX_BAD_VALSIZE (since - * the DB name is gone). - * - * Closing a database handle is not necessary, but lets mdbx_dbi_open() reuse - * the handle value. Usually it's better to set a bigger mdbx_env_set_maxdbs(), - * unless that value would be large. + * misbehavior from database corruption to errors like \ref MDBX_BAD_DBI + * (since the DB name is gone). * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] dbi A database handle returned by mdbx_dbi_open(). + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi); -/* Empty or delete and close a database. +/** \brief Empty or delete and close a database. + * \ingroup c_crud * - * See mdbx_dbi_close() for restrictions about closing the DB handle. + * \see mdbx_dbi_close() \see mdbx_dbi_open() * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] del 0 to empty the DB, 1 to delete it from the environment - * and close the DB handle. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] del `false` to empty the DB, `true` to delete it + * from the environment and close the DB handle. * - * Returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, int del); + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del); -/* Get items from a database. +/** \brief Get items from a database. + * \ingroup c_crud * * This function retrieves key/data pairs from the database. The address * and length of the data associated with the specified key are returned * in the structure to which data refers. - * If the database supports duplicate keys (MDBX_DUPSORT) then the + * If the database supports duplicate keys (\ref MDBX_DUPSORT) then the * first data item for the key will be returned. Retrieval of other - * items requires the use of mdbx_cursor_get(). + * items requires the use of \ref mdbx_cursor_get(). * - * NOTE: The memory pointed to by the returned values is owned by the + * \note The memory pointed to by the returned values is owned by the * database. The caller need not dispose of the memory, and may not * modify it in any way. For values returned in a read-only transaction - * any modification attempts will cause a SIGSEGV. + * any modification attempts will cause a `SIGSEGV`. * - * NOTE: Values returned from the database are valid only until a + * \note Values returned from the database are valid only until a * subsequent update operation, or the end of the transaction. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] key The key to search for in the database. - * [in,out] data The data corresponding to the key. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_NOTFOUND = the key was not in the database. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to search for in the database. + * \param [in,out] data The data corresponding to the key. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_NOTFOUND The key was not in the database. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_get(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data); -/* Get items from a database and optionaly number of data items for a given key. +/** \brief Get items from a database + * and optionally number of data items for a given key. * - * Briefly this function does the same as mdbx_get() with a few differences: + * \ingroup c_crud + * + * Briefly this function does the same as \ref mdbx_get() with a few + * differences: * 1. If values_count is NOT NULL, then returns the count * of multi-values/duplicates for a given key. * 2. Updates BOTH the key and the data for pointing to the actual key-value * pair inside the database. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in,out] key The key to search for in the database. - * [in,out] data The data corresponding to the key. - * [out] values_count The optional address to return number of values - * associated with given key, i.e. - * = 0 - in case MDBX_NOTFOUND error; - * = 1 - exactly for databases WITHOUT MDBX_DUPSORT; - * >= 1 for databases WITH MDBX_DUPSORT. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_NOTFOUND = the key was not in the database. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \param [in] txn A transaction handle returned + * by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in,out] key The key to search for in the database. + * \param [in,out] data The data corresponding to the key. + * \param [out] values_count The optional address to return number of values + * associated with given key: + * = 0 - in case \ref MDBX_NOTFOUND error; + * = 1 - exactly for databases + * WITHOUT \ref MDBX_DUPSORT; + * >= 1 for databases WITH \ref MDBX_DUPSORT. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_NOTFOUND The key was not in the database. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_get_ex(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, size_t *values_count); -/* Get nearest items from a database. +/** \brief Get equal or great item from a database. + * \ingroup c_crud * - * Briefly this function does the same as mdbx_get() with a few differences: - * 1. Return nearest (i.e. equal or great due comparison function) key-value + * Briefly this function does the same as \ref mdbx_get() with a few + * differences: + * 1. Return equal or great (due comparison function) key-value * pair, but not only exactly matching with the key. - * 2. On success return MDBX_SUCCESS if key found exactly, - * and MDBX_RESULT_TRUE otherwise. Moreover, for databases with MDBX_DUPSORT - * flag the data argument also will be used to match over - * multi-value/duplicates, and MDBX_SUCCESS will be returned only when BOTH - * the key and the data match exactly. + * 2. On success return \ref MDBX_SUCCESS if key found exactly, + * and \ref MDBX_RESULT_TRUE otherwise. Moreover, for databases with + * \ref MDBX_DUPSORT flag the data argument also will be used to match over + * multi-value/duplicates, and \ref MDBX_SUCCESS will be returned only when + * BOTH the key and the data match exactly. * 3. Updates BOTH the key and the data for pointing to the actual key-value * pair inside the database. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in,out] key The key to search for in the database. - * [in,out] data The data corresponding to the key. - * - * Returns A non-zero error value on failure and MDBX_RESULT_TRUE (0) or - * MDBX_RESULT_TRUE on success (as described above). - * Some possible errors are: - * - MDBX_NOTFOUND = the key was not in the database. - * - MDBX_EINVAL = an invalid parameter was specified. */ -LIBMDBX_API int mdbx_get_nearest(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data); - -/* Store items into a database. + * \param [in] txn A transaction handle returned + * by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in,out] key The key to search for in the database. + * \param [in,out] data The data corresponding to the key. + * + * \returns A non-zero error value on failure and \ref MDBX_RESULT_FALSE + * or \ref MDBX_RESULT_TRUE on success (as described above). + * Some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_NOTFOUND The key was not in the database. + * \retval MDBX_EINVAL An invalid parameter was specified. */ +LIBMDBX_API int mdbx_get_equal_or_great(MDBX_txn *txn, MDBX_dbi dbi, + MDBX_val *key, MDBX_val *data); + +/** \brief Store items into a database. + * \ingroup c_crud * * This function stores key/data pairs in the database. The default behavior * is to enter the new key/data pair, replacing any previously existing key * if duplicates are disallowed, or adding a duplicate data item if - * duplicates are allowed (MDBX_DUPSORT). - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] key The key to store in the database. - * [in,out] data The data to store. - * [in] flags Special options for this operation. This parameter must be - * set to 0 or by bitwise OR'ing together one or more of the - * values described here. - * - * - MDBX_NODUPDATA - * Enter the new key/data pair only if it does not already appear + * duplicates are allowed (see \ref MDBX_DUPSORT). + * + * \param [in] txn A transaction handle returned + * by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to store in the database. + * \param [in,out] data The data to store. + * \param [in] flags Special options for this operation. + * This parameter must be set to 0 or by bitwise OR'ing + * together one or more of the values described here: + * - \ref MDBX_NODUPDATA + * Enter the new key-value pair only if it does not already appear * in the database. This flag may only be specified if the database - * was opened with MDBX_DUPSORT. The function will return MDBX_KEYEXIST - * if the key/data pair already appears in the database. + * was opened with \ref MDBX_DUPSORT. The function will return + * \ref MDBX_KEYEXIST if the key/data pair already appears in the database. * - * - MDBX_NOOVERWRITE + * - \ref MDBX_NOOVERWRITE * Enter the new key/data pair only if the key does not already appear - * in the database. The function will return MDBX_KEYEXIST if the key + * in the database. The function will return \ref MDBX_KEYEXIST if the key * already appears in the database, even if the database supports - * duplicates (MDBX_DUPSORT). The data parameter will be set to point - * to the existing item. + * duplicates (see \ref MDBX_DUPSORT). The data parameter will be set + * to point to the existing item. * - * - MDBX_CURRENT - * Update an single existing entry, but not add new ones. The function - * will return MDBX_NOTFOUND if the given key not exist in the database. - * Or the MDBX_EMULTIVAL in case duplicates for the given key. + * - \ref MDBX_CURRENT + * Update an single existing entry, but not add new ones. The function will + * return \ref MDBX_NOTFOUND if the given key not exist in the database. + * In case multi-values for the given key, with combination of + * the \ref MDBX_ALLDUPS will replace all multi-values, + * otherwise return the \ref MDBX_EMULTIVAL. * - * - MDBX_RESERVE + * - \ref MDBX_RESERVE * Reserve space for data of the given size, but don't copy the given * data. Instead, return a pointer to the reserved space, which the * caller can fill in later - before the next update operation or the * transaction ends. This saves an extra memcpy if the data is being * generated later. MDBX does nothing else with this memory, the caller * is expected to modify all of the space requested. This flag must not - * be specified if the database was opened with MDBX_DUPSORT. + * be specified if the database was opened with \ref MDBX_DUPSORT. * - * - MDBX_APPEND + * - \ref MDBX_APPEND * Append the given key/data pair to the end of the database. This option * allows fast bulk loading when keys are already known to be in the * correct order. Loading unsorted keys with this flag will cause - * a MDBX_EKEYMISMATCH error. + * a \ref MDBX_EKEYMISMATCH error. * - * - MDBX_APPENDDUP + * - \ref MDBX_APPENDDUP * As above, but for sorted dup data. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_KEYEXIST - * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). - * - MDBX_TXN_FULL = the transaction has too many dirty pages. - * - MDBX_EACCES = an attempt was made to write in a read-only transaction. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * - \ref MDBX_MULTIPLE + * Store multiple contiguous data elements in a single request. This flag + * may only be specified if the database was opened with + * \ref MDBX_DUPFIXED. With combination the \ref MDBX_ALLDUPS + * will replace all multi-values. + * The data argument must be an array of two \ref MDBX_val. The `iov_len` + * of the first \ref MDBX_val must be the size of a single data element. + * The `iov_base` of the first \ref MDBX_val must point to the beginning + * of the array of contiguous data elements which must be properly aligned + * in case of database with \ref MDBX_INTEGERDUP flag. + * The `iov_len` of the second \ref MDBX_val must be the count of the + * number of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The `iov_base` of + * the second \ref MDBX_val is unused. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_KEYEXIST The key/value pair already exists in the database. + * \retval MDBX_MAP_FULL The database is full, see \ref mdbx_env_set_mapsize(). + * \retval MDBX_TXN_FULL The transaction has too many dirty pages. + * \retval MDBX_EACCES An attempt was made to write + * in a read-only transaction. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *data, unsigned flags); + MDBX_val *data, MDBX_put_flags_t flags); -/* Replace items in a database. +/** \brief Replace items in a database. + * \ingroup c_crud * * This function allows to update or delete an existing value at the same time * as the previous value is retrieved. If the argument new_data equal is NULL @@ -2943,303 +3473,447 @@ LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, * * The current value may be in an already changed (aka dirty) page. In this * case, the page will be overwritten during the update, and the old value will - * be lost. Therefore, an additional buffer must be passed via old_data argument - * initially to copy the old value. If the buffer passed in is too small, the - * function will return MDBX_RESULT_TRUE (-1) by setting iov_len field pointed - * by old_data argument to the appropriate value, without performing any - * changes. - * - * For databases with non-unique keys (i.e. with MDBX_DUPSORT flag), another use - * case is also possible, when by old_data argument selects a specific item from - * multi-value/duplicates with the same key for deletion or update. To select - * this scenario in flags should simultaneously specify MDBX_CURRENT and - * MDBX_NOOVERWRITE. This combination is chosen because it makes no sense, and - * thus allows you to identify the request of such a scenario. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] key The key to store in the database. - * [in,out] new_data The data to store, if NULL then deletion will be - * performed. - * [in,out] old_data The buffer for retrieve previous value as describe - * above. - * [in] flags Special options for this operation. This parameter must - * be set to 0 or by bitwise OR'ing together one or more of - * the values described in mdbx_put() description above, - * and additionally (MDBX_CURRENT | MDBX_NOOVERWRITE) - * combination for selection particular item from - * multi-value/duplicates. - * - * Returns A non-zero error value on failure and 0 on success. */ + * be lost. Therefore, an additional buffer must be passed via old_data + * argument initially to copy the old value. If the buffer passed in is too + * small, the function will return \ref MDBX_RESULT_TRUE by setting iov_len + * field pointed by old_data argument to the appropriate value, without + * performing any changes. + * + * For databases with non-unique keys (i.e. with \ref MDBX_DUPSORT flag), + * another use case is also possible, when by old_data argument selects a + * specific item from multi-value/duplicates with the same key for deletion or + * update. To select this scenario in flags should simultaneously specify + * \ref MDBX_CURRENT and \ref MDBX_NOOVERWRITE. This combination is chosen + * because it makes no sense, and thus allows you to identify the request of + * such a scenario. + * + * \param [in] txn A transaction handle returned + * by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to store in the database. + * \param [in] new_data The data to store, if NULL then deletion will + * be performed. + * \param [in,out] old_data The buffer for retrieve previous value as describe + * above. + * \param [in] flags Special options for this operation. + * This parameter must be set to 0 or by bitwise + * OR'ing together one or more of the values + * described in \ref mdbx_put() description above, + * and additionally + * (\ref MDBX_CURRENT | \ref MDBX_NOOVERWRITE) + * combination for selection particular item from + * multi-value/duplicates. + * + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data, - unsigned flags); + MDBX_put_flags_t flags); -/* Delete items from a database. +typedef int (*MDBX_preserve_func)(void *context, MDBX_val *target, + const void *src, size_t bytes); +LIBMDBX_API int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, + const MDBX_val *key, MDBX_val *new_data, + MDBX_val *old_data, MDBX_put_flags_t flags, + MDBX_preserve_func preserver, + void *preserver_context); + +/** \brief Delete items from a database. + * \ingroup c_crud * * This function removes key/data pairs from the database. * - * NOTE: The data parameter is NOT ignored regardless the database does + * \note The data parameter is NOT ignored regardless the database does * support sorted duplicate data items or not. If the data parameter * is non-NULL only the matching data item will be deleted. Otherwise, if data * parameter is NULL, any/all value(s) for specified key will be deleted. * - * This function will return MDBX_NOTFOUND if the specified key/data + * This function will return \ref MDBX_NOTFOUND if the specified key/data * pair is not in the database. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] key The key to delete from the database. - * [in] data The data to delete. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to delete from the database. + * \param [in] data The data to delete. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EACCES = an attempt was made to write in a read-only transaction. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EACCES An attempt was made to write + * in a read-only transaction. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, const MDBX_val *data); -/* Create a cursor handle. +/** \brief Create a cursor handle but not bind it to transaction nor DBI handle. + * \ingroup c_cursors * - * A cursor is associated with a specific transaction and database. A cursor - * cannot be used when its database handle is closed. Nor when its transaction - * has ended, except with mdbx_cursor_renew(). Also it can be discarded with - * mdbx_cursor_close(). + * An capable of operation cursor is associated with a specific transaction and + * database. A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with \ref mdbx_cursor_bind() and + * \ref mdbx_cursor_renew(). + * Also it can be discarded with \ref mdbx_cursor_close(). * * A cursor must be closed explicitly always, before or after its transaction - * ends. It can be reused with mdbx_cursor_renew() before finally closing it. + * ends. It can be reused with \ref mdbx_cursor_bind() + * or \ref mdbx_cursor_renew() before finally closing it. + * + * \note In contrast to LMDB, the MDBX required that any opened cursors can be + * reused and must be freed explicitly, regardless ones was opened in a + * read-only or write transaction. The REASON for this is eliminates ambiguity + * which helps to avoid errors such as: use-after-free, double-free, i.e. + * memory corruption and segfaults. + * + * \param [in] context A pointer to application context to be associated with + * created cursor and could be retrieved by + * \ref mdbx_cursor_get_userctx() until cursor closed. + * + * \returns Created cursor handle or NULL in case out of memory. */ +LIBMDBX_API MDBX_cursor *mdbx_cursor_create(void *context); + +/** \brief Set application information associated with the \ref MDBX_cursor. + * \ingroup c_crud + * \see mdbx_cursor_get_userctx() + * + * \param [in] cursor An cursor handle returned by \ref mdbx_cursor_create() + * or \ref mdbx_cursor_open(). + * \param [in] ctx An arbitrary pointer for whatever the application needs. + * + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_cursor_set_userctx(MDBX_cursor *cursor, void *ctx); + +/** \brief Get the application information associated with the MDBX_cursor. + * \ingroup c_crud + * \see mdbx_cursor_set_userctx() + * + * \param [in] cursor An cursor handle returned by \ref mdbx_cursor_create() + * or \ref mdbx_cursor_open(). + * \returns The pointer which was passed via the `context` parameter + * of `mdbx_cursor_create()` or set by \ref mdbx_cursor_set_userctx(), + * or `NULL` if something wrong. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * +mdbx_cursor_get_userctx(const MDBX_cursor *cursor); + +/** \brief Bind cursor to specified transaction and DBI handle. + * \ingroup c_cursors * - * NOTE: In contrast to LMDB, the MDBX required that any opened cursors can be + * Using of the `mdbx_cursor_bind()` is equivalent to calling + * \ref mdbx_cursor_renew() but with specifying an arbitrary dbi handle. + * + * An capable of operation cursor is associated with a specific transaction and + * database. The cursor may be associated with a new transaction, + * and referencing a new or the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. + * + * \note In contrast to LMDB, the MDBX required that any opened cursors can be * reused and must be freed explicitly, regardless ones was opened in a * read-only or write transaction. The REASON for this is eliminates ambiguity - * which helps to avoid errors such as: use-after-free, double-free, i.e. memory - * corruption and segfaults. + * which helps to avoid errors such as: use-after-free, double-free, i.e. + * memory corruption and segfaults. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [out] cursor A cursor handle returned by \ref mdbx_cursor_create(). * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [out] cursor Address where the new MDBX_cursor handle will be stored. + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL An invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_bind(MDBX_txn *txn, MDBX_cursor *cursor, + MDBX_dbi dbi); + +/** \brief Create a cursor handle for the specified transaction and DBI handle. + * \ingroup c_cursors + * + * Using of the `mdbx_cursor_open()` is equivalent to calling + * \ref mdbx_cursor_create() and then \ref mdbx_cursor_bind() functions. + * + * An capable of operation cursor is associated with a specific transaction and + * database. A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with \ref mdbx_cursor_bind() and + * \ref mdbx_cursor_renew(). + * Also it can be discarded with \ref mdbx_cursor_close(). * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ + * A cursor must be closed explicitly always, before or after its transaction + * ends. It can be reused with \ref mdbx_cursor_bind() + * or \ref mdbx_cursor_renew() before finally closing it. + * + * \note In contrast to LMDB, the MDBX required that any opened cursors can be + * reused and must be freed explicitly, regardless ones was opened in a + * read-only or write transaction. The REASON for this is eliminates ambiguity + * which helps to avoid errors such as: use-after-free, double-free, i.e. + * memory corruption and segfaults. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [out] cursor Address where the new \ref MDBX_cursor handle will be + * stored. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **cursor); -/* Close a cursor handle. +/** \brief Close a cursor handle. + * \ingroup c_cursors * * The cursor handle will be freed and must not be used again after this call, * but its transaction may still be live. * - * NOTE: In contrast to LMDB, the MDBX required that any opened cursors can be + * \note In contrast to LMDB, the MDBX required that any opened cursors can be * reused and must be freed explicitly, regardless ones was opened in a * read-only or write transaction. The REASON for this is eliminates ambiguity - * which helps to avoid errors such as: use-after-free, double-free, i.e. memory - * corruption and segfaults. + * which helps to avoid errors such as: use-after-free, double-free, i.e. + * memory corruption and segfaults. * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). */ + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open() + * or \ref mdbx_cursor_create(). */ LIBMDBX_API void mdbx_cursor_close(MDBX_cursor *cursor); -/* Renew a cursor handle. +/** \brief Renew a cursor handle. + * \ingroup c_cursors + * + * An capable of operation cursor is associated with a specific transaction and + * database. The cursor may be associated with a new transaction, + * and referencing a new or the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. * - * A cursor is associated with a specific transaction and database. The cursor - * may be associated with a new transaction, and referencing the same database - * handle as it was created with. This may be done whether the previous - * transaction is live or dead. + * Using of the `mdbx_cursor_renew()` is equivalent to calling + * \ref mdbx_cursor_bind() with the DBI handle that previously + * the cursor was used with. * - * NOTE: In contrast to LMDB, the MDBX allow any cursor to be re-used by using - * mdbx_cursor_renew(), to avoid unnecessary malloc/free overhead until it freed - * by mdbx_cursor_close(). + * \note In contrast to LMDB, the MDBX allow any cursor to be re-used by using + * \ref mdbx_cursor_renew(), to avoid unnecessary malloc/free overhead until it + * freed by \ref mdbx_cursor_close(). * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *cursor); -/* Return the cursor's transaction handle. +/** \brief Return the cursor's transaction handle. + * \ingroup c_cursors * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). */ -LIBMDBX_API MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *cursor); + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn * +mdbx_cursor_txn(const MDBX_cursor *cursor); -/* Return the cursor's database handle. +/** \brief Return the cursor's database handle. + * \ingroup c_cursors * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). */ + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). */ LIBMDBX_API MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *cursor); -/* Retrieve by cursor. +/** \brief Retrieve by cursor. + * \ingroup c_crud * * This function retrieves key/data pairs from the database. The address and * length of the key are returned in the object to which key refers (except - * for the case of the MDBX_SET option, in which the key object is unchanged), - * and the address and length of the data are returned in the object to which - * data refers. See mdbx_get() for restrictions on using the output values. - * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). - * [in,out] key The key for a retrieved item. - * [in,out] data The data of a retrieved item. - * [in] op A cursor operation MDBX_cursor_op. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_NOTFOUND = no matching key found. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * for the case of the \ref MDBX_SET option, in which the key object is + * unchanged), and the address and length of the data are returned in the object + * to which data refers. + * \see mdbx_get() + * + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). + * \param [in,out] key The key for a retrieved item. + * \param [in,out] data The data of a retrieved item. + * \param [in] op A cursor operation \ref MDBX_cursor_op. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_NOTFOUND No matching key found. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_cursor_get(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op); -/* Store by cursor. +/** \brief Store by cursor. + * \ingroup c_crud * * This function stores key/data pairs into the database. The cursor is * positioned at the new item, or on failure usually near it. * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). - * [in] key The key operated on. - * [in,out] data The data operated on. - * [in] flags Options for this operation. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of - * the values described here: - * - * - MDBX_CURRENT + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). + * \param [in] key The key operated on. + * \param [in,out] data The data operated on. + * \param [in] flags Options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together + * one or more of the values described here: + * - \ref MDBX_CURRENT * Replace the item at the current cursor position. The key parameter * must still be provided, and must match it, otherwise the function - * return MDBX_EKEYMISMATCH. + * return \ref MDBX_EKEYMISMATCH. With combination the + * \ref MDBX_ALLDUPS will replace all multi-values. * - * NOTE: MDBX allows (unlike LMDB) you to change the size of the data and - * automatically handles reordering for sorted duplicates (MDBX_DUPSORT). + * \note MDBX allows (unlike LMDB) you to change the size of the data and + * automatically handles reordering for sorted duplicates + * (see \ref MDBX_DUPSORT). * - * - MDBX_NODUPDATA - * Enter the new key/data pair only if it does not already appear in the + * - \ref MDBX_NODUPDATA + * Enter the new key-value pair only if it does not already appear in the * database. This flag may only be specified if the database was opened - * with MDBX_DUPSORT. The function will return MDBX_KEYEXIST if the - * key/data pair already appears in the database. + * with \ref MDBX_DUPSORT. The function will return \ref MDBX_KEYEXIST + * if the key/data pair already appears in the database. * - * - MDBX_NOOVERWRITE + * - \ref MDBX_NOOVERWRITE * Enter the new key/data pair only if the key does not already appear - * in the database. The function will return MDBX_KEYEXIST if the key + * in the database. The function will return \ref MDBX_KEYEXIST if the key * already appears in the database, even if the database supports - * duplicates (MDBX_DUPSORT). + * duplicates (\ref MDBX_DUPSORT). * - * - MDBX_RESERVE + * - \ref MDBX_RESERVE * Reserve space for data of the given size, but don't copy the given * data. Instead, return a pointer to the reserved space, which the * caller can fill in later - before the next update operation or the * transaction ends. This saves an extra memcpy if the data is being * generated later. This flag must not be specified if the database - * was opened with MDBX_DUPSORT. + * was opened with \ref MDBX_DUPSORT. * - * - MDBX_APPEND + * - \ref MDBX_APPEND * Append the given key/data pair to the end of the database. No key * comparisons are performed. This option allows fast bulk loading when * keys are already known to be in the correct order. Loading unsorted - * keys with this flag will cause a MDBX_KEYEXIST error. + * keys with this flag will cause a \ref MDBX_KEYEXIST error. * - * - MDBX_APPENDDUP + * - \ref MDBX_APPENDDUP * As above, but for sorted dup data. * - * - MDBX_MULTIPLE + * - \ref MDBX_MULTIPLE * Store multiple contiguous data elements in a single request. This flag - * may only be specified if the database was opened with MDBX_DUPFIXED. - * The data argument must be an array of two MDBX_vals. The iov_len of the - * first MDBX_val must be the size of a single data element. The iov_base - * of the first MDBX_val must point to the beginning of the array of - * contiguous data elements. The iov_len of the second MDBX_val must be - * the count of the number of data elements to store. On return this - * field will be set to the count of the number of elements actually - * written. The iov_base of the second MDBX_val is unused. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EKEYMISMATCH - * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). - * - MDBX_TXN_FULL = the transaction has too many dirty pages. - * - MDBX_EACCES = an attempt was made to write in a read-only transaction. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * may only be specified if the database was opened with + * \ref MDBX_DUPFIXED. With combination the \ref MDBX_ALLDUPS + * will replace all multi-values. + * The data argument must be an array of two \ref MDBX_val. The `iov_len` + * of the first \ref MDBX_val must be the size of a single data element. + * The `iov_base` of the first \ref MDBX_val must point to the beginning + * of the array of contiguous data elements which must be properly aligned + * in case of database with \ref MDBX_INTEGERDUP flag. + * The `iov_len` of the second \ref MDBX_val must be the count of the + * number of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The `iov_base` of + * the second \ref MDBX_val is unused. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EKEYMISMATCH The given key value is mismatched to the current + * cursor position + * \retval MDBX_MAP_FULL The database is full, + * see \ref mdbx_env_set_mapsize(). + * \retval MDBX_TXN_FULL The transaction has too many dirty pages. + * \retval MDBX_EACCES An attempt was made to write in a read-only + * transaction. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, const MDBX_val *key, - MDBX_val *data, unsigned flags); + MDBX_val *data, MDBX_put_flags_t flags); -/* Delete current key/data pair. +/** \brief Delete current key/data pair. + * \ingroup c_crud * - * This function deletes the key/data pair to which the cursor refers. This does - * not invalidate the cursor, so operations such as MDBX_NEXT can still be used - * on it. Both MDBX_NEXT and MDBX_GET_CURRENT will return the same record after - * this operation. + * This function deletes the key/data pair to which the cursor refers. This + * does not invalidate the cursor, so operations such as \ref MDBX_NEXT can + * still be used on it. Both \ref MDBX_NEXT and \ref MDBX_GET_CURRENT will + * return the same record after this operation. * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). - * [in] flags Options for this operation. This parameter must be set to 0 - * or one of the values described here. + * \param [in] cursor A cursor handle returned by mdbx_cursor_open(). + * \param [in] flags Options for this operation. This parameter must be set + * to one of the values described here. * - * - MDBX_NODUPDATA + * - \ref MDBX_CURRENT Delete only single entry at current cursor position. + * - \ref MDBX_ALLDUPS + * or \ref MDBX_NODUPDATA (supported for compatibility) * Delete all of the data items for the current key. This flag has effect - * only for database(s) was created with MDBX_DUPSORT. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EACCES = an attempt was made to write in a read-only transaction. - * - MDBX_EINVAL = an invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_del(MDBX_cursor *cursor, unsigned flags); - -/* Return count of duplicates for current key. + * only for database(s) was created with \ref MDBX_DUPSORT. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_MAP_FULL The database is full, + * see \ref mdbx_env_set_mapsize(). + * \retval MDBX_TXN_FULL The transaction has too many dirty pages. + * \retval MDBX_EACCES An attempt was made to write in a read-only + * transaction. + * \retval MDBX_EINVAL An invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_del(MDBX_cursor *cursor, MDBX_put_flags_t flags); + +/** \brief Return count of duplicates for current key. + * \ingroup c_crud * * This call is valid for all databases, but reasonable only for that support - * sorted duplicate data items MDBX_DUPSORT. + * sorted duplicate data items \ref MDBX_DUPSORT. * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). - * [out] countp Address where the count will be stored. + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). + * \param [out] pcount Address where the count will be stored. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EINVAL = cursor is not initialized, or an invalid parameter - * was specified. */ -LIBMDBX_API int mdbx_cursor_count(const MDBX_cursor *cursor, size_t *countp); + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL Cursor is not initialized, or an invalid parameter + * was specified. */ +LIBMDBX_API int mdbx_cursor_count(const MDBX_cursor *cursor, size_t *pcount); -/* Determines whether the cursor is pointed to a key-value pair or not, +/** \brief Determines whether the cursor is pointed to a key-value pair or not, * i.e. was not positioned or points to the end of data. + * \ingroup c_cursors * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). * - * Returns: - * - MDBX_RESULT_TRUE = no more data available or cursor not positioned; - * - MDBX_RESULT_FALSE = data available; - * - Otherwise the error code. */ -LIBMDBX_API int mdbx_cursor_eof(const MDBX_cursor *mc); + * \returns A \ref MDBX_RESULT_TRUE or \ref MDBX_RESULT_FALSE value, + * otherwise the error code: + * \retval MDBX_RESULT_TRUE No more data available or cursor not + * positioned + * \retval MDBX_RESULT_FALSE A data is available + * \retval Otherwise the error code */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int +mdbx_cursor_eof(const MDBX_cursor *cursor); -/* Determines whether the cursor is pointed to the first key-value pair or not. +/** \brief Determines whether the cursor is pointed to the first key-value pair + * or not. \ingroup c_cursors * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). * - * Returns: - * - MDBX_RESULT_TRUE = cursor positioned to the first key-value pair. - * - MDBX_RESULT_FALSE = cursor NOT positioned to the first key-value pair. - * - Otherwise the error code. */ -LIBMDBX_API int mdbx_cursor_on_first(const MDBX_cursor *mc); + * \returns A MDBX_RESULT_TRUE or MDBX_RESULT_FALSE value, + * otherwise the error code: + * \retval MDBX_RESULT_TRUE Cursor positioned to the first key-value pair + * \retval MDBX_RESULT_FALSE Cursor NOT positioned to the first key-value + * pair \retval Otherwise the error code */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int +mdbx_cursor_on_first(const MDBX_cursor *cursor); -/* Determines whether the cursor is pointed to the last key-value pair or not. +/** \brief Determines whether the cursor is pointed to the last key-value pair + * or not. \ingroup c_cursors * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). * - * Returns: - * - MDBX_RESULT_TRUE = cursor positioned to the last key-value pair. - * - MDBX_RESULT_FALSE = cursor NOT positioned to the last key-value pair. - * - Otherwise the error code. */ -LIBMDBX_API int mdbx_cursor_on_last(const MDBX_cursor *mc); + * \returns A \ref MDBX_RESULT_TRUE or \ref MDBX_RESULT_FALSE value, + * otherwise the error code: + * \retval MDBX_RESULT_TRUE Cursor positioned to the last key-value pair + * \retval MDBX_RESULT_FALSE Cursor NOT positioned to the last key-value pair + * \retval Otherwise the error code */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int +mdbx_cursor_on_last(const MDBX_cursor *cursor); -/* Estimates the distance between cursors as a number of elements. The results - * of such estimation can be used to build and/or optimize query execution - * plans. - * - * This function performs a rough estimate based only on b-tree pages that are - * common for the both cursor's stacks. - * - * NOTE: The result varies greatly depending on the filling of specific pages - * and the overall balance of the b-tree: +/** \addtogroup c_rqest + * \details \note The estimation result varies greatly depending on the filling + * of specific pages and the overall balance of the b-tree: * * 1. The number of items is estimated by analyzing the height and fullness of - * the b-tree. The accuracy of the result directly depends on the balance of the - * b-tree, which in turn is determined by the history of previous insert/delete - * operations and the nature of the data (i.e. variability of keys length and so - * on). Therefore, the accuracy of the estimation can vary greatly in a - * particular situation. + * the b-tree. The accuracy of the result directly depends on the balance of + * the b-tree, which in turn is determined by the history of previous + * insert/delete operations and the nature of the data (i.e. variability of + * keys length and so on). Therefore, the accuracy of the estimation can vary + * greatly in a particular situation. * * 2. To understand the potential spread of results, you should consider a * possible situations basing on the general criteria for splitting and merging @@ -3253,68 +3927,92 @@ LIBMDBX_API int mdbx_cursor_on_last(const MDBX_cursor *mc); * * 3. In practice, the probability of extreme cases of the above situation is * close to zero and in most cases the error does not exceed a few percent. On - * the other hand, it's just a chance you shouldn't overestimate. + * the other hand, it's just a chance you shouldn't overestimate. */ + +/** \brief Estimates the distance between cursors as a number of elements. + * \ingroup c_rqest + * + * This function performs a rough estimate based only on b-tree pages that are + * common for the both cursor's stacks. The results of such estimation can be + * used to build and/or optimize query execution plans. + * + * Please see notes on accuracy of the result in the details + * of \ref c_rqest section. * * Both cursors must be initialized for the same database and the same * transaction. * - * [in] first The first cursor for estimation. - * [in] last The second cursor for estimation. - * [out] distance_items A pointer to store estimated distance value, - * i.e. *distance_items = distance(first, last). + * \param [in] first The first cursor for estimation. + * \param [in] last The second cursor for estimation. + * \param [out] distance_items The pointer to store estimated distance value, + * i.e. `*distance_items = distance(first, last)`. * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, ptrdiff_t *distance_items); -/* Estimates the move distance, i.e. between the current cursor position and - * next position after the specified move-operation with given key and data. - * The results of such estimation can be used to build and/or optimize query - * execution plans. Current cursor position and state are preserved. +/** \brief Estimates the move distance. + * \ingroup c_rqest + * + * This function performs a rough estimate distance between the current + * cursor position and next position after the specified move-operation with + * given key and data. The results of such estimation can be used to build + * and/or optimize query execution plans. Current cursor position and state are + * preserved. * - * Please see notes on accuracy of the result in mdbx_estimate_distance() - * description above. + * Please see notes on accuracy of the result in the details + * of \ref c_rqest section. * - * [in] cursor Cursor for estimation. - * [in,out] key The key for a retrieved item. - * [in,out] data The data of a retrieved item. - * [in] op A cursor operation MDBX_cursor_op. - * [out] distance_items A pointer to store estimated move distance - * as the number of elements. + * \param [in] cursor Cursor for estimation. + * \param [in,out] key The key for a retrieved item. + * \param [in,out] data The data of a retrieved item. + * \param [in] move_op A cursor operation \ref MDBX_cursor_op. + * \param [out] distance_items A pointer to store estimated move distance + * as the number of elements. * - * Returns A non-zero error value on failure and 0 on success. */ + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op move_op, ptrdiff_t *distance_items); -/* Estimates the size of a range as a number of elements. The results - * of such estimation can be used to build and/or optimize query execution - * plans. +/** \brief Estimates the size of a range as a number of elements. + * \ingroup c_rqest * - * Please see notes on accuracy of the result in mdbx_estimate_distance() - * description above. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] begin_key The key of range beginning or NULL for explicit FIRST. - * [in] begin_data Optional additional data to seeking among sorted - * duplicates. Only for MDBX_DUPSORT, NULL otherwise. - * [in] end_key The key of range ending or NULL for explicit LAST. - * [in] end_data Optional additional data to seeking among sorted - * duplicates. Only for MDBX_DUPSORT, NULL otherwise. - * [out] distance_items A pointer to store range estimation result. - * - * Returns A non-zero error value on failure and 0 on success. */ -#define MDBX_EPSILON ((MDBX_val *)((ptrdiff_t)-1)) + * The results of such estimation can be used to build and/or optimize query + * execution plans. + * + * Please see notes on accuracy of the result in the details + * of \ref c_rqest section. + * + * + * \param [in] txn A transaction handle returned + * by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] begin_key The key of range beginning or NULL for explicit FIRST. + * \param [in] begin_data Optional additional data to seeking among sorted + * duplicates. + * Only for \ref MDBX_DUPSORT, NULL otherwise. + * \param [in] end_key The key of range ending or NULL for explicit LAST. + * \param [in] end_data Optional additional data to seeking among sorted + * duplicates. + * Only for \ref MDBX_DUPSORT, NULL otherwise. + * \param [out] distance_items A pointer to store range estimation result. + * + * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *begin_key, MDBX_val *begin_data, MDBX_val *end_key, MDBX_val *end_data, - ptrdiff_t *size_items); + ptrdiff_t *distance_items); + +/** \brief The EPSILON value for mdbx_estimate_range() + * \ingroup c_rqest */ +#define MDBX_EPSILON ((MDBX_val *)((ptrdiff_t)-1)) -/* Determines whether the given address is on a dirty database page of the - * transaction or not. Ultimately, this allows to avoid copy data from non-dirty - * pages. +/** \brief Determines whether the given address is on a dirty database page of + * the transaction or not. \ingroup c_statinfo + * + * Ultimately, this allows to avoid copy data from non-dirty pages. * * "Dirty" pages are those that have already been changed during a write * transaction. Accordingly, any further changes may result in such pages being @@ -3324,27 +4022,30 @@ LIBMDBX_API int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, * * In other words, data from dirty pages must either be copied before being * passed as arguments for further processing or rejected at the argument - * validation stage. Thus, mdbx_is_dirty() allows you to get rid of unnecessary - * copying, and perform a more complete check of the arguments. + * validation stage. Thus, `mdbx_is_dirty()` allows you to get rid of + * unnecessary copying, and perform a more complete check of the arguments. * - * NOTE: The address passed must point to the beginning of the data. This is the - * only way to ensure that the actual page header is physically located in the - * same memory page, including for multi-pages with long data. + * \note The address passed must point to the beginning of the data. This is + * the only way to ensure that the actual page header is physically located in + * the same memory page, including for multi-pages with long data. * - * NOTE: In rare cases the function may return a false positive answer - * (DBX_RESULT_TRUE when data is NOT on a dirty page), but never a false + * \note In rare cases the function may return a false positive answer + * (\ref MDBX_RESULT_TRUE when data is NOT on a dirty page), but never a false * negative if the arguments are correct. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] ptr The address of data to check. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] ptr The address of data to check. * - * Returns: - * - MDBX_RESULT_TRUE = given address is on the dirty page. - * - MDBX_RESULT_FALSE = given address is NOT on the dirty page. - * - Otherwise the error code. */ -LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr); + * \returns A MDBX_RESULT_TRUE or MDBX_RESULT_FALSE value, + * otherwise the error code: + * \retval MDBX_RESULT_TRUE Given address is on the dirty page. + * \retval MDBX_RESULT_FALSE Given address is NOT on the dirty page. + * \retval Otherwise the error code. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, + const void *ptr); -/* Sequence generation for a database. +/** \brief Sequence generation for a database. + * \ingroup c_crud * * The function allows to create a linear sequence of unique positive integers * for each database. The function can be called for a read transaction to @@ -3352,371 +4053,508 @@ LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr); * Sequence changes become visible outside the current write transaction after * it is committed, and discarded on abort. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [out] result The optional address where the value of sequence before the - * change will be stored. - * [in] increment Value to increase the sequence, - * must be 0 for read-only transactions. - * - * Returns A non-zero error value on failure and 0 on success, - * some possible errors are: - * - MDBX_RESULT_TRUE = Increasing the sequence has resulted in an overflow - * and therefore cannot be executed. */ + * \param [in] txn A transaction handle returned + * by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [out] result The optional address where the value of sequence + * before the change will be stored. + * \param [in] increment Value to increase the sequence, + * must be 0 for read-only transactions. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_RESULT_TRUE Increasing the sequence has resulted in an + * overflow and therefore cannot be executed. */ LIBMDBX_API int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, uint64_t increment); -/* Compare two data items according to a particular database. +/** \brief Compare two keys according to a particular database. + * \ingroup c_crud * * This returns a comparison as if the two data items were keys in the * specified database. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] a The first item to compare. - * [in] b The second item to compare. + * \warning There ss a Undefined behavior if one of arguments is invalid. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] a The first item to compare. + * \param [in] b The second item to compare. * - * Returns < 0 if a < b, 0 if a == b, > 0 if a > b */ -LIBMDBX_API int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, - const MDBX_val *b); -LIBMDBX_API MDBX_cmp_func *mdbx_get_keycmp(unsigned flags); + * \returns < 0 if a < b, 0 if a == b, > 0 if a > b */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cmp(const MDBX_txn *txn, + MDBX_dbi dbi, + const MDBX_val *a, + const MDBX_val *b); + +/** \brief Returns default internal key's comparator for given database flags. + * \ingroup c_extra */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func * +mdbx_get_keycmp(MDBX_db_flags_t flags); -/* Compare two data items according to a particular database. +/** \brief Compare two data items according to a particular database. + * \ingroup c_crud * * This returns a comparison as if the two items were data items of the - * specified database. The database must have the MDBX_DUPSORT flag. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] a The first item to compare. - * [in] b The second item to compare. - * - * Returns < 0 if a < b, 0 if a == b, > 0 if a > b */ -LIBMDBX_API int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, - const MDBX_val *b); -LIBMDBX_API MDBX_cmp_func *mdbx_get_datacmp(unsigned flags); - -/* A callback function used to enumerate the reader lock table. - * - * [in] ctx An arbitrary context pointer for the callback. - * [in] num The serial number during enumeration, starting from 1. - * [in] slot The reader lock table slot number. - * [in] txnid The ID of the transaction being read, - * i.e. the MVCC-snaphot number. - * [in] lag The lag from a recent MVCC-snapshot, i.e. the number of - * committed transaction since read transaction started. - * [in] pid The reader process ID. - * [in] thread The reader thread ID. - * [in] bytes_used The number of last used page in the MVCC-snapshot which - * being read, i.e. database file can't shrinked beyond this. - * [in] bytes_retired The total size of the database pages that were retired by - * committed write transactions after the reader's - * MVCC-snapshot, i.e. the space which would be freed after - * the Reader releases the MVCC-snapshot for reuse by - * completion read transaction. - * - * Returns < 0 on failure, >= 0 on success. */ + * specified database. + * + * \warning There ss a Undefined behavior if one of arguments is invalid. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] a The first item to compare. + * \param [in] b The second item to compare. + * + * \returns < 0 if a < b, 0 if a == b, > 0 if a > b */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_dcmp(const MDBX_txn *txn, + MDBX_dbi dbi, + const MDBX_val *a, + const MDBX_val *b); + +/** \brief Returns default internal data's comparator for given database flags + * \ingroup c_extra */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func * +mdbx_get_datacmp(MDBX_db_flags_t flags); + +/** \brief A callback function used to enumerate the reader lock table. + * \ingroup c_statinfo + * + * \param [in] ctx An arbitrary context pointer for the callback. + * \param [in] num The serial number during enumeration, + * starting from 1. + * \param [in] slot The reader lock table slot number. + * \param [in] txnid The ID of the transaction being read, + * i.e. the MVCC-snapshot number. + * \param [in] lag The lag from a recent MVCC-snapshot, + * i.e. the number of committed write transactions + * since the current read transaction started. + * \param [in] pid The reader process ID. + * \param [in] thread The reader thread ID. + * \param [in] bytes_used The number of last used page in the MVCC-snapshot + * which being read, + * i.e. database file can't shrinked beyond this. + * \param [in] bytes_retired The total size of the database pages that were + * retired by committed write transactions after + * the reader's MVCC-snapshot, + * i.e. the space which would be freed after + * the Reader releases the MVCC-snapshot + * for reuse by completion read transaction. + * + * \returns < 0 on failure, >= 0 on success. \see mdbx_reader_list() */ typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, uint64_t lag, size_t bytes_used, - size_t bytes_retained); + size_t bytes_retained) MDBX_CXX17_NOEXCEPT; -/* Enumarete the entries in the reader lock table. +/** \brief Enumerate the entries in the reader lock table. + * + * \ingroup c_statinfo * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] func A MDBX_reader_list_func function. - * [in] ctx An arbitrary context pointer for the enumeration function. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] func A \ref MDBX_reader_list_func function. + * \param [in] ctx An arbitrary context pointer for the enumeration + * function. * - * Returns A non-zero error value on failure and 0 on success, - * or MDBX_RESULT_TRUE (-1) if the reader lock table is empty. */ + * \returns A non-zero error value on failure and 0 on success, + * or \ref MDBX_RESULT_TRUE if the reader lock table is empty. */ LIBMDBX_API int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, void *ctx); -/* Check for stale entries in the reader lock table. +/** \brief Check for stale entries in the reader lock table. + * \ingroup c_extra * - * [in] env An environment handle returned by mdbx_env_create(). - * [out] dead Number of stale slots that were cleared. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [out] dead Number of stale slots that were cleared. * - * Returns A non-zero error value on failure and 0 on success, - * or MDBX_RESULT_TRUE (-1) if a dead reader(s) found or mutex was recovered. */ + * \returns A non-zero error value on failure and 0 on success, + * or \ref MDBX_RESULT_TRUE if a dead reader(s) found or mutex was recovered. */ LIBMDBX_API int mdbx_reader_check(MDBX_env *env, int *dead); -/* Returns a lag of the reading for the given transaction. +/** \brief Returns a lag of the reading for the given transaction. + * \ingroup c_statinfo * * Returns an information for estimate how much given read-only * transaction is lagging relative the to actual head. - * This is deprecated function, use mdbx_txn_info() instead. + * \deprecated Please use \ref mdbx_txn_info() instead. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [out] percent Percentage of page allocation in the database. + * + * \returns Number of transactions committed after the given was started for + * read, or negative value on failure. */ +MDBX_DEPRECATED LIBMDBX_API int mdbx_txn_straggler(const MDBX_txn *txn, + int *percent); + +/** \brief Registers the current thread as a reader for the environment. + * \ingroup c_extra + * + * To perform read operations without blocking, a reader slot must be assigned + * for each thread. However, this assignment requires a short-term lock + * acquisition which is performed automatically. This function allows you to + * assign the reader slot in advance and thus avoid capturing the blocker when + * the read transaction starts firstly from current thread. + * \see mdbx_thread_unregister() + * + * \note Threads are registered automatically the first time a read transaction + * starts. Therefore, there is no need to use this function, except in + * special cases. + * + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * + * \returns A non-zero error value on failure and 0 on success, + * or \ref MDBX_RESULT_TRUE if thread is already registered. */ +LIBMDBX_API int mdbx_thread_register(const MDBX_env *env); + +/** \brief Unregisters the current thread as a reader for the environment. + * \ingroup c_extra + * + * To perform read operations without blocking, a reader slot must be assigned + * for each thread. However, the assigned reader slot will remain occupied until + * the thread ends or the environment closes. This function allows you to + * explicitly release the assigned reader slot. + * \see mdbx_thread_register() * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [out] percent Percentage of page allocation in the database. + * \param [in] env An environment handle returned by \ref mdbx_env_create(). * - * Returns Number of transactions committed after the given was started for - * read, or negative value on failure. */ -__deprecated LIBMDBX_API int mdbx_txn_straggler(const MDBX_txn *txn, - int *percent); + * \returns A non-zero error value on failure and 0 on success, or + * \ref MDBX_RESULT_TRUE if thread is not registered or already unregistered. */ +LIBMDBX_API int mdbx_thread_unregister(const MDBX_env *env); -/* A lack-of-space callback function to resolve issues with a laggard readers. +/** \brief A Handle-Slow-Readers callback function to resolve database + * full/overflow issue due to a reader(s) which prevents the old data from being + * recycled. + * \ingroup c_err * * Read transactions prevent reuse of pages freed by newer write transactions, * thus the database can grow quickly. This callback will be called when there - * is not enough space in the database (ie. before increasing the database size - * or before MDBX_MAP_FULL error) and thus can be used to resolve issues with - * a "long-lived" read transactions. - * - * Depending on the arguments and needs, your implementation may wait, terminate - * a process or thread that is performing a long read, or perform some other - * action. In doing so it is important that the returned code always corresponds - * to the performed action. - * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] pid A pid of the reader process. - * [in] tid A thread_id of the reader thread. - * [in] txn A transaction number on which stalled. - * [in] gap A lag from the last commited txn. - * [in] space A space that actually become available for reuse after this - * reader finished. The callback function can take this value into - * account to evaluate the impact that a long-running transaction - * has. - * [in] retry A retry number starting from 0. if callback has returned 0 - * at least once, then at end of current OOM-handler loop callback - * will be called additionally with negative value to notify about - * the end of loop. The callback function can use this value to - * implement timeout logic while waiting for readers. - * - * The RETURN CODE determines the further actions libmdbx and must match the - * action which was executed by the callback: - * - * -2 or less = An error condition and the reader was not killed. - * - * -1 = The callback was unable to solve the problem and agreed - * on MDBX_MAP_FULL error, libmdbx should increase the - * database size or return MDBX_MAP_FULL error. - * - * 0 (zero) = The callback solved the problem or just waited for + * is not enough space in the database (i.e. before increasing the database size + * or before \ref MDBX_MAP_FULL error) and thus can be used to resolve issues + * with a "long-lived" read transactions. + * \see long-lived-read + * + * Using this callback you can choose how to resolve the situation: + * - abort the write transaction with an error; + * - wait for the read transaction(s) to complete; + * - notify a thread performing a long-lived read transaction + * and wait for an effect; + * - kill the thread or whole process that performs the long-lived read + * transaction; + * + * Depending on the arguments and needs, your implementation may wait, + * terminate a process or thread that is performing a long read, or perform + * some other action. In doing so it is important that the returned code always + * corresponds to the performed action. + * + * \param [in] env An environment handle returned by \ref mdbx_env_create(). + * \param [in] txn The current write transaction which internally at + * the \ref MDBX_MAP_FULL condition. + * \param [in] pid A pid of the reader process. + * \param [in] tid A thread_id of the reader thread. + * \param [in] laggard An oldest read transaction number on which stalled. + * \param [in] gap A lag from the last commited txn. + * \param [in] space A space that actually become available for reuse after + * this reader finished. The callback function can take + * this value into account to evaluate the impact that + * a long-running transaction has. + * \param [in] retry A retry number starting from 0. + * If callback has returned 0 at least once, then at end + * of current handling loop the callback function will be + * called additionally with negative value to notify about + * the end of loop. The callback function can use this value + * to implement timeout logic while waiting for readers. + * + * \returns The RETURN CODE determines the further actions libmdbx and must + * match the action which was executed by the callback: + * + * \retval -2 or less An error condition and the reader was not killed. + * + * \retval -1 The callback was unable to solve the problem and + * agreed on \ref MDBX_MAP_FULL error; + * libmdbx should increase the database size or + * return \ref MDBX_MAP_FULL error. + * + * \retval 0 (zero) The callback solved the problem or just waited for * a while, libmdbx should rescan the reader lock table and * retry. This also includes a situation when corresponding - * transaction terminated in normal way by mdbx_txn_abort() - * or mdbx_txn_reset(), and my be restarted. I.e. reader - * slot don't needed to be cleaned from transaction. + * transaction terminated in normal way by + * \ref mdbx_txn_abort() or \ref mdbx_txn_reset(), + * and my be restarted. I.e. reader slot don't needed + * to be cleaned from transaction. * - * 1 = Transaction aborted asynchronous and reader slot should - * be cleared immediately, i.e. read transaction will not - * continue but mdbx_txn_abort() or mdbx_txn_reset() will - * be called later. + * \retval 1 Transaction aborted asynchronous and reader slot + * should be cleared immediately, i.e. read transaction + * will not continue but \ref mdbx_txn_abort() + * or \ref mdbx_txn_reset() will be called later. * - * 2 or great = The reader process was terminated or killed, and libmdbx - * should entirely reset reader registration. */ -typedef int(MDBX_oom_func)(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, - uint64_t txn, unsigned gap, size_t space, int retry); + * \retval 2 or great The reader process was terminated or killed, + * and libmdbx should entirely reset reader registration. + * + * \see mdbx_env_set_hsr() \see mdbx_env_get_hsr() + */ +typedef int(MDBX_hsr_func)(const MDBX_env *env, const MDBX_txn *txn, + mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, + unsigned gap, size_t space, + int retry) MDBX_CXX17_NOEXCEPT; -/* Set the OOM callback. +/** \brief Sets a Handle-Slow-Readers callback to resolve database full/overflow + * issue due to a reader(s) which prevents the old data from being recycled. + * \ingroup c_err + * + * The callback will only be triggered when the database is full due to a + * reader(s) prevents the old data from being recycled. * - * The callback will only be triggered on lack of space to resolve issues with - * lagging reader(s) (i.e. to kill it) for resume reuse pages from the garbage - * collector. + * \see mdbx_env_get_hsr() + * \see long-lived-read * - * [in] env An environment handle returned by mdbx_env_create(). - * [in] oom_func A MDBX_oom_func function or NULL to disable. + * \param [in] env An environment handle returned + * by \ref mdbx_env_create(). + * \param [in] hsr_callback A \ref MDBX_hsr_func function + * or NULL to disable. * - * Returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_set_oomfunc(MDBX_env *env, MDBX_oom_func *oom_func); + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr_callback); -/* Get the current oom_func callback. +/** \brief Gets current Handle-Slow-Readers callback used to resolve database + * full/overflow issue due to a reader(s) which prevents the old data from being + * recycled. + * \see mdbx_env_set_hsr() * - * [in] env An environment handle returned by mdbx_env_create(). + * \param [in] env An environment handle returned by \ref mdbx_env_create(). * - * Returns A MDBX_oom_func function or NULL if disabled. */ -LIBMDBX_API MDBX_oom_func *mdbx_env_get_oomfunc(const MDBX_env *env); + * \returns A MDBX_hsr_func function or NULL if disabled + * or something wrong. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_hsr_func * +mdbx_env_get_hsr(const MDBX_env *env); -/**** B-tree Traversal ********************************************************* +/** \defgroup btree_traversal B-tree Traversal * This is internal API for mdbx_chk tool. You should avoid to use it, except - * some extremal special cases. */ - -/* Page types for traverse the b-tree. */ -typedef enum { - MDBX_page_void, + * some extremal special cases. + * \ingroup c_extra + * @{ */ + +/** \brief Page types for traverse the b-tree. + * \see mdbx_env_pgwalk() \see MDBX_pgvisitor_func */ +enum MDBX_page_type_t { + MDBX_page_broken, MDBX_page_meta, MDBX_page_large, MDBX_page_branch, MDBX_page_leaf, MDBX_page_dupfixed_leaf, MDBX_subpage_leaf, - MDBX_subpage_dupfixed_leaf -} MDBX_page_type_t; + MDBX_subpage_dupfixed_leaf, + MDBX_subpage_broken, +}; +#ifndef __cplusplus +typedef enum MDBX_page_type_t MDBX_page_type_t; +#endif +/** \brief Pseudo-name for MainDB */ #define MDBX_PGWALK_MAIN ((const char *)((ptrdiff_t)0)) +/** \brief Pseudo-name for GarbageCollectorDB */ #define MDBX_PGWALK_GC ((const char *)((ptrdiff_t)-1)) +/** \brief Pseudo-name for MetaPages */ #define MDBX_PGWALK_META ((const char *)((ptrdiff_t)-2)) -/* Callback function for traverse the b-tree. */ -typedef int -MDBX_pgvisitor_func(const uint64_t pgno, const unsigned number, void *const ctx, - const int deep, const char *const dbi, - const size_t page_size, const MDBX_page_type_t type, - const size_t nentries, const size_t payload_bytes, - const size_t header_bytes, const size_t unused_bytes); +/** \brief Callback function for traverse the b-tree. \see mdbx_env_pgwalk() */ +typedef int MDBX_pgvisitor_func( + const uint64_t pgno, const unsigned number, void *const ctx, const int deep, + const char *const dbi, const size_t page_size, const MDBX_page_type_t type, + const MDBX_error_t err, const size_t nentries, const size_t payload_bytes, + const size_t header_bytes, const size_t unused_bytes) MDBX_CXX17_NOEXCEPT; -/* B-tree traversal function. */ +/** \brief B-tree traversal function. */ LIBMDBX_API int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, - void *ctx, int dont_check_keys_ordering); + void *ctx, bool dont_check_keys_ordering); -/**** Attribute support functions for Nexenta *********************************/ -#ifdef MDBX_NEXENTA_ATTRS +/** \brief Open an environment instance using specific meta-page + * for checking and recovery. + * + * This function mostly of internal API for `mdbx_chk` utility and subject to + * change at any time. Do not use this function to avoid shooting your own + * leg(s). */ +LIBMDBX_API int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, + unsigned target_meta, + bool writeable); + +/** \brief Turn database to the specified meta-page. + * + * This function mostly of internal API for `mdbx_chk` utility and subject to + * change at any time. Do not use this function to avoid shooting your own + * leg(s). */ +LIBMDBX_API int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target_meta); + +/** @} B-tree Traversal */ + +/**** Attribute support functions for Nexenta + * *******************************************/ +#if defined(MDBX_NEXENTA_ATTRS) || defined(DOXYGEN) +/** \defgroup nexenta Attribute support functions for Nexenta + * \ingroup c_crud + * @{ */ typedef uint_fast64_t mdbx_attr_t; -/* Store by cursor with attribute. +/** Store by cursor with attribute. * * This function stores key/data pairs into the database. The cursor is * positioned at the new item, or on failure usually near it. * - * NOTE: Internally based on MDBX_RESERVE feature, - * therefore doesn't support MDBX_DUPSORT. - * - * [in] cursor A cursor handle returned by mdbx_cursor_open() - * [in] key The key operated on. - * [in] data The data operated on. - * [in] attr The attribute. - * [in] flags Options for this operation. This parameter must be set to 0 - * or one of the values described here: + * \note Internally based on \ref MDBX_RESERVE feature, + * therefore doesn't support \ref MDBX_DUPSORT. * - * - MDBX_CURRENT + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open() + * \param [in] key The key operated on. + * \param [in] data The data operated on. + * \param [in] attr The attribute. + * \param [in] flags Options for this operation. This parameter must be set + * to 0 or one of the values described here: + * - \ref MDBX_CURRENT * Replace the item at the current cursor position. The key parameter * must still be provided, and must match it, otherwise the function - * return MDBX_EKEYMISMATCH. + * return \ref MDBX_EKEYMISMATCH. * - * - MDBX_APPEND + * - \ref MDBX_APPEND * Append the given key/data pair to the end of the database. No key * comparisons are performed. This option allows fast bulk loading when * keys are already known to be in the correct order. Loading unsorted - * keys with this flag will cause a MDBX_KEYEXIST error. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_EKEYMISMATCH - * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). - * - MDBX_TXN_FULL = the transaction has too many dirty pages. - * - MDBX_EACCES = an attempt was made to write in a read-only transaction. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * keys with this flag will cause a \ref MDBX_KEYEXIST error. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_EKEYMISMATCH + * \retval MDBX_MAP_FULL The database is full, see \ref mdbx_env_set_mapsize(). + * \retval MDBX_TXN_FULL The transaction has too many dirty pages. + * \retval MDBX_EACCES An attempt was made to write in a read-only + * transaction. + * \retval MDBX_EINVAL an invalid parameter was specified. */ LIBMDBX_API int mdbx_cursor_put_attr(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, mdbx_attr_t attr, - unsigned flags); + MDBX_put_flags_t flags); -/* Store items and attributes into a database. +/** Store items and attributes into a database. * * This function stores key/data pairs in the database. The default behavior * is to enter the new key/data pair, replacing any previously existing key * if duplicates are disallowed. * - * NOTE: Internally based on MDBX_RESERVE feature, - * therefore doesn't support MDBX_DUPSORT. - * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] key The key to store in the database. - * [in] attr The attribute to store in the database. - * [in,out] data The data to store. - * [in] flags Special options for this operation. This parameter must be - * set to 0 or by bitwise OR'ing together one or more of the - * values described here: - * - * - MDBX_NOOVERWRITE + * \note Internally based on \ref MDBX_RESERVE feature, + * therefore doesn't support \ref MDBX_DUPSORT. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to store in the database. + * \param [in] attr The attribute to store in the database. + * \param [in,out] data The data to store. + * \param [in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or + * more of the values described here: + * - \ref MDBX_NOOVERWRITE * Enter the new key/data pair only if the key does not already appear - * in the database. The function will return MDBX_KEYEXIST if the key + * in the database. The function will return \ref MDBX_KEYEXIST if the key * already appears in the database. The data parameter will be set to * point to the existing item. * - * - MDBX_CURRENT + * - \ref MDBX_CURRENT * Update an single existing entry, but not add new ones. The function - * will return MDBX_NOTFOUND if the given key not exist in the database. - * Or the MDBX_EMULTIVAL in case duplicates for the given key. + * will return \ref MDBX_NOTFOUND if the given key not exist in the + * database. Or the \ref MDBX_EMULTIVAL in case duplicates for the given + * key. * - * - MDBX_APPEND + * - \ref MDBX_APPEND * Append the given key/data pair to the end of the database. This option * allows fast bulk loading when keys are already known to be in the * correct order. Loading unsorted keys with this flag will cause - * a MDBX_EKEYMISMATCH error. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_KEYEXIST - * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). - * - MDBX_TXN_FULL = the transaction has too many dirty pages. - * - MDBX_EACCES = an attempt was made to write in a read-only transaction. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * a \ref MDBX_EKEYMISMATCH error. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_KEYEXIST + * \retval MDBX_MAP_FULL The database is full, see \ref mdbx_env_set_mapsize(). + * \retval MDBX_TXN_FULL The transaction has too many dirty pages. + * \retval MDBX_EACCES An attempt was made to write + * in a read-only transaction. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_put_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data, mdbx_attr_t attr, unsigned flags); + MDBX_val *data, mdbx_attr_t attr, + MDBX_put_flags_t flags); -/* Set items attribute from a database. +/** Set items attribute from a database. * * This function stores key/data pairs attribute to the database. * - * NOTE: Internally based on MDBX_RESERVE feature, - * therefore doesn't support MDBX_DUPSORT. + * \note Internally based on \ref MDBX_RESERVE feature, + * therefore doesn't support \ref MDBX_DUPSORT. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] key The key to search for in the database. - * [in] data The data to be stored or NULL to save previous value. - * [in] attr The attribute to be stored. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to search for in the database. + * \param [in] data The data to be stored or NULL to save previous value. + * \param [in] attr The attribute to be stored. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_NOTFOUND = the key-value pair was not in the database. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_NOTFOUND The key-value pair was not in the database. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_set_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, mdbx_attr_t attr); -/* Get items attribute from a database cursor. +/** Get items attribute from a database cursor. * * This function retrieves key/data pairs from the database. The address and * length of the key are returned in the object to which key refers (except - * for the case of the MDBX_SET option, in which the key object is unchanged), - * and the address and length of the data are returned in the object to which - * data refers. See mdbx_get() for restrictions on using the output values. - * - * [in] cursor A cursor handle returned by mdbx_cursor_open(). - * [in,out] key The key for a retrieved item. - * [in,out] data The data of a retrieved item. - * [in] op A cursor operation MDBX_cursor_op. - * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_NOTFOUND = no matching key found. - * - MDBX_EINVAL = an invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_get_attr(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data, mdbx_attr_t *attrptr, + * for the case of the \ref MDBX_SET option, in which the key object is + * unchanged), and the address and length of the data are returned in the object + * to which data refers. + * \see mdbx_get() + * + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). + * \param [in,out] key The key for a retrieved item. + * \param [in,out] data The data of a retrieved item. + * \param [out] pattr The pointer to retrieve attribute. + * \param [in] op A cursor operation MDBX_cursor_op. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_NOTFOUND No matching key found. + * \retval MDBX_EINVAL An invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_get_attr(MDBX_cursor *cursor, MDBX_val *key, + MDBX_val *data, mdbx_attr_t *pattr, MDBX_cursor_op op); -/* Get items attribute from a database. +/** Get items attribute from a database. * * This function retrieves key/data pairs from the database. The address * and length of the data associated with the specified key are returned * in the structure to which data refers. - * If the database supports duplicate keys (MDBX_DUPSORT) then the + * If the database supports duplicate keys (see \ref MDBX_DUPSORT) then the * first data item for the key will be returned. Retrieval of other - * items requires the use of mdbx_cursor_get(). + * items requires the use of \ref mdbx_cursor_get(). * - * NOTE: The memory pointed to by the returned values is owned by the + * \note The memory pointed to by the returned values is owned by the * database. The caller need not dispose of the memory, and may not * modify it in any way. For values returned in a read-only transaction - * any modification attempts will cause a SIGSEGV. + * any modification attempts will cause a `SIGSEGV`. * - * NOTE: Values returned from the database are valid only until a + * \note Values returned from the database are valid only until a * subsequent update operation, or the end of the transaction. * - * [in] txn A transaction handle returned by mdbx_txn_begin(). - * [in] dbi A database handle returned by mdbx_dbi_open(). - * [in] key The key to search for in the database. - * [in,out] data The data corresponding to the key. + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to search for in the database. + * \param [in,out] data The data corresponding to the key. + * \param [out] pattr The pointer to retrieve attribute. * - * Returns A non-zero error value on failure and 0 on success, some - * possible errors are: - * - MDBX_NOTFOUND = the key was not in the database. - * - MDBX_EINVAL = an invalid parameter was specified. */ + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_NOTFOUND The key was not in the database. + * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data, mdbx_attr_t *attrptr); + MDBX_val *data, mdbx_attr_t *pattr); +/** @} end of Attribute support functions for Nexenta */ #endif /* MDBX_NEXENTA_ATTRS */ +/** @} end of C API */ + /******************************************************************************* * Workaround for mmaped-lookahead-cross-page-boundary bug * in an obsolete versions of Elbrus's libc and kernels. */ @@ -3730,6 +4568,28 @@ LIBMDBX_API int mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2, LIBMDBX_API size_t mdbx_e2k_strlen_bug_workaround(const char *s); LIBMDBX_API size_t mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen); +#ifdef __cplusplus +namespace std { +inline int mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2, + size_t n) { + return ::mdbx_e2k_memcmp_bug_workaround(s1, s2, n); +} +inline int mdbx_e2k_strcmp_bug_workaround(const char *s1, const char *s2) { + return ::mdbx_e2k_strcmp_bug_workaround(s1, s2); +} +inline int mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2, + size_t n) { + return ::mdbx_e2k_strncmp_bug_workaround(s1, s2, n); +} +inline size_t mdbx_e2k_strlen_bug_workaround(const char *s) { + return ::mdbx_e2k_strlen_bug_workaround(s); +} +inline size_t mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen) { + return ::mdbx_e2k_strnlen_bug_workaround(s, maxlen); +} +} // namespace std +#endif /* __cplusplus */ + #include <string.h> #include <strings.h> #undef memcmp @@ -3747,7 +4607,7 @@ LIBMDBX_API size_t mdbx_e2k_strnlen_bug_workaround(const char *s, #endif /* MDBX_E2K_MLHCPB_WORKAROUND */ #ifdef __cplusplus -} +} /* extern "C" */ #endif #endif /* LIBMDBX_H */ diff --git a/libs/libmdbx/src/mdbx.h++ b/libs/libmdbx/src/mdbx.h++ new file mode 100644 index 0000000000..2050f4b925 --- /dev/null +++ b/libs/libmdbx/src/mdbx.h++ @@ -0,0 +1,4649 @@ +/// \file mdbx.h++ +/// \brief The libmdbx C++ API header file (preliminary). +/// +/// \author Copyright (c) 2020, Leonid Yuriev <leo@yuriev.ru>. +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// +/// Tested with: +/// - LCC >= 1.23 (http://www.mcst.ru/lcc), +/// - GNU C++ >= 4.8, +/// - clang >= 4.0, +/// - MSVC >= 19.0 (Visual Studio 2015), +/// but 19.2x could hang due optimizer bug. +/// + +#pragma once + +#if (!defined(__cplusplus) || __cplusplus < 201103L) && \ + !(defined( \ + _MSC_VER) /* MSVC is mad and don't define __cplusplus properly */ \ + && _MSC_VER == 1900) +#error "C++11 or better is required" +#endif + +#if (defined(_WIN32) || defined(_WIN64)) && MDBX_AVOID_CRT +#error "CRT is required for C++ API, the MDBX_AVOID_CRT option must be disabled" +#endif /* Windows */ + +#ifndef __has_include +#define __has_include(header) (0) +#endif /* __has_include */ + +#if __has_include(<version>) +#include <version> +#endif /* <version> */ + +/* Disable min/max macros from C' headers */ +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#include <algorithm> // for std::min/max +#include <cassert> // for assert() +#include <cstring> // for std::strlen, str:memcmp +#include <exception> // for std::exception_ptr +#include <ostream> // for std::ostream +#include <sstream> // for std::ostringstream +#include <stdexcept> // for std::invalid_argument +#include <string> // for std::string +#include <type_traits> // for std::is_pod<>, etc. +#include <vector> // for std::vector<> as template args + +// Unused for now +// #if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L +// #include <bit> +// #endif + +#if defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L +#include <memory_resource> +#endif + +#if defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L +#include <string_view> +#endif + +#if defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L +#include <filesystem> +#endif + +#include "mdbx.h" + +#if defined(DOXYGEN) || \ + defined(__cpp_constexpr) && __cpp_constexpr >= 201603L && \ + ((defined(_MSC_VER) && _MSC_VER >= 1915) || \ + (defined(__clang__) && __clang_major__ > 5) || \ + (defined(__GNUC__) && __GNUC__ > 7) || \ + (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) +#define MDBX_CXX17_CONSTEXPR constexpr +#else +#define MDBX_CXX17_CONSTEXPR inline +#endif /* MDBX_CXX17_CONSTEXPR */ + +#if defined(DOXYGEN) || defined(__cpp_lib_is_constant_evaluated) && \ + __cpp_lib_is_constant_evaluated >= 201811L && \ + defined(__cpp_lib_constexpr_string) && \ + __cpp_lib_constexpr_string >= 201907L +#define MDBX_CXX20_CONSTEXPR constexpr +#else +#define MDBX_CXX20_CONSTEXPR inline +#endif /* MDBX_CXX20_CONSTEXPR */ + +#if defined(CONSTEXPR_ASSERT) +#define MDBX_CONSTEXPR_ASSERT(expr) CONSTEXPR_ASSERT(expr) +#elif defined NDEBUG +#define MDBX_CONSTEXPR_ASSERT(expr) void(0) +#else +#define MDBX_CONSTEXPR_ASSERT(expr) \ + ((expr) ? void(0) : [] { assert(!#expr); }()) +#endif /* MDBX_CONSTEXPR_ASSERT */ + +#ifndef MDBX_LIKELY +#if defined(DOXYGEN) || \ + (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ + !defined(__COVERITY__) +#define MDBX_LIKELY(cond) __builtin_expect(!!(cond), 1) +#else +#define MDBX_LIKELY(x) (x) +#endif +#endif /* MDBX_LIKELY */ + +#ifndef MDBX_UNLIKELY +#if defined(DOXYGEN) || \ + (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ + !defined(__COVERITY__) +#define MDBX_UNLIKELY(cond) __builtin_expect(!!(cond), 0) +#else +#define MDBX_UNLIKELY(x) (x) +#endif +#endif /* MDBX_UNLIKELY */ + +#if defined(DOXYGEN) || \ + (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || \ + __cplusplus >= 201703L +#define MDBX_CXX17_FALLTHROUGH [[fallthrough]] +#else +#define MDBX_CXX17_FALLTHROUGH +#endif /* MDBX_CXX17_FALLTHROUGH */ + +#if defined(DOXYGEN) || __has_cpp_attribute(likely) +#define MDBX_CXX20_LIKELY [[likely]] +#else +#define MDBX_CXX20_LIKELY +#endif /* MDBX_CXX20_LIKELY */ + +#ifndef MDBX_CXX20_UNLIKELY +#if defined(DOXYGEN) || __has_cpp_attribute(unlikely) +#define MDBX_CXX20_UNLIKELY [[unlikely]] +#else +#define MDBX_CXX20_UNLIKELY +#endif +#endif /* MDBX_CXX20_UNLIKELY */ + +#ifdef _MSC_VER +#pragma warning(push, 4) +#pragma warning(disable : 4251) /* 'std::FOO' needs to have dll-interface to \ + be used by clients of 'mdbx::BAR' */ +#pragma warning(disable : 4275) /* non dll-interface 'std::FOO' used as \ + base for dll-interface 'mdbx::BAR' */ +/* MSVC is mad and can generate this warning for its own intermediate + * automatically generated code, which becomes unreachable after some kinds of + * optimization (copy elision, etc). */ +#pragma warning(disable : 4702) /* unreachable code */ +#endif /* _MSC_VER (warnings) */ + +//------------------------------------------------------------------------------ +/// \defgroup cxx_api C++ API +/// @{ + +namespace mdbx { + +// Functions whose signature depends on the `mdbx::byte` type +// must be strictly defined as inline! +#if defined(DOXYGEN) || (defined(__cpp_char8_t) && __cpp_char8_t >= 201811) +// Wanna using a non-aliasing type to release more power of an optimizer. +using byte = char8_t; +#else +// Wanna not using std::byte since it doesn't add features, +// but add inconvenient restrictions. +using byte = unsigned char; +#endif /* __cpp_char8_t >= 201811*/ + +/// \copydoc MDBX_version_info +using version_info = ::MDBX_version_info; +/// \brief Returns libmdbx version information. +MDBX_CXX11_CONSTEXPR const version_info &get_version() noexcept; +/// \copydoc MDBX_build_info +using build_info = ::MDBX_build_info; +/// \brief Returns libmdbx build information. +MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept; + +/// \brief constexpr-compatible strlen(). +static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept; + +struct slice; +class env; +class env_managed; +class txn; +class txn_managed; +class cursor; +class cursor_managed; + +/// \brief Legacy default allocator +/// but it is recommended to use \ref polymorphic_allocator. +using legacy_allocator = ::std::string::allocator_type; + +#if defined(DOXYGEN) || \ + defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L +/// \brief Default polymorphic allocator for modern code. +using polymorphic_allocator = ::std::pmr::string::allocator_type; +#endif /* __cpp_lib_memory_resource >= 201603L */ + +/// \brief Default singe-byte string. +template <class ALLOCATOR = legacy_allocator> +using string = ::std::basic_string<char, ::std::char_traits<char>, ALLOCATOR>; + +using filehandle = ::mdbx_filehandle_t; +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && \ + (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || \ + __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500)) +using path = std::filesystem::path; +#elif defined(_WIN32) || defined(_WIN64) +using path = ::std::wstring; +#else +using path = ::std::string; +#endif + +/// \brief Transfers C++ exceptions thru C callbacks. +/// \details Implements saving exceptions before returning +/// from an C++'s environment to the intermediate C code and re-throwing after +/// returning from C to the C++'s environment. +class LIBMDBX_API_TYPE exception_thunk { + ::std::exception_ptr captured_; + +public: + exception_thunk() noexcept = default; + exception_thunk(const exception_thunk &) = delete; + exception_thunk(exception_thunk &&) = delete; + exception_thunk &operator=(const exception_thunk &) = delete; + exception_thunk &operator=(exception_thunk &&) = delete; + inline bool is_clean() const noexcept; + inline void capture() noexcept; + inline void rethrow_captured() const; +}; + +/// \brief Implements error information and throwing corresponding exceptions. +class LIBMDBX_API_TYPE error { + MDBX_error_t code_; + inline error &operator=(MDBX_error_t error_code) noexcept; + +public: + MDBX_CXX11_CONSTEXPR error(MDBX_error_t error_code) noexcept; + error(const error &) = default; + error(error &&) = default; + error &operator=(const error &) = default; + error &operator=(error &&) = default; + + MDBX_CXX11_CONSTEXPR friend bool operator==(const error &a, + const error &b) noexcept; + MDBX_CXX11_CONSTEXPR friend bool operator!=(const error &a, + const error &b) noexcept; + + MDBX_CXX11_CONSTEXPR bool is_success() const noexcept; + MDBX_CXX11_CONSTEXPR bool is_result_true() const noexcept; + MDBX_CXX11_CONSTEXPR bool is_result_false() const noexcept; + MDBX_CXX11_CONSTEXPR bool is_failure() const noexcept; + + /// \brief Returns error code. + MDBX_CXX11_CONSTEXPR MDBX_error_t code() const noexcept; + + /// \brief Returns message for MDBX's errors only and "SYSTEM" for others. + const char *what() const noexcept; + + /// \brief Returns message for any errors. + ::std::string message() const; + + /// \brief Returns true for MDBX's errors. + MDBX_CXX11_CONSTEXPR bool is_mdbx_error() const noexcept; + /// \brief Panics on unrecoverable errors inside destructors etc. + [[noreturn]] void panic(const char *context_where_when, + const char *func_who_what) const noexcept; + [[noreturn]] void throw_exception() const; + [[noreturn]] static inline void throw_exception(int error_code); + inline void throw_on_failure() const; + inline void success_or_throw() const; + inline void success_or_throw(const exception_thunk &) const; + inline void panic_on_failure(const char *context_where, + const char *func_who) const noexcept; + inline void success_or_panic(const char *context_where, + const char *func_who) const noexcept; + static inline void throw_on_nullptr(const void *ptr, MDBX_error_t error_code); + static inline void success_or_throw(MDBX_error_t error_code); + static void success_or_throw(int error_code) { + success_or_throw(static_cast<MDBX_error_t>(error_code)); + } + static inline void throw_on_failure(int error_code); + static inline bool boolean_or_throw(int error_code); + static inline void success_or_throw(int error_code, const exception_thunk &); + static inline void panic_on_failure(int error_code, const char *context_where, + const char *func_who) noexcept; + static inline void success_or_panic(int error_code, const char *context_where, + const char *func_who) noexcept; +}; + +/// \brief Base class for all libmdbx's exceptions that are corresponds +/// to libmdbx errors. +/// +/// \see MDBX_error_t +class LIBMDBX_API_TYPE exception : public ::std::runtime_error { + using base = ::std::runtime_error; + ::mdbx::error error_; + +public: + exception(const ::mdbx::error &) noexcept; + exception(const exception &) = default; + exception(exception &&) = default; + exception &operator=(const exception &) = default; + exception &operator=(exception &&) = default; + virtual ~exception() noexcept; + const mdbx::error error() const noexcept { return error_; } +}; + +/// \brief Fatal exception that lead termination anyway +/// in dangerous unrecoverable cases. +class LIBMDBX_API_TYPE fatal : public exception { + using base = exception; + +public: + fatal(const ::mdbx::error &) noexcept; + fatal(const exception &src) noexcept : fatal(src.error()) {} + fatal(exception &&src) noexcept : fatal(src.error()) {} + fatal(const fatal &src) noexcept : fatal(src.error()) {} + fatal(fatal &&src) noexcept : fatal(src.error()) {} + fatal &operator=(fatal &&) = default; + fatal &operator=(const fatal &) = default; + virtual ~fatal() noexcept; +}; + +#define MDBX_DECLARE_EXCEPTION(NAME) \ + struct LIBMDBX_API_TYPE NAME : public exception { \ + NAME(const ::mdbx::error &); \ + virtual ~NAME() noexcept; \ + } +MDBX_DECLARE_EXCEPTION(bad_map_id); +MDBX_DECLARE_EXCEPTION(bad_transaction); +MDBX_DECLARE_EXCEPTION(bad_value_size); +MDBX_DECLARE_EXCEPTION(db_corrupted); +MDBX_DECLARE_EXCEPTION(db_full); +MDBX_DECLARE_EXCEPTION(db_invalid); +MDBX_DECLARE_EXCEPTION(db_too_large); +MDBX_DECLARE_EXCEPTION(db_unable_extend); +MDBX_DECLARE_EXCEPTION(db_version_mismatch); +MDBX_DECLARE_EXCEPTION(db_wanna_write_for_recovery); +MDBX_DECLARE_EXCEPTION(incompatible_operation); +MDBX_DECLARE_EXCEPTION(internal_page_full); +MDBX_DECLARE_EXCEPTION(internal_problem); +MDBX_DECLARE_EXCEPTION(key_exists); +MDBX_DECLARE_EXCEPTION(key_mismatch); +MDBX_DECLARE_EXCEPTION(max_maps_reached); +MDBX_DECLARE_EXCEPTION(max_readers_reached); +MDBX_DECLARE_EXCEPTION(multivalue); +MDBX_DECLARE_EXCEPTION(no_data); +MDBX_DECLARE_EXCEPTION(not_found); +MDBX_DECLARE_EXCEPTION(operation_not_permitted); +MDBX_DECLARE_EXCEPTION(permission_denied_or_not_writeable); +MDBX_DECLARE_EXCEPTION(reader_slot_busy); +MDBX_DECLARE_EXCEPTION(remote_media); +MDBX_DECLARE_EXCEPTION(something_busy); +MDBX_DECLARE_EXCEPTION(thread_mismatch); +MDBX_DECLARE_EXCEPTION(transaction_full); +MDBX_DECLARE_EXCEPTION(transaction_overlapping); +#undef MDBX_DECLARE_EXCEPTION + +[[noreturn]] LIBMDBX_API void throw_too_small_target_buffer(); +[[noreturn]] LIBMDBX_API void throw_max_length_exceeded(); +[[noreturn]] LIBMDBX_API void throw_out_range(); +MDBX_CXX14_CONSTEXPR size_t check_length(size_t bytes); + +//------------------------------------------------------------------------------ + +/// \brief References a data located outside the slice. +/// +/// The `slice` is similar in many ways to `std::string_view`, but it +/// implements specific capabilities and manipulates with bytes but +/// not a characters. +/// +/// \copydetails MDBX_val +struct LIBMDBX_API_TYPE slice : public ::MDBX_val { + /// \todo slice& operator<<(slice&, ...) for reading + /// \todo key-to-value (parse/unpack) functions + /// \todo template<class X> key(X); for decoding keys while reading + + enum { max_length = MDBX_MAXDATASIZE }; + + /// \brief Create an empty slice. + MDBX_CXX11_CONSTEXPR slice() noexcept; + + /// \brief Create a slice that refers to [0,bytes-1] of memory bytes pointed + /// by ptr. + MDBX_CXX14_CONSTEXPR slice(const void *ptr, size_t bytes); + + /// \brief Create a slice that refers to [begin,end] of memory bytes. + MDBX_CXX14_CONSTEXPR slice(const void *begin, const void *end); + + /// \brief Create a slice that refers to text[0,strlen(text)-1]. + template <size_t SIZE> + MDBX_CXX14_CONSTEXPR slice(const char (&text)[SIZE]) noexcept + : slice(text, SIZE - 1) { + static_assert(SIZE > 0 && text[SIZE - 1] == '\0', + "Must be a null-terminated C-string"); + } + /// \brief Create a slice that refers to c_str[0,strlen(c_str)-1]. + explicit MDBX_CXX17_CONSTEXPR slice(const char *c_str); + + /// \brief Create a slice that refers to the contents of "str". + /* 'explicit' to avoid reference to the temporary std::string instance */ + template <class C, class T, class A> + explicit MDBX_CXX20_CONSTEXPR slice(const ::std::basic_string<C, T, A> &str) + : slice(str.data(), str.length() * sizeof(C)) {} + + MDBX_CXX14_CONSTEXPR slice(const MDBX_val &src); + MDBX_CXX11_CONSTEXPR slice(const slice &) noexcept = default; +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + /// \brief Create a slice that refers to the same contents as "sv" + template <class C, class T> + explicit MDBX_CXX14_CONSTEXPR slice(const ::std::basic_string_view<C, T> &sv) + : slice(sv.data(), sv.data() + sv.length()) {} +#endif /* __cpp_lib_string_view >= 201606L */ + + inline slice(MDBX_val &&src); + inline slice(slice &&src) noexcept; +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + slice(::std::basic_string_view<C, T> &&sv) : slice(sv) { + sv = {}; + } +#endif /* __cpp_lib_string_view >= 201606L */ + + template <size_t SIZE> + static MDBX_CXX14_CONSTEXPR slice wrap(const char (&text)[SIZE]) { + return slice(text); + } + + template <typename POD> + MDBX_CXX14_CONSTEXPR static slice wrap(const POD &pod) { + static_assert(::std::is_standard_layout<POD>::value && + !std::is_pointer<POD>::value, + "Must be a standard layout type!"); + return slice(&pod, sizeof(pod)); + } + + inline slice &assign(const void *ptr, size_t bytes); + inline slice &assign(const slice &src) noexcept; + inline slice &assign(const ::MDBX_val &src); + inline slice &assign(slice &&src) noexcept; + inline slice &assign(::MDBX_val &&src); + inline slice &assign(const void *begin, const void *end); + template <class C, class T, class A> + slice &assign(const ::std::basic_string<C, T, A> &str) { + return assign(str.data(), str.length() * sizeof(C)); + } + inline slice &assign(const char *c_str); +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + slice &assign(const ::std::basic_string_view<C, T> &view) { + return assign(view.begin(), view.end()); + } + template <class C, class T> + slice &assign(::std::basic_string_view<C, T> &&view) { + assign(view); + view = {}; + return *this; + } +#endif /* __cpp_lib_string_view >= 201606L */ + + slice &operator=(const slice &) noexcept = default; + inline slice &operator=(slice &&src) noexcept; + inline slice &operator=(::MDBX_val &&src); +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + slice &operator=(const ::std::basic_string_view<C, T> &view) { + return assign(view); + } + template <class C, class T> + slice &operator=(::std::basic_string_view<C, T> &&view) { + return assign(view); + } +#endif /* __cpp_lib_string_view >= 201606L */ + + template <class C = char, class T = ::std::char_traits<C>, + class A = legacy_allocator> + MDBX_CXX20_CONSTEXPR ::std::basic_string<C, T, A> + string(const A &allocator = A()) const { + static_assert(sizeof(C) == 1, "Must be single byte characters"); + return ::std::basic_string<C, T, A>(char_ptr(), length(), allocator); + } + + template <class C, class T, class A> + MDBX_CXX20_CONSTEXPR operator ::std::basic_string<C, T, A>() const { + return this->string<C, T, A>(); + } + + /// \brief Fills the buffer by hexadecimal data dump of slice content. + /// \throws std::length_error if given buffer is too small. + char *to_hex(char *dest, size_t dest_size, bool uppercase = false, + unsigned wrap_width = 0) const; + + /// \brief Returns the buffer size in bytes needed for hexadecimal data dump + /// of slice content. + MDBX_CXX11_CONSTEXPR size_t + to_hex_bytes(unsigned wrap_width = 0) const noexcept { + const size_t bytes = length() << 1; + return wrap_width ? bytes + bytes / wrap_width : bytes; + } + + /// \brief Fills the buffer with data converted from hexadecimal dump + /// from slice content. + /// \throws std::length_error if given buffer is too small. + byte *from_hex(byte *dest, size_t dest_size, + bool ignore_spaces = false) const; + + /// \brief Returns the buffer size in bytes needed for conversion + /// hexadecimal dump from slice content to data. + MDBX_CXX11_CONSTEXPR size_t from_hex_bytes() const noexcept { + return length() >> 1; + } + + /// \brief Fills the buffer by [Base58](https://en.wikipedia.org/wiki/Base58) + /// data dump of slice content. + /// \throws std::length_error if given buffer is too small. + char *to_base58(char *dest, size_t dest_size, unsigned wrap_width = 0) const; + + /// \brief Returns the buffer size in bytes needed for + /// [Base58](https://en.wikipedia.org/wiki/Base58) data dump of slice content. + MDBX_CXX11_CONSTEXPR size_t + to_base58_bytes(unsigned wrap_width = 0) const noexcept { + const size_t bytes = length() / 8 * 11 + (length() % 8 * 43 + 31) / 32; + return wrap_width ? bytes + bytes / wrap_width : bytes; + } + + /// \brief Fills the buffer with data converted from + /// [Base58](https://en.wikipedia.org/wiki/Base58) dump from slice content. + /// \throws std::length_error if given buffer is too small. + byte *from_base58(byte *dest, size_t dest_size, + bool ignore_spaces = false) const; + + /// \brief Returns the buffer size in bytes needed for conversion + /// [Base58](https://en.wikipedia.org/wiki/Base58) dump to data. + MDBX_CXX11_CONSTEXPR size_t from_base58_bytes() const noexcept { + return length() / 11 * 8 + length() % 11 * 32 / 43; + } + + /// \brief Fills the buffer by [Base64](https://en.wikipedia.org/wiki/Base64) + /// data dump. + /// \throws std::length_error if given buffer is too small. + char *to_base64(char *dest, size_t dest_size, unsigned wrap_width = 0) const; + + /// \brief Returns the buffer size in bytes needed for + /// [Base64](https://en.wikipedia.org/wiki/Base64) data dump. + MDBX_CXX11_CONSTEXPR size_t + to_base64_bytes(unsigned wrap_width = 0) const noexcept { + const size_t bytes = (length() + 2) / 3 * 4; + return wrap_width ? bytes + bytes / wrap_width : bytes; + } + + /// \brief Fills the buffer with data converted from + /// [Base64](https://en.wikipedia.org/wiki/Base64) dump. + /// \throws std::length_error if given buffer is too small. + byte *from_base64(byte *dest, size_t dest_size, + bool ignore_spaces = false) const; + + /// \brief Returns the buffer size in bytes needed for conversion + /// [Base64](https://en.wikipedia.org/wiki/Base64) dump to data. + MDBX_CXX11_CONSTEXPR size_t from_base64_bytes() const noexcept { + return (length() + 3) / 4 * 3; + } + + /// \brief Returns a string with a hexadecimal dump of the slice content. + template <class ALLOCATOR = legacy_allocator> + inline ::mdbx::string<ALLOCATOR> + hex_encode(bool uppercase = false, + const ALLOCATOR &allocator = ALLOCATOR()) const; + + /// \brief Decodes hexadecimal dump from the slice content into returned data + /// string. + template <class ALLOCATOR = legacy_allocator> + inline ::mdbx::string<ALLOCATOR> + hex_decode(const ALLOCATOR &allocator = ALLOCATOR()) const; + + /// \brief Returns a string with a + /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. + template <class ALLOCATOR = legacy_allocator> + inline ::mdbx::string<ALLOCATOR> + base58_encode(const ALLOCATOR &allocator = ALLOCATOR()) const; + + /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump + /// from the slice content into returned data string. + template <class ALLOCATOR = legacy_allocator> + inline ::mdbx::string<ALLOCATOR> + base58_decode(const ALLOCATOR &allocator = ALLOCATOR()) const; + + /// \brief Returns a string with a + /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the slice content. + template <class ALLOCATOR = legacy_allocator> + inline ::mdbx::string<ALLOCATOR> + base64_encode(const ALLOCATOR &allocator = ALLOCATOR()) const; + + /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump + /// from the slice content into returned data string. + template <class ALLOCATOR = legacy_allocator> + inline ::mdbx::string<ALLOCATOR> + base64_decode(const ALLOCATOR &allocator = ALLOCATOR()) const; + + /// \brief Checks whether the content of the slice is printable. + /// \param [in] disable_utf8 By default if `disable_utf8` is `false` function + /// checks that content bytes are printable ASCII-7 characters or a valid UTF8 + /// sequences. Otherwise, if if `disable_utf8` is `true` function checks that + /// content bytes are printable extended 8-bit ASCII codes. + MDBX_NOTHROW_PURE_FUNCTION bool + is_printable(bool disable_utf8 = false) const noexcept; + + /// \brief Checks whether the content of the slice is a hexadecimal dump. + /// \param [in] ignore_spaces If `true` function will skips spaces surrounding + /// (before, between and after) a encoded bytes. However, spaces should not + /// break a pair of characters encoding a single byte. + MDBX_NOTHROW_PURE_FUNCTION bool + is_hex(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION bool + + /// \brief Checks whether the content of the slice is a + /// [Base58](https://en.wikipedia.org/wiki/Base58) dump. + /// \param [in] ignore_spaces If `true` function will skips spaces surrounding + /// (before, between and after) a encoded bytes. However, spaces should not + /// break a code group of characters. + is_base58(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION bool + + /// \brief Checks whether the content of the slice is a + /// [Base64](https://en.wikipedia.org/wiki/Base64) dump. + /// \param [in] ignore_spaces If `true` function will skips spaces surrounding + /// (before, between and after) a encoded bytes. However, spaces should not + /// break a code group of characters. + is_base64(bool ignore_spaces = false) const noexcept; + +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + /// \brief Return a string_view that references the same data as this slice. + template <class C, class T> + MDBX_CXX11_CONSTEXPR explicit + operator ::std::basic_string_view<C, T>() const noexcept { + static_assert(sizeof(C) == 1, "Must be single byte characters"); + return ::std::basic_string_view<C, T>(char_ptr(), length()); + } + + /// \brief Return a string_view that references the same data as this slice. + template <class C = char, class T = ::std::char_traits<C>> + MDBX_CXX11_CONSTEXPR ::std::basic_string_view<C, T> + string_view() const noexcept { + static_assert(sizeof(C) == 1, "Must be single byte characters"); + return ::std::basic_string_view<C, T>(char_ptr(), length()); + } +#endif /* __cpp_lib_string_view >= 201606L */ + + inline void swap(slice &other) noexcept; +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + void swap(::std::basic_string_view<C, T> &view) noexcept { + static_assert(sizeof(C) == 1, "Must be single byte characters"); + const auto temp = ::std::basic_string_view<C, T>(*this); + *this = view; + view = temp; + } +#endif /* __cpp_lib_string_view >= 201606L */ + + /// \brief Returns casted to pointer to byte an address of data. + MDBX_CXX11_CONSTEXPR const byte *byte_ptr() const noexcept; + + /// \brief Returns casted to pointer to char an address of data. + MDBX_CXX11_CONSTEXPR const char *char_ptr() const noexcept; + + /// \brief Return a pointer to the beginning of the referenced data. + MDBX_CXX11_CONSTEXPR const void *data() const noexcept; + + /// \brief Returns the number of bytes. + MDBX_CXX11_CONSTEXPR size_t length() const noexcept; + + /// \brief Checks whether the slice is empty. + MDBX_CXX11_CONSTEXPR bool empty() const noexcept; + + /// \brief Checks whether the slice data pointer is nullptr. + MDBX_CXX11_CONSTEXPR bool is_null() const noexcept; + + /// \brief Returns the number of bytes. + MDBX_CXX11_CONSTEXPR size_t size() const noexcept; + + /// \brief Returns true if slice is not empty. + MDBX_CXX11_CONSTEXPR operator bool() const noexcept; + + /// \brief Depletes content of slice and make it invalid. + inline void invalidate() noexcept; + + /// \brief Makes the slice empty and referencing to nothing. + inline void clear() noexcept; + + /// \brief Drops the first "n" bytes from this slice. + /// \pre REQUIRES: `n <= size()` + inline void remove_prefix(size_t n) noexcept; + + /// \brief Drops the last "n" bytes from this slice. + /// \pre REQUIRES: `n <= size()` + inline void remove_suffix(size_t n) noexcept; + + /// \brief Drops the first "n" bytes from this slice. + /// \throws std::out_of_range if `n > size()` + inline void safe_remove_prefix(size_t n); + + /// \brief Drops the last "n" bytes from this slice. + /// \throws std::out_of_range if `n > size()` + inline void safe_remove_suffix(size_t n); + + /// \brief Checks if the data starts with the given prefix. + MDBX_NOTHROW_PURE_FUNCTION inline bool + starts_with(const slice &prefix) const noexcept; + + /// \brief Checks if the data ends with the given suffix. + MDBX_NOTHROW_PURE_FUNCTION inline bool + ends_with(const slice &suffix) const noexcept; + + /// \brief Returns the nth byte in the referenced data. + /// \pre REQUIRES: `n < size()` + inline byte operator[](size_t n) const noexcept; + + /// \brief Returns the nth byte in the referenced data with bounds checking. + /// \throws std::out_of_range if `n >= size()` + inline byte at(size_t n) const; + + /// \brief Returns the first "n" bytes of the slice. + /// \pre REQUIRES: `n <= size()` + inline slice head(size_t n) const noexcept; + + /// \brief Returns the last "n" bytes of the slice. + /// \pre REQUIRES: `n <= size()` + inline slice tail(size_t n) const noexcept; + + /// \brief Returns the middle "n" bytes of the slice. + /// \pre REQUIRES: `from + n <= size()` + inline slice middle(size_t from, size_t n) const noexcept; + + /// \brief Returns the first "n" bytes of the slice. + /// \throws std::out_of_range if `n >= size()` + inline slice safe_head(size_t n) const; + + /// \brief Returns the last "n" bytes of the slice. + /// \throws std::out_of_range if `n >= size()` + inline slice safe_tail(size_t n) const; + + /// \brief Returns the middle "n" bytes of the slice. + /// \throws std::out_of_range if `from + n >= size()` + inline slice safe_middle(size_t from, size_t n) const; + + /// \brief Returns the hash value of referenced data. + /// \attention Function implementation and returned hash values may changed + /// version to version, and in future the t1ha3 will be used here. Therefore + /// values obtained from this function shouldn't be persisted anywhere. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t + hash_value() const noexcept; + + /// \brief Three-way fast non-lexicographically length-based comparison. + /// \return value: + /// == 0 if "a" == "b", + /// < 0 if "a" shorter than "b", + /// > 0 if "a" longer than "b", + /// < 0 if "a" length-equal and lexicographically less than "b", + /// > 0 if "a" length-equal and lexicographically great than "b". + MDBX_NOTHROW_PURE_FUNCTION static inline intptr_t + compare_fast(const slice &a, const slice &b) noexcept; + + /// \brief Three-way lexicographically comparison. + /// \return value: + /// < 0 if "a" < "b", + /// == 0 if "a" == "b", + /// > 0 if "a" > "b". + MDBX_NOTHROW_PURE_FUNCTION static inline intptr_t + compare_lexicographically(const slice &a, const slice &b) noexcept; + friend inline bool operator==(const slice &a, const slice &b) noexcept; + friend inline bool operator<(const slice &a, const slice &b) noexcept; + friend inline bool operator>(const slice &a, const slice &b) noexcept; + friend inline bool operator<=(const slice &a, const slice &b) noexcept; + friend inline bool operator>=(const slice &a, const slice &b) noexcept; + friend inline bool operator!=(const slice &a, const slice &b) noexcept; + + /// \brief Checks the slice is not refers to null address or has zero length. + MDBX_CXX11_CONSTEXPR bool is_valid() const noexcept { + return !(iov_base == nullptr && iov_len != 0); + } + + /// \brief Build an invalid slice which non-zero length and refers to null + /// address. + MDBX_CXX11_CONSTEXPR static slice invalid() noexcept { + return slice(size_t(-1)); + } + +protected: + MDBX_CXX11_CONSTEXPR slice(size_t invalid_length) noexcept + : ::MDBX_val({nullptr, invalid_length}) {} +}; + +//------------------------------------------------------------------------------ + +/// \brief The chunk of data stored inside the buffer or located outside it. +template <class ALLOCATOR = legacy_allocator> class buffer { + friend class txn; + using silo = ::mdbx::string<ALLOCATOR>; + silo silo_; + ::mdbx::slice slice_; + + void insulate() { + assert(is_reference()); + silo_.assign(slice_.char_ptr(), slice_.length()); + slice_.iov_base = const_cast<char *>(silo_.data()); + } + + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte * + silo_begin() const noexcept { + return static_cast<const byte *>(static_cast<const void *>(silo_.data())); + } + + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte * + silo_end() const noexcept { + return silo_begin() + silo_.capacity(); + } + + struct data_preserver : public exception_thunk { + buffer data; + data_preserver(ALLOCATOR &allocator) : data(allocator) {} + static int callback(void *context, MDBX_val *target, const void *src, + size_t bytes) noexcept; + MDBX_CXX11_CONSTEXPR operator MDBX_preserve_func() const noexcept { + return callback; + } + MDBX_CXX11_CONSTEXPR operator const buffer &() const noexcept { + return data; + } + MDBX_CXX11_CONSTEXPR operator buffer &() noexcept { return data; } + }; + +public: + /// \todo buffer& operator<<(buffer&, ...) for writing + /// \todo buffer& operator>>(buffer&, ...) for reading (delegated to slice) + /// \todo template<class X> key(X) for encoding keys while writing + + using allocator_type = ALLOCATOR; + enum : size_t { + max_length = MDBX_MAXDATASIZE, + default_shrink_threshold = 1024 + }; + + /// \brief Returns the associated allocator. + MDBX_CXX20_CONSTEXPR allocator_type get_allocator() const { + return silo_.get_allocator(); + } + + /// \brief Checks whether data chunk stored inside the buffer, otherwise + /// buffer just refers to data located outside the buffer. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool + is_freestanding() const noexcept { + return size_t(byte_ptr() - silo_begin()) < silo_.capacity(); + } + + /// \brief Checks whether the buffer just refers to data located outside + /// the buffer, rather than stores it. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool + is_reference() const noexcept { + return !is_freestanding(); + } + + /// \brief Returns the number of bytes that can be held in currently allocated + /// storage. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t + capacity() const noexcept { + return is_freestanding() ? silo_.capacity() : 0; + } + + /// \brief Returns the number of bytes that available in currently allocated + /// storage ahead the currently beginning of data. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t + headroom() const noexcept { + return is_freestanding() ? slice_.byte_ptr() - silo_begin() : 0; + } + + /// \brief Returns the number of bytes that available in currently allocated + /// storage after the currently data end. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t + tailroom() const noexcept { + return is_freestanding() ? capacity() - headroom() - slice_.length() : 0; + } + + /// \brief Returns casted to const pointer to byte an address of data. + MDBX_CXX11_CONSTEXPR const byte *byte_ptr() const noexcept { + return slice_.byte_ptr(); + } + + /// \brief Returns casted to pointer to byte an address of data. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to + /// an external one. + MDBX_CXX11_CONSTEXPR byte *byte_ptr() noexcept { + assert(is_freestanding()); + return const_cast<byte *>(slice_.byte_ptr()); + } + + /// \brief Returns casted to const pointer to char an address of data. + MDBX_CXX11_CONSTEXPR const char *char_ptr() const noexcept { + return slice_.char_ptr(); + } + + /// \brief Returns casted to pointer to char an address of data. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to + /// an external one. + MDBX_CXX11_CONSTEXPR char *char_ptr() noexcept { + assert(is_freestanding()); + return const_cast<char *>(slice_.char_ptr()); + } + + /// \brief Return a const pointer to the beginning of the referenced data. + MDBX_CXX11_CONSTEXPR const void *data() const noexcept { + return slice_.data(); + } + + /// \brief Return a pointer to the beginning of the referenced data. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to + /// an external one. + MDBX_CXX11_CONSTEXPR void *data() noexcept { + assert(is_freestanding()); + return const_cast<void *>(slice_.data()); + } + + /// \brief Returns the number of bytes. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t + length() const noexcept { + return MDBX_CONSTEXPR_ASSERT(is_reference() || + slice_.length() + headroom() == + silo_.length()), + slice_.length(); + } + + void make_freestanding() { + if (is_reference()) + insulate(); + } + + buffer(const ::mdbx::slice &src, bool make_reference, + const allocator_type &allocator = allocator_type()) + : silo_(allocator), slice_(src) { + if (!make_reference) + insulate(); + } + + buffer(const buffer &src, bool make_reference, + const allocator_type &allocator = allocator_type()) + : buffer(src.slice_, make_reference, allocator) {} + + buffer(const void *ptr, size_t bytes, bool make_reference, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(ptr, bytes), make_reference, allocator) {} + + template <class C, class T, class A> + buffer(const ::std::basic_string<C, T, A> &str, bool make_reference, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(str), make_reference, allocator) {} + + buffer(const char *c_str, bool make_reference, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(c_str), make_reference, allocator) {} + +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + buffer(const ::std::basic_string_view<C, T> &view, bool make_reference, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(view), make_reference, allocator) {} +#endif /* __cpp_lib_string_view >= 201606L */ + + MDBX_CXX20_CONSTEXPR + buffer(const ::mdbx::slice &src, + const allocator_type &allocator = allocator_type()) + : silo_(src.char_ptr(), src.length(), allocator), slice_(silo_) {} + + MDBX_CXX20_CONSTEXPR + buffer(const buffer &src, const allocator_type &allocator = allocator_type()) + : buffer(src.slice_, allocator) {} + + MDBX_CXX20_CONSTEXPR + buffer(const void *ptr, size_t bytes, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(ptr, bytes), allocator) {} + + template <class C, class T, class A> + MDBX_CXX20_CONSTEXPR + buffer(const ::std::basic_string<C, T, A> &str, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(str), allocator) {} + + MDBX_CXX20_CONSTEXPR + buffer(const char *c_str, const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(c_str), allocator) {} + +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + MDBX_CXX20_CONSTEXPR + buffer(const ::std::basic_string_view<C, T> &view, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(view), allocator) {} +#endif /* __cpp_lib_string_view >= 201606L */ + + inline buffer(size_t head_room, size_t tail_room, + const allocator_type &allocator = allocator_type()); + + inline buffer(size_t capacity, + const allocator_type &allocator = allocator_type()); + + inline buffer(size_t head_room, const ::mdbx::slice &src, size_t tail_room, + const allocator_type &allocator = allocator_type()); + + buffer(size_t head_room, const buffer &src, size_t tail_room, + const allocator_type &allocator = allocator_type()) + : buffer(head_room, src.slice_, tail_room, allocator) {} + + MDBX_CXX20_CONSTEXPR + buffer(const allocator_type &allocator = allocator_type()) noexcept + : silo_(allocator) {} + + inline buffer(const txn &txn, const ::mdbx::slice &src, + const allocator_type &allocator = allocator_type()); + + buffer(buffer &&src) noexcept + : silo_(::std::move(src.silo_)), slice_(::std::move(src.slice_)) {} + + buffer(silo &&str) noexcept : silo_(::std::move(str)), slice_(silo_) {} + + MDBX_CXX11_CONSTEXPR const ::mdbx::slice &slice() const noexcept { + return slice_; + } + + MDBX_CXX11_CONSTEXPR operator const ::mdbx::slice &() const noexcept { + return slice_; + } + + template <typename POD> + static buffer wrap(const POD &pod, bool make_reference = false, + const allocator_type &allocator = allocator_type()) { + return buffer(::mdbx::slice::wrap(pod), make_reference, allocator); + } + + /// \brief Reserves storage. + inline void reserve(size_t wanna_headroom, size_t wanna_tailroom, + size_t shrink_threshold = default_shrink_threshold); + + buffer &assign_reference(const void *ptr, size_t bytes) noexcept { + silo_.clear(); + slice_.assign(ptr, bytes); + return *this; + } + + buffer &assign_freestanding(const void *ptr, size_t bytes) { + silo_.assign(static_cast<const typename silo::value_type *>(ptr), + check_length(bytes)); + slice_.assign(silo_); + return *this; + } + + void swap(buffer &other) +#if defined(__cpp_noexcept_function_type) && \ + __cpp_noexcept_function_type >= 201510L + noexcept( + std::allocator_traits<ALLOCATOR>::propagate_on_container_swap::value +#if defined(__cpp_lib_allocator_traits_is_always_equal) && \ + __cpp_lib_allocator_traits_is_always_equal >= 201411L + || std::allocator_traits<ALLOCATOR>::is_always_equal::value +#endif /* __cpp_lib_allocator_traits_is_always_equal */ + ) +#endif /* __cpp_noexcept_function_type */ + ; + + buffer &assign(buffer &&src) +#if defined(__cpp_noexcept_function_type) && \ + __cpp_noexcept_function_type >= 201510L + noexcept(std::allocator_traits< + ALLOCATOR>::propagate_on_container_move_assignment::value +#if defined(__cpp_lib_allocator_traits_is_always_equal) && \ + __cpp_lib_allocator_traits_is_always_equal >= 201411L + || std::allocator_traits<ALLOCATOR>::is_always_equal::value +#endif /* __cpp_lib_allocator_traits_is_always_equal */ + ) +#endif /* __cpp_noexcept_function_type */ + { + silo_.assign(::std::move(src.silo_)); + slice_.assign(::std::move(src.slice_)); + return *this; + } + + buffer &assign(silo &&src) +#if defined(__cpp_noexcept_function_type) && \ + __cpp_noexcept_function_type >= 201510L + noexcept(std::allocator_traits< + ALLOCATOR>::propagate_on_container_move_assignment::value +#if defined(__cpp_lib_allocator_traits_is_always_equal) && \ + __cpp_lib_allocator_traits_is_always_equal >= 201411L + || std::allocator_traits<ALLOCATOR>::is_always_equal::value +#endif /* __cpp_lib_allocator_traits_is_always_equal */ + ) +#endif /* __cpp_noexcept_function_type */ + { + return assign(buffer(::std::move(src))); + } + + static buffer clone(const buffer &src, + const allocator_type &allocator = allocator_type()) { + return buffer(src.headroom(), src.slice_, src.tailroom(), allocator); + } + + buffer &assign(const buffer &src, bool make_reference = false) { + return assign(src.slice_, make_reference); + } + + buffer &assign(const void *ptr, size_t bytes, bool make_reference = false) { + return make_reference ? assign_reference(ptr, bytes) + : assign_freestanding(ptr, bytes); + } + + buffer &assign(const ::mdbx::slice &src, bool make_reference = false) { + return assign(src.data(), src.length(), make_reference); + } + + buffer &assign(const ::MDBX_val &src, bool make_reference = false) { + return assign(src.iov_base, src.iov_len, make_reference); + } + + buffer &assign(::mdbx::slice &&src, bool make_reference = false) { + assign(src.data(), src.length(), make_reference); + src.invalidate(); + return *this; + } + + buffer &assign(::MDBX_val &&src, bool make_reference = false) { + assign(src.iov_base, src.iov_len, make_reference); + src.iov_base = nullptr; + return *this; + } + + buffer &assign(const void *begin, const void *end, + bool make_reference = false) { + return assign(begin, + static_cast<const byte *>(end) - + static_cast<const byte *>(begin), + make_reference); + } + + template <class C, class T, class A> + buffer &assign(const ::std::basic_string<C, T, A> &str, + bool make_reference = false) { + return assign(str.data(), str.length(), make_reference); + } + + buffer &assign(const char *c_str, bool make_reference = false) { + return assign(c_str, ::mdbx::strlen(c_str), make_reference); + } + +#if defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L + template <class C, class T> + buffer &assign(const ::std::basic_string_view<C, T> &view, + bool make_reference = false) { + return assign(view.data(), view.length(), make_reference); + } + + template <class C, class T> + buffer &assign(::std::basic_string_view<C, T> &&view, + bool make_reference = false) { + assign(view.data(), view.length(), make_reference); + view = {}; + return *this; + } +#endif /* __cpp_lib_string_view >= 201606L */ + + buffer &operator=(const buffer &src) { return assign(src); } + + buffer &operator=(buffer &&src) noexcept { return assign(::std::move(src)); } + + buffer &operator=(silo &&src) noexcept { return assign(::std::move(src)); } + + buffer &operator=(const ::mdbx::slice &src) { return assign(src); } + + buffer &operator=(::mdbx::slice &&src) { return assign(::std::move(src)); } + +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + buffer &operator=(const ::std::basic_string_view<C, T> &view) noexcept { + return assign(view); + } + + /// \brief Return a string_view that references the data of this buffer. + template <class C = char, class T = ::std::char_traits<C>> + ::std::basic_string_view<C, T> string_view() const noexcept { + return slice_.string_view<C, T>(); + } + + /// \brief Return a string_view that references the data of this buffer. + template <class C, class T> + operator ::std::basic_string_view<C, T>() const noexcept { + return string_view<C, T>(); + } +#endif /* __cpp_lib_string_view >= 201606L */ + + /// \brief Decodes hexadecimal dump from the given slice to the returned + /// buffer. + static buffer decode_hex(const ::mdbx::slice &hex, + const allocator_type &allocator = allocator_type()) { +#if __cplusplus >= 201703L + return buffer(hex.hex_decode(allocator)); +#else + silo data(hex.hex_decode(allocator)); + return buffer(::std::move(data)); +#endif + } + + /// \brief Returns a buffer with a hexadecimal dump of the given slice. + static buffer encode_hex(const ::mdbx::slice &data, bool uppercase = false, + const allocator_type &allocator = allocator_type()) { +#if __cplusplus >= 201703L + return buffer(data.hex_encode(uppercase, allocator)); +#else + silo hex(data.hex_encode(uppercase, allocator)); + return buffer(::std::move(hex)); +#endif + } + + /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump from + /// the given slice to the returned buffer. + static buffer + decode_base58(const ::mdbx::slice &base58, + const allocator_type &allocator = allocator_type()) { +#if __cplusplus >= 201703L + return buffer(base58.base58_decode(allocator)); +#else + silo data(base58.base58_decode(allocator)); + return buffer(::std::move(data)); +#endif + } + + /// \brief Returns a buffer with a + /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the given slice. + static buffer + encode_base58(const ::mdbx::slice &data, + const allocator_type &allocator = allocator_type()) { +#if __cplusplus >= 201703L + return buffer(data.base58_encode(allocator)); +#else + silo base58(data.base58_encode(allocator)); + return buffer(::std::move(base58)); +#endif + } + + /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump from + /// the given slice to the returned buffer. + static buffer + decode_base64(const ::mdbx::slice &base64, + const allocator_type &allocator = allocator_type()) { +#if __cplusplus >= 201703L + return buffer(base64.base64_decode(allocator)); +#else + silo data(base64.base64_decode(allocator)); + return buffer(::std::move(data)); +#endif + } + + /// \brief Returns a buffer with a + /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the given slice. + static buffer + encode_base64(const ::mdbx::slice &data, + const allocator_type &allocator = allocator_type()) { +#if __cplusplus >= 201703L + return buffer(data.base64_encode(allocator)); +#else + silo base64(data.base64_encode(allocator)); + return buffer(::std::move(base64)); +#endif + } + + /// \brief Checks whether the string is empty. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool empty() const noexcept { + return length() == 0; + } + + /// \brief Checks whether the data pointer of the buffer is nullptr. + MDBX_CXX11_CONSTEXPR bool is_null() const noexcept { + return data() == nullptr; + } + + /// \brief Returns the number of bytes. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t size() const noexcept { + return length(); + } + + /// \brief Returns the hash value of the data. + /// \attention Function implementation and returned hash values may changed + /// version to version, and in future the t1ha3 will be used here. Therefore + /// values obtained from this function shouldn't be persisted anywhere. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t + hash_value() const noexcept { + return slice_.hash_value(); + } + + template <class C = char, class T = ::std::char_traits<C>, + class A = legacy_allocator> + MDBX_CXX20_CONSTEXPR ::std::basic_string<C, T, A> + string(const A &allocator = A()) const { + return slice_.string<C, T, A>(allocator); + } + + template <class C, class T, class A> + MDBX_CXX20_CONSTEXPR operator ::std::basic_string<C, T, A>() const { + return this->string<C, T, A>(); + } + + /// \brief Checks if the data starts with the given prefix. + MDBX_NOTHROW_PURE_FUNCTION bool + starts_with(const ::mdbx::slice &prefix) const noexcept { + return slice_.starts_with(prefix); + } + + /// \brief Checks if the data ends with the given suffix. + MDBX_NOTHROW_PURE_FUNCTION bool + ends_with(const ::mdbx::slice &suffix) const noexcept { + return slice_.ends_with(suffix); + } + + /// \brief Clears the contents and storage. + void clear() noexcept { + slice_.clear(); + silo_.clear(); + } + + /// \brief Reduces memory usage by freeing unused storage space. + void shrink_to_fit(size_t threshold = 64) { reserve(0, 0, threshold); } + + /// \brief Drops the first "n" bytes from the data chunk. + /// \pre REQUIRES: `n <= size()` + void remove_prefix(size_t n) noexcept { slice_.remove_prefix(n); } + + /// \brief Drops the last "n" bytes from the data chunk. + /// \pre REQUIRES: `n <= size()` + void remove_suffix(size_t n) noexcept { slice_.remove_suffix(n); } + + /// \brief Drops the first "n" bytes from the data chunk. + /// \throws std::out_of_range if `n > size()` + void safe_remove_prefix(size_t n) { slice_.safe_remove_prefix(n); } + + /// \brief Drops the last "n" bytes from the data chunk. + /// \throws std::out_of_range if `n > size()` + void safe_remove_suffix(size_t n) { slice_.safe_remove_suffix(n); } + + /// \brief Accesses the specified byte of data chunk. + /// \pre REQUIRES: `n < size()` + byte operator[](size_t n) const noexcept { return slice_[n]; } + + /// \brief Accesses the specified byte of data chunk. + /// \pre REQUIRES: `n < size()` + byte &operator[](size_t n) noexcept { + assert(n < size()); + return byte_ptr()[n]; + } + + /// \brief Accesses the specified byte of data chunk with bounds checking. + /// \throws std::out_of_range if `n >= size()` + byte at(size_t n) const { return slice_.at(n); } + + /// \brief Accesses the specified byte of data chunk with bounds checking. + /// \throws std::out_of_range if `n >= size()` + byte &at(size_t n) { + if (MDBX_UNLIKELY(n >= size())) + MDBX_CXX20_UNLIKELY throw_out_range(); + return byte_ptr()[n]; + } + + /// \brief Returns the first "n" bytes of the data chunk. + /// \pre REQUIRES: `n <= size()` + ::mdbx::slice head(size_t n) const noexcept { return slice_.head(n); } + + /// \brief Returns the last "n" bytes of the data chunk. + /// \pre REQUIRES: `n <= size()` + ::mdbx::slice tail(size_t n) const noexcept { return slice_.tail(n); } + + /// \brief Returns the middle "n" bytes of the data chunk. + /// \pre REQUIRES: `from + n <= size()` + ::mdbx::slice middle(size_t from, size_t n) const noexcept { + return slice_.middle(from, n); + } + + /// \brief Returns the first "n" bytes of the data chunk. + /// \throws std::out_of_range if `n >= size()` + ::mdbx::slice safe_head(size_t n) const { return slice_.safe_head(n); } + + /// \brief Returns the last "n" bytes of the data chunk. + /// \throws std::out_of_range if `n >= size()` + ::mdbx::slice safe_tail(size_t n) const { return slice_.safe_tail(n); } + + /// \brief Returns the middle "n" bytes of the data chunk. + /// \throws std::out_of_range if `from + n >= size()` + ::mdbx::slice safe_middle(size_t from, size_t n) const { + return slice_.safe_middle(from, n); + } + + inline buffer &append(const void *src, size_t bytes); + + buffer &append(const ::mdbx::slice &chunk) { + return append(chunk.data(), chunk.size()); + } + + inline buffer &add_header(const void *src, size_t bytes); + + buffer &add_header(const ::mdbx::slice &chunk) { + return add_header(chunk.data(), chunk.size()); + } + + //---------------------------------------------------------------------------- + + template <size_t SIZE> + static buffer key_from(const char (&text)[SIZE], bool make_reference = true) { + return buffer(::mdbx::slice(text), make_reference); + } + +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template <class C, class T> + static buffer key_from(const ::std::basic_string_view<C, T> &src, + bool make_reference = false) { + return buffer(src, make_reference); + } +#endif /* __cpp_lib_string_view >= 201606L */ + + static buffer key_from(const char *src, bool make_reference = false) { + return buffer(src, make_reference); + } + + template <class C, class T, class A> + static buffer key_from(const ::std::basic_string<C, T, A> &src, + bool make_reference = false) { + return buffer(src, make_reference); + } + + static buffer key_from(const silo &&src) noexcept { + return buffer(::std::move(src)); + } + + static buffer key_from(const double ieee754_64bit) { + return wrap(::mdbx_key_from_double(ieee754_64bit)); + } + + static buffer key_from(const double *ieee754_64bit) { + return wrap(::mdbx_key_from_ptrdouble(ieee754_64bit)); + } + + static buffer key_from(const uint64_t unsigned_int64) { + return wrap(unsigned_int64); + } + + static buffer key_from(const int64_t signed_int64) { + return wrap(::mdbx_key_from_int64(signed_int64)); + } + + static buffer key_from_jsonInteger(const int64_t json_integer) { + return wrap(::mdbx_key_from_jsonInteger(json_integer)); + } + + static buffer key_from(const float ieee754_32bit) { + return wrap(::mdbx_key_from_float(ieee754_32bit)); + } + + static buffer key_from(const float *ieee754_32bit) { + return wrap(::mdbx_key_from_ptrfloat(ieee754_32bit)); + } + + static buffer key_from(const uint32_t unsigned_int32) { + return wrap(unsigned_int32); + } + + static buffer key_from(const int32_t signed_int32) { + return wrap(::mdbx_key_from_int32(signed_int32)); + } +}; + +/// \brief Combines data slice with boolean flag to represent result of certain +/// operations. +struct value_result { + slice value; + bool done; + value_result(const slice &value, bool done) noexcept + : value(value), done(done) {} + value_result(const value_result &) noexcept = default; + value_result &operator=(const value_result &) noexcept = default; + MDBX_CXX14_CONSTEXPR operator bool() const noexcept { + assert(!done || bool(value)); + return done; + } +}; + +/// \brief Combines pair of slices for key and value to represent result of +/// certain operations. +struct pair { + slice key, value; + pair(const slice &key, const slice &value) noexcept + : key(key), value(value) {} + pair(const pair &) noexcept = default; + pair &operator=(const pair &) noexcept = default; + MDBX_CXX14_CONSTEXPR operator bool() const noexcept { + assert(bool(key) == bool(value)); + return key; + } +}; + +/// \brief Combines pair of slices for key and value with boolean flag to +/// represent result of certain operations. +struct pair_result : public pair { + bool done; + pair_result(const slice &key, const slice &value, bool done) noexcept + : pair(key, value), done(done) {} + pair_result(const pair_result &) noexcept = default; + pair_result &operator=(const pair_result &) noexcept = default; + MDBX_CXX14_CONSTEXPR operator bool() const noexcept { + assert(!done || (bool(key) && bool(value))); + return done; + } +}; + +//------------------------------------------------------------------------------ + +/// \brief Loop control constants for readers enumeration functor and other +/// cases. \see env::enumerate_readers() +enum loop_control { continue_loop = 0, exit_loop = INT32_MIN }; + +/// \brief Kinds of the keys and corresponding modes of comparing it. +enum class key_mode { + usual = MDBX_DB_DEFAULTS, ///< Usual variable length keys with byte-by-byte + ///< lexicographic comparison like `std::memcmp()`. + reverse = MDBX_REVERSEKEY, ///< Variable length keys with byte-by-byte + ///< lexicographic comparison in reverse order, + ///< from the end of the keys to the beginning. + ordinal = MDBX_INTEGERKEY, ///< Keys are binary integers in native byte order, + ///< either `uint32_t` or `uint64_t`, and will be + ///< sorted as such. The keys must all be of the + ///< same size and must be aligned while passing + ///< as arguments. + msgpack = -1 ///< Keys are in [MessagePack](https://msgpack.org/) + ///< format with appropriate comparison. + ///< \note Not yet implemented and PRs are welcome. +}; + +/// \brief Kind of the values and sorted multi-values with corresponding +/// comparison. +enum class value_mode { + single = MDBX_DB_DEFAULTS, ///< Usual single value for each key. In terms of + ///< keys, they are unique. + multi = + MDBX_DUPSORT, ///< A more than one data value could be associated with + ///< each key. Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison like `std::memcmp()`. + ///< In terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data values. +#if !defined(__cpp_constexpr) && !defined(DOXYGEN) + multi_reverse = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_REVERSEDUP), + multi_samelength = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED), + multi_ordinal = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED) | + uint32_t(MDBX_INTEGERDUP), + multi_reverse_samelength = uint32_t(MDBX_DUPSORT) | + uint32_t(MDBX_REVERSEDUP) | uint32_t(MDBX_DUPFIXED) +#else + multi_reverse = + MDBX_DUPSORT | + MDBX_REVERSEDUP, ///< A more than one data value could be associated with + ///< each key. Internally each key is stored once, and + ///< the corresponding data values are sorted by + ///< byte-by-byte lexicographic comparison in reverse + ///< order, from the end of the keys to the beginning. + ///< In terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data + ///< values. + multi_samelength = + MDBX_DUPSORT | + MDBX_DUPFIXED, ///< A more than one data value could be associated with + ///< each key, and all data values must be same length. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison like `std::memcmp()`. In + ///< terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data values. + multi_ordinal = + MDBX_DUPSORT | MDBX_DUPFIXED | + MDBX_INTEGERDUP, ///< A more than one data value could be associated with + ///< each key, and all data values are binary integers in + ///< native byte order, either `uint32_t` or `uint64_t`, + ///< and will be sorted as such. Internally each key is + ///< stored once, and the corresponding data values are + ///< sorted. In terms of keys, they are not unique, i.e. + ///< has duplicates which are sorted by associated data + ///< values. + multi_reverse_samelength = + MDBX_DUPSORT | MDBX_REVERSEDUP | + MDBX_DUPFIXED, ///< A more than one data value could be associated with + ///< each key, and all data values must be same length. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison in reverse order, from the + ///< end of the keys to the beginning. In terms of keys, + ///< they are not unique, i.e. has duplicates which are + ///< sorted by associated data values. + msgpack = -1 ///< A more than one data value could be associated with each + ///< key. Values are in [MessagePack](https://msgpack.org/) + ///< format with appropriate comparison. Internally each key is + ///< stored once, and the corresponding data values are sorted. + ///< In terms of keys, they are not unique, i.e. has duplicates + ///< which are sorted by associated data values. + ///< \note Not yet implemented and PRs are welcome. +#endif +}; + +/// \brief A handle for an individual database (key-value spaces) in the +/// environment. +/// \see txn::open_map() \see txn::create_map() +/// \see txn::clear_map() \see txn::drop_map() +/// \see txn::get_handle_info() \see txn::get_map_stat() +/// \see env::close_amp() +/// \see cursor::map() +struct LIBMDBX_API_TYPE map_handle { + MDBX_dbi dbi{0}; + MDBX_CXX11_CONSTEXPR map_handle() noexcept {} + MDBX_CXX11_CONSTEXPR map_handle(MDBX_dbi dbi) noexcept : dbi(dbi) {} + map_handle(const map_handle &) noexcept = default; + map_handle &operator=(const map_handle &) noexcept = default; + operator bool() const noexcept { return dbi != 0; } + + using flags = ::MDBX_db_flags_t; + using state = ::MDBX_dbi_state_t; + struct LIBMDBX_API_TYPE info { + map_handle::flags flags; + map_handle::state state; + MDBX_CXX11_CONSTEXPR info(map_handle::flags flags, + map_handle::state state) noexcept; + info(const info &) noexcept = default; + info &operator=(const info &) noexcept = default; + MDBX_CXX11_CONSTEXPR ::mdbx::key_mode key_mode() const noexcept; + MDBX_CXX11_CONSTEXPR ::mdbx::value_mode value_mode() const noexcept; + }; +}; + +/// \brief Key-value pairs put mode. +enum put_mode { + insert = MDBX_NOOVERWRITE, ///< Insert only unique keys. + upsert = MDBX_UPSERT, ///< Insert or update. + update = MDBX_CURRENT, ///< Update existing, don't insert new. +}; + +/// \brief Unmanaged database environment. +/// +/// Like other unmanaged classes, `env` allows copying and assignment for +/// instances, but does not destroys the represented underlying object from the +/// own class destructor. +/// +/// An environment supports multiple key-value sub-databases (aka key-value +/// spaces or tables), all residing in the same shared-memory map. +class LIBMDBX_API_TYPE env { + friend class txn; + +protected: + MDBX_env *handle_{nullptr}; + MDBX_CXX11_CONSTEXPR env(MDBX_env *ptr) noexcept; + +public: + MDBX_CXX11_CONSTEXPR env() noexcept = default; + env(const env &) noexcept = default; + inline env &operator=(env &&other) noexcept; + inline env(env &&other) noexcept; + inline ~env() noexcept; + + MDBX_CXX14_CONSTEXPR operator bool() const noexcept; + MDBX_CXX14_CONSTEXPR operator const MDBX_env *() const; + MDBX_CXX14_CONSTEXPR operator MDBX_env *(); + friend MDBX_CXX11_CONSTEXPR bool operator==(const env &a, + const env &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, + const env &b) noexcept; + + //---------------------------------------------------------------------------- + + /// Database geometry for size management. + struct LIBMDBX_API_TYPE geometry { + enum : int64_t { + default_value = -1, ///< Means "keep current or use default" + minimal_value = 0, ///< Means "minimal acceptable" + maximal_value = INTPTR_MAX, ///< Means "maximal acceptable" + kB = 1000, ///< \f$10^{3}\f$ bytes + MB = kB * 1000, ///< \f$10^{6}\f$ bytes + GB = MB * 1000, ///< \f$10^{9}\f$ bytes + TB = GB * 1000, ///< \f$10^{12}\f$ bytes + PB = TB * 1000, ///< \f$10^{15}\f$ bytes + EB = PB * 1000, ///< \f$10^{18}\f$ bytes + KiB = 1024, ///< \f$2^{10}\f$ bytes + MiB = KiB << 10, ///< \f$2^{20}\f$ bytes + GiB = MiB << 10, ///< \f$2^{30}\f$ bytes + TiB = GiB << 10, ///< \f$2^{40}\f$ bytes + PiB = TiB << 10, ///< \f$2^{50}\f$ bytes + EiB = PiB << 10, ///< \f$2^{60}\f$ bytes + }; + + /// \brief Tagged type for output to std::ostream + struct size { + intptr_t bytes; + MDBX_CXX11_CONSTEXPR size(intptr_t bytes) noexcept : bytes(bytes) {} + MDBX_CXX11_CONSTEXPR operator intptr_t() const noexcept { return bytes; } + }; + + /// \brief The lower bound of database size in bytes. + intptr_t size_lower{minimal_value}; + + /// \brief The size in bytes to setup the database size for now. + /// \details It is recommended always pass \ref default_value in this + /// argument except some special cases. + intptr_t size_now{default_value}; + + /// \brief The upper bound of database size in bytes. + /// \details It is recommended to avoid change upper bound while database is + /// used by other processes or threaded (i.e. just pass \ref default_value + /// in this argument except absolutely necessary). Otherwise you must be + /// ready for \ref MDBX_UNABLE_EXTEND_MAPSIZE error(s), unexpected pauses + /// during remapping and/or system errors like "address busy", and so on. In + /// other words, there is no way to handle a growth of the upper bound + /// robustly because there may be a lack of appropriate system resources + /// (which are extremely volatile in a multi-process multi-threaded + /// environment). + intptr_t size_upper{maximal_value}; + + /// \brief The growth step in bytes, must be greater than zero to allow the + /// database to grow. + intptr_t growth_step{default_value}; + + /// \brief The shrink threshold in bytes, must be greater than zero to allow + /// the database to shrink. + intptr_t shrink_threshold{default_value}; + + /// \brief The database page size for new database creation + /// or \ref default_value otherwise. + /// \details Must be power of 2 in the range between \ref MDBX_MIN_PAGESIZE + /// and \ref MDBX_MAX_PAGESIZE. + intptr_t pagesize{default_value}; + + inline geometry &make_fixed(intptr_t size) noexcept; + inline geometry &make_dynamic(intptr_t lower = minimal_value, + intptr_t upper = maximal_value) noexcept; + }; + + /// \brief Operation mode. + enum mode { + readonly, ///< \copydoc MDBX_RDONLY + write_file_io, // don't available on OpenBSD + write_mapped_io ///< \copydoc MDBX_WRITEMAP + }; + + /// \brief Durability level. + enum durability { + robust_synchronous, ///< \copydoc MDBX_SYNC_DURABLE + half_synchronous_weak_last, ///< \copydoc MDBX_NOMETASYNC + lazy_weak_tail, ///< \copydoc MDBX_SAFE_NOSYNC + whole_fragile ///< \copydoc MDBX_UTTERLY_NOSYNC + }; + + /// \brief Garbage reclaiming options. + struct LIBMDBX_API_TYPE reclaiming_options { + /// \copydoc MDBX_LIFORECLAIM + bool lifo{false}; + /// \copydoc MDBX_COALESCE + bool coalesce{false}; + MDBX_CXX11_CONSTEXPR reclaiming_options() noexcept {} + reclaiming_options(MDBX_env_flags_t) noexcept; + }; + + /// \brief Operate options. + struct LIBMDBX_API_TYPE operate_options { + /// \copydoc MDBX_NOTLS + bool orphan_read_transactions{false}; + bool nested_write_transactions{false}; + /// \copydoc MDBX_EXCLUSIVE + bool exclusive{false}; + /// \copydoc MDBX_NORDAHEAD + bool disable_readahead{false}; + /// \copydoc MDBX_NOMEMINIT + bool disable_clear_memory{false}; + MDBX_CXX11_CONSTEXPR operate_options() noexcept {} + operate_options(MDBX_env_flags_t) noexcept; + }; + + /// \brief Operate parameters. + struct LIBMDBX_API_TYPE operate_parameters { + /// \brief The maximum number of named databases for the environment. + /// Zero means default value. + unsigned max_maps{0}; + /// \brief The maximum number of threads/reader slots for the environment. + /// Zero means default value. + unsigned max_readers{0}; + env::mode mode{write_mapped_io}; + env::durability durability{robust_synchronous}; + env::reclaiming_options reclaiming; + env::operate_options options; + + MDBX_CXX11_CONSTEXPR operate_parameters() noexcept {} + MDBX_env_flags_t make_flags(bool accede = true, ///< \copydoc MDBX_ACCEDE + bool use_subdirectory = false) const; + static env::mode mode_from_flags(MDBX_env_flags_t) noexcept; + static env::durability durability_from_flags(MDBX_env_flags_t) noexcept; + inline static env::reclaiming_options + reclaiming_from_flags(MDBX_env_flags_t flags) noexcept; + inline static env::operate_options + options_from_flags(MDBX_env_flags_t flags) noexcept; + operate_parameters(const env &); + }; + + /// \brief Returns current operation parameters. + inline env::operate_parameters get_operation_parameters() const; + /// \brief Returns current operation mode. + inline env::mode get_mode() const; + /// \brief Returns current durability mode. + inline env::durability get_durability() const; + /// \brief Returns current reclaiming options. + inline env::reclaiming_options get_reclaiming() const; + /// \brief Returns current operate options. + inline env::operate_options get_options() const; + + /// \brief Returns `true` for a freshly created database, + /// but `false` if at least one transaction was committed. + bool is_pristine() const; + + /// \brief Checks whether the database is empty. + bool is_empty() const; + + /// \brief Returns default page size for current system/platform. + static size_t default_pagesize() noexcept; + + struct limits { + limits() = delete; + /// \brief Returns the minimal database page size in bytes. + static inline size_t pagesize_min() noexcept; + /// \brief Returns the maximal database page size in bytes. + static inline size_t pagesize_max() noexcept; + /// \brief Returns the minimal database size in bytes for specified page + /// size. + static inline size_t dbsize_min(intptr_t pagesize); + /// \brief Returns the maximal database size in bytes for specified page + /// size. + static inline size_t dbsize_max(intptr_t pagesize); + /// \brief Returns the minimal key size in bytes for specified database + /// flags. + static inline size_t key_min(MDBX_db_flags_t flags) noexcept; + /// \brief Returns the minimal key size in bytes for specified keys mode. + static inline size_t key_min(key_mode mode) noexcept; + /// \brief Returns the maximal key size in bytes for specified page size and + /// database flags. + static inline size_t key_max(intptr_t pagesize, MDBX_db_flags_t flags); + /// \brief Returns the maximal key size in bytes for specified page size and + /// keys mode. + static inline size_t key_max(intptr_t pagesize, key_mode mode); + /// \brief Returns the maximal key size in bytes for given environment and + /// database flags. + static inline size_t key_max(const env &, MDBX_db_flags_t flags); + /// \brief Returns the maximal key size in bytes for given environment and + /// keys mode. + static inline size_t key_max(const env &, key_mode mode); + /// \brief Returns the minimal values size in bytes for specified database + /// flags. + static inline size_t value_min(MDBX_db_flags_t flags) noexcept; + /// \brief Returns the minimal values size in bytes for specified values + /// mode. + static inline size_t value_min(value_mode) noexcept; + /// \brief Returns the maximal value size in bytes for specified page size + /// and database flags. + static inline size_t value_max(intptr_t pagesize, MDBX_db_flags_t flags); + /// \brief Returns the maximal value size in bytes for specified page size + /// and values mode. + static inline size_t value_max(intptr_t pagesize, value_mode); + /// \brief Returns the maximal value size in bytes for given environment and + /// database flags. + static inline size_t value_max(const env &, MDBX_db_flags_t flags); + /// \brief Returns the maximal value size in bytes for specified page size + /// and values mode. + static inline size_t value_max(const env &, value_mode); + /// \brief Returns the maximal write transaction size (i.e. limit for + /// summary volume of dirty pages) in bytes for specified page size. + static inline size_t transaction_size_max(intptr_t pagesize); + }; + + /// \brief Returns the minimal database size in bytes for the environment. + size_t dbsize_min() const { return limits::dbsize_min(this->get_pagesize()); } + /// \brief Returns the maximal database size in bytes for the environment. + size_t dbsize_max() const { return limits::dbsize_max(this->get_pagesize()); } + /// \brief Returns the minimal key size in bytes for specified keys mode. + size_t key_min(key_mode mode) const noexcept { return limits::key_min(mode); } + /// \brief Returns the maximal key size in bytes for specified keys mode. + size_t key_max(key_mode mode) const { return limits::key_max(*this, mode); } + /// \brief Returns the minimal value size in bytes for specified values mode. + size_t value_min(value_mode mode) const noexcept { + return limits::value_min(mode); + } + /// \brief Returns the maximal value size in bytes for specified values mode. + size_t value_max(value_mode mode) const { + return limits::value_max(*this, mode); + } + /// \brief Returns the maximal write transaction size (i.e. limit for summary + /// volume of dirty pages) in bytes. + size_t transaction_size_max() const { + return limits::transaction_size_max(this->get_pagesize()); + } + + /// \brief Make a copy (backup) of an existing environment to the specified + /// path. + env ©(const path &destination, bool compactify, + bool force_dynamic_size = false); + + /// \brief Copy an environment to the specified file descriptor. + env ©(filehandle fd, bool compactify, bool force_dynamic_size = false); + + /// \brief Statistics for a database in the MDBX environment. + using stat = ::MDBX_stat; + + /// \brief Information about the environment. + using info = ::MDBX_envinfo; + + /// \brief Returns snapshot statistics about the MDBX environment. + inline stat get_stat() const; + + /// \brief Returns pagesize of this MDBX environment. + size_t get_pagesize() const { return get_stat().ms_psize; } + + /// \brief Return snapshot information about the MDBX environment. + inline info get_info() const; + + /// \brief Return statistics about the MDBX environment accordingly to the + /// specified transaction. + inline stat get_stat(const txn &) const; + + /// \brief Return information about the MDBX environment accordingly to the + /// specified transaction. + inline info get_info(const txn &) const; + + /// \brief Returns the file descriptor for the DXB file of MDBX environment. + inline filehandle get_filehandle() const; + + /// \brief Return the path that was used for opening the environment. + path get_path() const; + + /// Returns environment flags. + inline MDBX_env_flags_t get_flags() const; + + /// \brief Returns the maximum number of threads/reader slots for the + /// environment. + inline unsigned max_readers() const; + + /// \brief Returns the maximum number of named databases for the environment. + inline unsigned max_maps() const; + + /// \brief Returns the application context associated with the environment. + inline void *get_context() const noexcept; + + /// \brief Sets the application context associated with the environment. + inline env &set_context(void *); + + /// \brief Sets threshold to force flush the data buffers to disk, for + /// non-sync durability modes. + /// + /// The threshold value affects all processes which operates with given + /// environment until the last process close environment or a new value will + /// be settled. + /// Data is always written to disk when \ref txn_managed::commit() is called, + /// but the operating system may keep it buffered. MDBX always flushes the OS + /// buffers upon commit as well, unless the environment was opened with \ref + /// whole_fragile, \ref lazy_weak_tail or in part \ref + /// half_synchronous_weak_last. The default is 0, than mean no any threshold + /// checked, and no additional flush will be made. + /// + inline env &set_sync_threshold(size_t bytes); + + /// \brief Sets relative period since the last unsteady commit to force flush + /// the data buffers to disk, for non-sync durability modes. + /// + /// The relative period value affects all processes which operates with given + /// environment until the last process close environment or a new value will + /// be settled. + /// Data is always written to disk when \ref txn_managed::commit() is called, + /// but the operating system may keep it buffered. MDBX always flushes the OS + /// buffers upon commit as well, unless the environment was opened with \ref + /// whole_fragile, \ref lazy_weak_tail or in part \ref + /// half_synchronous_weak_last. Settled period don't checked asynchronously, + /// but only by the \ref txn_managed::commit() and \ref env::sync_to_disk() + /// functions. Therefore, in cases where transactions are committed + /// infrequently and/or irregularly, polling by \ref env::poll_sync_to_disk() + /// may be a reasonable solution to timeout enforcement. The default is 0, + /// than mean no any timeout checked, and no additional flush will be made. + /// + /// \param [in] seconds_16dot16 The period in 1/65536 of second when a + /// synchronous flush would be made since the last unsteady commit. + inline env &set_sync_period(unsigned seconds_16dot16); + + /// \brief Sets relative period since the last unsteady commit to force flush + /// the data buffers to disk, for non-sync durability modes. + /// + /// The relative period value affects all processes which operates with given + /// environment until the last process close environment or a new value will + /// be settled. + /// Data is always written to disk when \ref txn_managed::commit() is called, + /// but the operating system may keep it buffered. MDBX always flushes the OS + /// buffers upon commit as well, unless the environment was opened with \ref + /// whole_fragile, \ref lazy_weak_tail or in part \ref + /// half_synchronous_weak_last. Settled period don't checked asynchronously, + /// but only by the \ref txn_managed::commit() and \ref env::sync_to_disk() + /// functions. Therefore, in cases where transactions are committed + /// infrequently and/or irregularly, polling by \ref env::poll_sync_to_disk() + /// may be a reasonable solution to timeout enforcement. The default is 0, + /// than mean no any timeout checked, and no additional flush will be made. + /// + /// \param [in] seconds The period in second when a synchronous flush would + /// be made since the last unsteady commit. + inline env &set_sync_period(double seconds); + + /// \brief Alter environment flags. + inline env &alter_flags(MDBX_env_flags_t flags, bool on_off); + + /// \brief Set all size-related parameters of environment. + inline env &set_geometry(const geometry &size); + + /// \brief Flush the environment data buffers. + /// \return `True` if sync done or no data to sync, or `false` if the + /// environment is busy by other thread or none of the thresholds are reached. + inline bool sync_to_disk(bool force = true, bool nonblock = false); + + /// \brief Performs non-blocking polling of sync-to-disk thresholds. + /// \return `True` if sync done or no data to sync, or `false` if the + /// environment is busy by other thread or none of the thresholds are reached. + bool poll_sync_to_disk() { return sync_to_disk(false, true); } + + /// \brief Close a key-value map (aka sub-database) handle. Normally + /// unnecessary. + /// + /// Closing a database handle is not necessary, but lets \ref txn::open_map() + /// reuse the handle value. Usually it's better to set a bigger + /// \ref env::operate_parameters::max_maps, unless that value would be + /// large. + /// + /// \note Use with care. + /// This call is synchronized via mutex with other calls \ref close_map(), but + /// NOT with other transactions running by other threads. The "next" version + /// of libmdbx (\ref MithrilDB) will solve this issue. + /// + /// Handles should only be closed if no other threads are going to reference + /// the database handle or one of its cursors any further. Do not close a + /// handle if an existing transaction has modified its database. Doing so can + /// cause misbehavior from database corruption to errors like + /// \ref MDBX_BAD_DBI (since the DB name is gone). + inline void close_map(const map_handle &); + + /// \brief Readed information + struct reader_info { + int slot; ///< The reader lock table slot number. + mdbx_pid_t pid; ///< The reader process ID. + mdbx_tid_t thread; ///< The reader thread ID. + uint64_t transaction_id; ///< The ID of the transaction being read, + ///< i.e. the MVCC-snapshot number. + uint64_t transaction_lag; ///< The lag from a recent MVCC-snapshot, + ///< i.e. the number of committed write + /// transactions since the current read + /// transaction started. + size_t bytes_used; ///< The number of last used page in the MVCC-snapshot + ///< which being read, i.e. database file can't shrinked + ///< beyond this. + size_t bytes_retained; ///< The total size of the database pages that + ///< were retired by committed write transactions + ///< after the reader's MVCC-snapshot, i.e. the space + ///< which would be freed after the Reader releases + ///< the MVCC-snapshot for reuse by completion read + ///< transaction. + + MDBX_CXX11_CONSTEXPR reader_info(int slot, mdbx_pid_t pid, + mdbx_tid_t thread, uint64_t txnid, + uint64_t lag, size_t used, + size_t retained) noexcept; + }; + + /// \brief Enumerate readers. + /// + /// The VISITOR class must have `int operator(const reader_info&, int serial)` + /// which should return \ref continue_loop (zero) to continue enumeration, + /// or any non-zero value to exit. + /// + /// \returns The last value returned from visitor' functor. + template <typename VISITOR> inline int enumerate_readers(VISITOR &visitor); + + /// \brief Checks for stale readers in the lock table and + /// return number of cleared slots. + inline unsigned check_readers(); + + /// \brief Sets a Handle-Slow-Readers callback to resolve database + /// full/overflow issue due to a reader(s) which prevents the old data from + /// being recycled. + /// + /// Such callback will be triggered in a case where there is not enough free + /// space in the database due to long read transaction(s) which impedes + /// reusing the pages of an old MVCC snapshot(s). + /// + /// Using this callback you can choose how to resolve the situation: + /// - abort the write transaction with an error; + /// - wait for the read transaction(s) to complete; + /// - notify a thread performing a long-lived read transaction + /// and wait for an effect; + /// - kill the thread or whole process that performs the long-lived read + /// transaction; + /// + /// \see long-lived-read + inline env &set_HandleSlowReaders(MDBX_hsr_func *); + + /// \brief Returns the current Handle-Slow-Readers callback used to resolve + /// database full/overflow issue due to a reader(s) which prevents the old + /// data from being recycled. + /// \see set_HandleSlowReaders() + inline MDBX_hsr_func *get_HandleSlowReaders() const noexcept; + + /// \brief Starts read (read-only) transaction. + inline txn_managed start_read() const; + + /// \brief Creates but not start read transaction. + inline txn_managed prepare_read() const; + + /// \brief Starts write (read-write) transaction. + inline txn_managed start_write(bool dont_wait = false); + + /// \brief Tries to start write (read-write) transaction without blocking. + inline txn_managed try_start_write(); +}; + +/// \brief Managed database environment. +/// +/// As other managed classes, `env_managed` destroys the represented underlying +/// object from the own class destructor, but disallows copying and assignment +/// for instances. +/// +/// An environment supports multiple key-value databases (aka key-value spaces +/// or tables), all residing in the same shared-memory map. +class LIBMDBX_API_TYPE env_managed : public env { + using inherited = env; + /// delegated constructor for RAII + MDBX_CXX11_CONSTEXPR env_managed(MDBX_env *ptr) noexcept : inherited(ptr) {} + void setup(unsigned max_maps, unsigned max_readers = 0); + +public: + MDBX_CXX11_CONSTEXPR env_managed() noexcept = default; + + /// \brief Open existing database. + env_managed(const path &, const operate_parameters &, bool accede = true); + + /// \brief Additional parameters for creating a new database. + struct create_parameters { + env::geometry geometry; + mdbx_mode_t file_mode_bits{0640}; + bool use_subdirectory{false}; + }; + + /// \brief Create new or open existing database. + env_managed(const path &, const create_parameters &, + const operate_parameters &, bool accede = true); + + /// \brief Explicitly closes the environment and release the memory map. + /// + /// Only a single thread may call this function. All transactions, databases, + /// and cursors must already be closed before calling this function. Attempts + /// to use any such handles after calling this function will cause a + /// `SIGSEGV`. The environment handle will be freed and must not be used again + /// after this call. + /// + /// \param [in] dont_sync A dont'sync flag, if non-zero the last checkpoint + /// will be kept "as is" and may be still "weak" in the \ref lazy_weak_tail + /// or \ref whole_fragile modes. Such "weak" checkpoint will be ignored + /// on opening next time, and transactions since the last non-weak checkpoint + /// (meta-page update) will rolledback for consistency guarantee. + void close(bool dont_sync = false); + + env_managed(env_managed &&) = default; + env_managed &operator=(env_managed &&) = default; + env_managed(const env_managed &) = delete; + env_managed &operator=(const env_managed &) = delete; + virtual ~env_managed() noexcept; +}; + +/// \brief Unmanaged database transaction. +/// +/// Like other unmanaged classes, `txn` allows copying and assignment for +/// instances, but does not destroys the represented underlying object from the +/// own class destructor. +/// +/// All database operations require a transaction handle. Transactions may be +/// read-only or read-write. +class LIBMDBX_API_TYPE txn { +protected: + friend class cursor; + MDBX_txn *handle_{nullptr}; + MDBX_CXX11_CONSTEXPR txn(MDBX_txn *ptr) noexcept; + +public: + MDBX_CXX11_CONSTEXPR txn() noexcept = default; + txn(const txn &) noexcept = default; + inline txn &operator=(txn &&other) noexcept; + inline txn(txn &&other) noexcept; + inline ~txn() noexcept; + + MDBX_CXX14_CONSTEXPR operator bool() const noexcept; + MDBX_CXX14_CONSTEXPR operator const MDBX_txn *() const; + MDBX_CXX14_CONSTEXPR operator MDBX_txn *(); + friend MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, + const txn &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, + const txn &b) noexcept; + + /// \brief Returns the transaction's environment. + inline ::mdbx::env env() const noexcept; + /// \brief Returns transaction's flags. + inline MDBX_txn_flags_t flags() const; + /// \brief Return the transaction's ID. + inline uint64_t id() const; + + /// \brief Checks whether the given data is on a dirty page. + inline bool is_dirty(const void *ptr) const; + + /// \brief Checks whether the transaction is read-only. + bool is_readonly() const { return (flags() & MDBX_TXN_RDONLY) != 0; } + + /// \brief Checks whether the transaction is read-write. + bool is_readwrite() const { return (flags() & MDBX_TXN_RDONLY) == 0; } + + using info = ::MDBX_txn_info; + /// \brief Returns information about the MDBX transaction. + inline info get_info(bool scan_reader_lock_table = false) const; + + /// \brief Returns maximal write transaction size (i.e. limit for summary + /// volume of dirty pages) in bytes. + size_t size_max() const { return env().transaction_size_max(); } + + /// \brief Returns current write transaction size (i.e.summary volume of dirty + /// pages) in bytes. + size_t size_current() const { + assert(is_readwrite()); + return size_t(get_info().txn_space_dirty); + } + + //---------------------------------------------------------------------------- + + /// \brief Reset a read-only transaction. + inline void reset_reading(); + + /// \brief Renew a read-only transaction. + inline void renew_reading(); + + /// \brief Start nested write transaction. + txn_managed start_nested(); + + /// \brief Opens cursor for specified key-value map handle. + inline cursor_managed open_cursor(map_handle map); + + /// \brief Open existing key-value map. + inline map_handle open_map( + const char *name, + const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + /// \brief Open existing key-value map. + inline map_handle open_map( + const ::std::string &name, + const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + + /// \brief Create new or open existing key-value map. + inline map_handle + create_map(const char *name, + const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); + /// \brief Create new or open existing key-value map. + inline map_handle + create_map(const ::std::string &name, + const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); + + /// \brief Drops key-value map using handle. + inline void drop_map(map_handle map); + /// \brief Drops key-value map using name. + /// \return `True` if the key-value map existed and was deleted, either + /// `false` if the key-value map did not exist and there is nothing to delete. + bool drop_map(const char *name, bool throw_if_absent = false); + /// \brief Drop key-value map. + /// \return `True` if the key-value map existed and was deleted, either + /// `false` if the key-value map did not exist and there is nothing to delete. + inline bool drop_map(const ::std::string &name, bool throw_if_absent = false); + + /// \brief Clear key-value map. + inline void clear_map(map_handle map); + /// \return `True` if the key-value map existed and was cleared, either + /// `false` if the key-value map did not exist and there is nothing to clear. + bool clear_map(const char *name, bool throw_if_absent = false); + /// \return `True` if the key-value map existed and was cleared, either + /// `false` if the key-value map did not exist and there is nothing to clear. + inline bool clear_map(const ::std::string &name, + bool throw_if_absent = false); + + using map_stat = ::MDBX_stat; + /// \brief Returns statistics for a sub-database. + inline map_stat get_map_stat(map_handle map) const; + /// \brief Returns depth (bitmask) information of nested dupsort (multi-value) + /// B+trees for given database. + inline uint32_t get_tree_deepmask(map_handle map) const; + /// \brief Returns information about key-value map (aka sub-database) handle. + inline map_handle::info get_handle_info(map_handle map) const; + + using canary = ::MDBX_canary; + /// \brief Set integers markers (aka "canary") associated with the + /// environment. + inline txn &put_canary(const canary &); + /// \brief Returns fours integers markers (aka "canary") associated with the + /// environment. + inline canary get_canary() const; + + /// Reads sequence generator associated with a key-value map (aka + /// sub-database). + inline uint64_t sequence(map_handle map) const; + /// \brief Reads and increment sequence generator associated with a key-value + /// map (aka sub-database). + inline uint64_t sequence(map_handle map, uint64_t increment); + + /// \brief Compare two keys according to a particular key-value map (aka + /// sub-database). + inline int compare_keys(map_handle map, const slice &a, + const slice &b) const noexcept; + /// \brief Compare two values according to a particular key-value map (aka + /// sub-database). + inline int compare_values(map_handle map, const slice &a, + const slice &b) const noexcept; + /// \brief Compare keys of two pairs according to a particular key-value map + /// (aka sub-database). + inline int compare_keys(map_handle map, const pair &a, + const pair &b) const noexcept; + /// \brief Compare values of two pairs according to a particular key-value map + /// (aka sub-database). + inline int compare_values(map_handle map, const pair &a, + const pair &b) const noexcept; + + /// \brief Get value by key from a key-value map (aka sub-database). + inline slice get(map_handle map, const slice &key) const; + /// \brief Get first of multi-value and values count by key from a key-value + /// multimap (aka sub-database). + inline slice get(map_handle map, slice key, size_t &values_count) const; + /// \brief Get value by key from a key-value map (aka sub-database). + inline slice get(map_handle map, const slice &key, + const slice &value_at_absence) const; + /// \brief Get first of multi-value and values count by key from a key-value + /// multimap (aka sub-database). + inline slice get(map_handle map, slice key, size_t &values_count, + const slice &value_at_absence) const; + /// \brief Get value for equal or great key from a database. + /// \return Bundle of key-value pair and boolean flag, + /// which will be `true` if the exact key was found and `false` otherwise. + inline pair_result get_equal_or_great(map_handle map, const slice &key) const; + /// \brief Get value for equal or great key from a database. + /// \return Bundle of key-value pair and boolean flag, + /// which will be `true` if the exact key was found and `false` otherwise. + inline pair_result get_equal_or_great(map_handle map, const slice &key, + const slice &value_at_absence) const; + + inline MDBX_error_t put(map_handle map, const slice &key, slice *value, + MDBX_put_flags_t flags) noexcept; + inline void put(map_handle map, const slice &key, slice value, + put_mode mode) noexcept; + inline void insert(map_handle map, const slice &key, slice value); + inline value_result try_insert(map_handle map, const slice &key, slice value); + inline slice insert_reserve(map_handle map, const slice &key, + size_t value_length); + inline value_result try_insert_reserve(map_handle map, const slice &key, + size_t value_length); + + inline void upsert(map_handle map, const slice &key, const slice &value); + inline slice upsert_reserve(map_handle map, const slice &key, + size_t value_length); + + inline void update(map_handle map, const slice &key, const slice &value); + inline bool try_update(map_handle map, const slice &key, const slice &value); + inline slice update_reserve(map_handle map, const slice &key, + size_t value_length); + inline value_result try_update_reserve(map_handle map, const slice &key, + size_t value_length); + + inline bool erase(map_handle map, const slice &key); + + /// \brief Removes the particular multi-value entry of the key. + inline bool erase(map_handle map, const slice &key, const slice &value); + + /// \brief Replaces the particular multi-value of the key with a new value. + inline void replace(map_handle map, const slice &key, slice old_value, + const slice &new_value); + + /// \brief Removes and return a value of the key. + template <class ALLOCATOR> + inline buffer<ALLOCATOR> extract(map_handle map, const slice &key, + const ALLOCATOR &allocator = ALLOCATOR()); + + /// \brief Replaces and returns a value of the key with new one. + template <class ALLOCATOR> + inline buffer<ALLOCATOR> replace(map_handle map, const slice &key, + const slice &new_value, + const ALLOCATOR &allocator = ALLOCATOR()); + + template <class ALLOCATOR> + inline buffer<ALLOCATOR> + replace_reserve(map_handle map, const slice &key, slice &new_value, + const ALLOCATOR &allocator = ALLOCATOR()); + + /// \brief Adding a key-value pair, provided that ascending order of the keys + /// and (optionally) values are preserved. + /// + /// Instead of splitting the full b+tree pages, the data will be placed on new + /// ones. Thus appending is about two times faster than insertion, and the + /// pages will be filled in completely mostly but not half as after splitting + /// ones. On the other hand, any subsequent insertion or update with an + /// increase in the length of the value will be twice as slow, since it will + /// require splitting already filled pages. + /// + /// \param [in] multivalue_order_preserved + /// If `multivalue_order_preserved == true` then the same rules applied for + /// to pages of nested b+tree of multimap's values. + inline void append(map_handle map, const slice &key, const slice &value, + bool multivalue_order_preserved = true); + + size_t put_multiple(map_handle map, const slice &key, + const size_t value_length, const void *values_array, + size_t values_count, put_mode mode, + bool allow_partial = false); + template <typename VALUE> + void put_multiple(map_handle map, const slice &key, + const std::vector<VALUE> &vector, put_mode mode) { + put_multiple(map, key, sizeof(VALUE), vector.data(), vector.size(), mode, + false); + } + + inline ptrdiff_t estimate(map_handle map, pair from, pair to) const; + inline ptrdiff_t estimate(map_handle map, slice from, slice to) const; + inline ptrdiff_t estimate_from_first(map_handle map, slice to) const; + inline ptrdiff_t estimate_to_last(map_handle map, slice from) const; +}; + +/// \brief Managed database transaction. +/// +/// As other managed classes, `txn_managed` destroys the represented underlying +/// object from the own class destructor, but disallows copying and assignment +/// for instances. +/// +/// All database operations require a transaction handle. Transactions may be +/// read-only or read-write. +class LIBMDBX_API_TYPE txn_managed : public txn { + using inherited = txn; + friend class env; + friend class txn; + /// delegated constructor for RAII + MDBX_CXX11_CONSTEXPR txn_managed(MDBX_txn *ptr) noexcept : inherited(ptr) {} + +public: + MDBX_CXX11_CONSTEXPR txn_managed() noexcept = default; + txn_managed(txn_managed &&) = default; + txn_managed &operator=(txn_managed &&) = default; + txn_managed(const txn_managed &) = delete; + txn_managed &operator=(const txn_managed &) = delete; + ~txn_managed() noexcept; + + //---------------------------------------------------------------------------- + + /// \brief Abandon all the operations of the transaction instead of saving + /// them. + void abort(); + + /// \brief Commit all the operations of a transaction into the database. + void commit(); +}; + +/// \brief Unmanaged cursor. +/// +/// Like other unmanaged classes, `cursor` allows copying and assignment for +/// instances, but does not destroys the represented underlying object from the +/// own class destructor. +/// +/// \copydetails MDBX_cursor +class LIBMDBX_API_TYPE cursor { +protected: + MDBX_cursor *handle_{nullptr}; + MDBX_CXX11_CONSTEXPR cursor(MDBX_cursor *ptr) noexcept; + +public: + MDBX_CXX11_CONSTEXPR cursor() noexcept = default; + cursor(const cursor &) noexcept = default; + inline cursor &operator=(cursor &&other) noexcept; + inline cursor(cursor &&other) noexcept; + inline ~cursor() noexcept; + MDBX_CXX14_CONSTEXPR operator bool() const noexcept; + MDBX_CXX14_CONSTEXPR operator const MDBX_cursor *() const; + MDBX_CXX14_CONSTEXPR operator MDBX_cursor *(); + friend MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, + const cursor &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, + const cursor &b) noexcept; + + enum move_operation { + first = MDBX_FIRST, + last = MDBX_LAST, + next = MDBX_NEXT, + previous = MDBX_PREV, + get_current = MDBX_GET_CURRENT, + + multi_prevkey_lastvalue = MDBX_PREV_NODUP, + multi_currentkey_firstvalue = MDBX_FIRST_DUP, + multi_currentkey_prevvalue = MDBX_PREV_DUP, + multi_currentkey_nextvalue = MDBX_NEXT_DUP, + multi_currentkey_lastvalue = MDBX_LAST_DUP, + multi_nextkey_firstvalue = MDBX_NEXT_NODUP, + + multi_find_pair = MDBX_GET_BOTH, + multi_exactkey_lowerboundvalue = MDBX_GET_BOTH_RANGE, + + find_key = MDBX_SET, + key_exact = MDBX_SET_KEY, + key_lowerbound = MDBX_SET_RANGE + }; + + struct move_result : public pair_result { + inline move_result(const cursor &cursor, bool throw_notfound); + inline move_result(cursor &cursor, move_operation operation, + bool throw_notfound); + inline move_result(cursor &cursor, move_operation operation, + const slice &key, bool throw_notfound); + inline move_result(cursor &cursor, move_operation operation, + const slice &key, const slice &value, + bool throw_notfound); + move_result(const move_result &) noexcept = default; + }; + +protected: + inline bool move(move_operation operation, MDBX_val *key, MDBX_val *value, + bool throw_notfound) const + /* fake const, i.e. for some operations */; + inline ptrdiff_t estimate(move_operation operation, MDBX_val *key, + MDBX_val *value) const; + +public: + inline move_result move(move_operation operation, bool throw_notfound); + inline move_result to_first(bool throw_notfound = true); + inline move_result to_previous(bool throw_notfound = true); + inline move_result to_previous_last_multi(bool throw_notfound = true); + inline move_result to_current_first_multi(bool throw_notfound = true); + inline move_result to_current_prev_multi(bool throw_notfound = true); + inline move_result current(bool throw_notfound = true) const; + inline move_result to_current_next_multi(bool throw_notfound = true); + inline move_result to_current_last_multi(bool throw_notfound = true); + inline move_result to_next_first_multi(bool throw_notfound = true); + inline move_result to_next(bool throw_notfound = true); + inline move_result to_last(bool throw_notfound = true); + + inline move_result move(move_operation operation, const slice &key, + bool throw_notfound); + inline move_result find(const slice &key, bool throw_notfound = true); + inline move_result lower_bound(const slice &key, bool throw_notfound = true); + + inline move_result move(move_operation operation, const slice &key, + const slice &value, bool throw_notfound); + inline move_result find_multivalue(const slice &key, const slice &value, + bool throw_notfound = true); + inline move_result lower_bound_multivalue(const slice &key, + const slice &value, + bool throw_notfound = false); + + inline bool seek(const slice &key); + inline bool move(move_operation operation, slice &key, slice &value, + bool throw_notfound); + + /// \brief Return count of duplicates for current key. + inline size_t count_multivalue() const; + + inline bool eof() const; + inline bool on_first() const; + inline bool on_last() const; + inline ptrdiff_t estimate(slice key, slice value) const; + inline ptrdiff_t estimate(slice key) const; + inline ptrdiff_t estimate(move_operation operation) const; + + //---------------------------------------------------------------------------- + + /// \brief Renew/bind a cursor with a new transaction and previously used + /// key-value map handle. + inline void renew(::mdbx::txn &txn); + + /// \brief Bind/renew a cursor with a new transaction and specified key-value + /// map handle. + inline void bind(::mdbx::txn &txn, ::mdbx::map_handle map_handle); + + /// \brief Returns the cursor's transaction. + inline ::mdbx::txn txn() const; + inline map_handle map() const; + + inline operator ::mdbx::txn() const { return txn(); } + inline operator ::mdbx::map_handle() const { return map(); } + + inline MDBX_error_t put(const slice &key, slice *value, + MDBX_put_flags_t flags) noexcept; + inline void insert(const slice &key, slice value); + inline value_result try_insert(const slice &key, slice value); + inline slice insert_reserve(const slice &key, size_t value_length); + inline value_result try_insert_reserve(const slice &key, size_t value_length); + + inline void upsert(const slice &key, const slice &value); + inline slice upsert_reserve(const slice &key, size_t value_length); + + inline void update(const slice &key, const slice &value); + inline bool try_update(const slice &key, const slice &value); + inline slice update_reserve(const slice &key, size_t value_length); + inline value_result try_update_reserve(const slice &key, size_t value_length); + + inline bool erase(bool whole_multivalue = false); +}; + +/// \brief Managed cursor. +/// +/// As other managed classes, `cursor_managed` destroys the represented +/// underlying object from the own class destructor, but disallows copying and +/// assignment for instances. +/// +/// \copydetails MDBX_cursor +class LIBMDBX_API_TYPE cursor_managed : public cursor { + using inherited = cursor; + friend class txn; + /// delegated constructor for RAII + MDBX_CXX11_CONSTEXPR cursor_managed(MDBX_cursor *ptr) noexcept + : inherited(ptr) {} + +public: + /// \brief Creates a new managed cursor with underlying object. + inline cursor_managed(); + + /// \brief Explicitly closes the cursor. + void close(); + + cursor_managed(cursor_managed &&) = default; + cursor_managed &operator=(cursor_managed &&) = default; + cursor_managed(const cursor_managed &) = delete; + cursor_managed &operator=(const cursor_managed &) = delete; + ~cursor_managed() noexcept { ::mdbx_cursor_close(handle_); } +}; + +//------------------------------------------------------------------------------ + +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const slice &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const pair &); +template <class ALLOCATOR> +inline ::std::ostream &operator<<(::std::ostream &out, + const buffer<ALLOCATOR> &it) { + return (it.is_freestanding() + ? out << "buf-" << it.headroom() << "." << it.tailroom() + : out << "ref-") + << it.slice(); +} +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const env::geometry::size &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::geometry &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const env::operate_parameters &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::mode &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const env::durability &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const env::reclaiming_options &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const env::operate_options &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const env_managed::create_parameters &); + +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const MDBX_log_level_t &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, + const MDBX_debug_flags_t &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const error &); +inline ::std::ostream &operator<<(::std::ostream &out, + const MDBX_error_t &errcode) { + return out << error(errcode); +} + +//============================================================================== +// +// Inline body of the libmdbx C++ API (preliminary draft) +// + +MDBX_CXX11_CONSTEXPR const version_info &get_version() noexcept { + return ::mdbx_version; +} +MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept { + return ::mdbx_build; +} + +static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept { +#if defined(__cpp_lib_is_constant_evaluated) && \ + __cpp_lib_is_constant_evaluated >= 201811L + if (::std::is_constant_evaluated()) { + for (size_t i = 0; c_str; ++i) + if (!c_str[i]) + return i; + return 0; + } +#endif /* __cpp_lib_is_constant_evaluated >= 201811 */ +#if defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L + return c_str ? ::std::string_view(c_str).length() : 0; +#else + return c_str ? ::std::strlen(c_str) : 0; +#endif +} + +MDBX_CXX14_CONSTEXPR size_t check_length(size_t bytes) { + if (MDBX_UNLIKELY(bytes > size_t(MDBX_MAXDATASIZE))) + MDBX_CXX20_UNLIKELY throw_max_length_exceeded(); + return bytes; +} + +inline bool exception_thunk::is_clean() const noexcept { return !captured_; } + +inline void exception_thunk::capture() noexcept { + assert(is_clean()); + captured_ = ::std::current_exception(); +} + +inline void exception_thunk::rethrow_captured() const { + if (captured_) + MDBX_CXX20_UNLIKELY ::std::rethrow_exception(captured_); +} + +//------------------------------------------------------------------------------ + +MDBX_CXX11_CONSTEXPR error::error(MDBX_error_t error_code) noexcept + : code_(error_code) {} + +inline error &error::operator=(MDBX_error_t error_code) noexcept { + code_ = error_code; + return *this; +} + +MDBX_CXX11_CONSTEXPR bool operator==(const error &a, const error &b) noexcept { + return a.code_ == b.code_; +} + +MDBX_CXX11_CONSTEXPR bool operator!=(const error &a, const error &b) noexcept { + return !(a == b); +} + +MDBX_CXX11_CONSTEXPR bool error::is_success() const noexcept { + return code_ == MDBX_SUCCESS; +} + +MDBX_CXX11_CONSTEXPR bool error::is_result_true() const noexcept { + return code_ == MDBX_RESULT_FALSE; +} + +MDBX_CXX11_CONSTEXPR bool error::is_result_false() const noexcept { + return code_ == MDBX_RESULT_TRUE; +} + +MDBX_CXX11_CONSTEXPR bool error::is_failure() const noexcept { + return code_ != MDBX_SUCCESS && code_ != MDBX_RESULT_TRUE; +} + +MDBX_CXX11_CONSTEXPR MDBX_error_t error::code() const noexcept { return code_; } + +MDBX_CXX11_CONSTEXPR bool error::is_mdbx_error() const noexcept { + return (code() >= MDBX_FIRST_LMDB_ERRCODE && + code() <= MDBX_LAST_LMDB_ERRCODE) || + (code() >= MDBX_FIRST_ADDED_ERRCODE && + code() <= MDBX_LAST_ADDED_ERRCODE); +} + +inline void error::throw_exception(int error_code) { + const error trouble(static_cast<MDBX_error_t>(error_code)); + trouble.throw_exception(); +} + +inline void error::throw_on_failure() const { + if (MDBX_UNLIKELY(is_failure())) + MDBX_CXX20_UNLIKELY throw_exception(); +} + +inline void error::success_or_throw() const { + if (MDBX_UNLIKELY(!is_success())) + MDBX_CXX20_UNLIKELY throw_exception(); +} + +inline void error::success_or_throw(const exception_thunk &thunk) const { + assert(thunk.is_clean() || code() != MDBX_SUCCESS); + if (MDBX_UNLIKELY(!is_success())) { + MDBX_CXX20_UNLIKELY if (!thunk.is_clean()) thunk.rethrow_captured(); + else throw_exception(); + } +} + +inline void error::panic_on_failure(const char *context_where, + const char *func_who) const noexcept { + if (MDBX_UNLIKELY(is_failure())) + MDBX_CXX20_UNLIKELY panic(context_where, func_who); +} + +inline void error::success_or_panic(const char *context_where, + const char *func_who) const noexcept { + if (MDBX_UNLIKELY(!is_success())) + MDBX_CXX20_UNLIKELY panic(context_where, func_who); +} + +inline void error::throw_on_nullptr(const void *ptr, MDBX_error_t error_code) { + if (MDBX_UNLIKELY(ptr == nullptr)) + MDBX_CXX20_UNLIKELY error(error_code).throw_exception(); +} + +inline void error::throw_on_failure(int error_code) { + error rc(static_cast<MDBX_error_t>(error_code)); + rc.throw_on_failure(); +} + +inline void error::success_or_throw(MDBX_error_t error_code) { + error rc(error_code); + rc.success_or_throw(); +} + +inline bool error::boolean_or_throw(int error_code) { + switch (error_code) { + case MDBX_RESULT_FALSE: + return false; + case MDBX_RESULT_TRUE: + return true; + default: + MDBX_CXX20_UNLIKELY throw_exception(error_code); + } +} + +inline void error::success_or_throw(int error_code, + const exception_thunk &thunk) { + error rc(static_cast<MDBX_error_t>(error_code)); + rc.success_or_throw(thunk); +} + +inline void error::panic_on_failure(int error_code, const char *context_where, + const char *func_who) noexcept { + error rc(static_cast<MDBX_error_t>(error_code)); + rc.panic_on_failure(context_where, func_who); +} + +inline void error::success_or_panic(int error_code, const char *context_where, + const char *func_who) noexcept { + error rc(static_cast<MDBX_error_t>(error_code)); + rc.success_or_panic(context_where, func_who); +} + +//------------------------------------------------------------------------------ + +MDBX_CXX11_CONSTEXPR slice::slice() noexcept : ::MDBX_val({nullptr, 0}) {} + +MDBX_CXX14_CONSTEXPR slice::slice(const void *ptr, size_t bytes) + : ::MDBX_val({const_cast<void *>(ptr), check_length(bytes)}) {} + +MDBX_CXX14_CONSTEXPR slice::slice(const void *begin, const void *end) + : slice(begin, static_cast<const byte *>(end) - + static_cast<const byte *>(begin)) {} + +MDBX_CXX17_CONSTEXPR slice::slice(const char *c_str) + : slice(c_str, ::mdbx::strlen(c_str)) {} + +MDBX_CXX14_CONSTEXPR slice::slice(const MDBX_val &src) + : slice(src.iov_base, src.iov_len) {} + +inline slice::slice(MDBX_val &&src) : slice(src) { src.iov_base = nullptr; } + +inline slice::slice(slice &&src) noexcept : slice(src) { src.invalidate(); } + +inline slice &slice::assign(const void *ptr, size_t bytes) { + iov_base = const_cast<void *>(ptr); + iov_len = check_length(bytes); + return *this; +} + +inline slice &slice::assign(const slice &src) noexcept { + iov_base = src.iov_base; + iov_len = src.iov_len; + return *this; +} + +inline slice &slice::assign(const ::MDBX_val &src) { + return assign(src.iov_base, src.iov_len); +} + +slice &slice::assign(slice &&src) noexcept { + assign(src); + src.invalidate(); + return *this; +} + +inline slice &slice::assign(::MDBX_val &&src) { + assign(src.iov_base, src.iov_len); + src.iov_base = nullptr; + return *this; +} + +inline slice &slice::assign(const void *begin, const void *end) { + return assign(begin, static_cast<const byte *>(end) - + static_cast<const byte *>(begin)); +} + +inline slice &slice::assign(const char *c_str) { + return assign(c_str, ::mdbx::strlen(c_str)); +} + +inline slice &slice::operator=(slice &&src) noexcept { + return assign(::std::move(src)); +} + +inline slice &slice::operator=(::MDBX_val &&src) { + return assign(::std::move(src)); +} + +inline void slice::swap(slice &other) noexcept { + const auto temp = *this; + *this = other; + other = temp; +} + +MDBX_CXX11_CONSTEXPR const mdbx::byte *slice::byte_ptr() const noexcept { + return static_cast<const byte *>(iov_base); +} + +MDBX_CXX11_CONSTEXPR const char *slice::char_ptr() const noexcept { + return static_cast<const char *>(iov_base); +} + +MDBX_CXX11_CONSTEXPR const void *slice::data() const noexcept { + return iov_base; +} + +MDBX_CXX11_CONSTEXPR size_t slice::length() const noexcept { return iov_len; } + +MDBX_CXX11_CONSTEXPR bool slice::empty() const noexcept { + return length() == 0; +} + +MDBX_CXX11_CONSTEXPR bool slice::is_null() const noexcept { + return data() == nullptr; +} + +MDBX_CXX11_CONSTEXPR size_t slice::size() const noexcept { return length(); } + +MDBX_CXX11_CONSTEXPR slice::operator bool() const noexcept { + return !is_null(); +} + +inline void slice::invalidate() noexcept { iov_base = nullptr; } + +inline void slice::clear() noexcept { + iov_base = nullptr; + iov_len = 0; +} + +inline void slice::remove_prefix(size_t n) noexcept { + assert(n <= size()); + iov_base = static_cast<byte *>(iov_base) + n; + iov_len -= n; +} + +inline void slice::safe_remove_prefix(size_t n) { + if (MDBX_UNLIKELY(n > size())) + MDBX_CXX20_UNLIKELY throw_out_range(); + remove_prefix(n); +} + +inline void slice::remove_suffix(size_t n) noexcept { + assert(n <= size()); + iov_len -= n; +} + +inline void slice::safe_remove_suffix(size_t n) { + if (MDBX_UNLIKELY(n > size())) + MDBX_CXX20_UNLIKELY throw_out_range(); + remove_suffix(n); +} + +inline bool slice::starts_with(const slice &prefix) const noexcept { + return length() >= prefix.length() && + ::std::memcmp(data(), prefix.data(), prefix.length()) == 0; +} + +inline bool slice::ends_with(const slice &suffix) const noexcept { + return length() >= suffix.length() && + ::std::memcmp(byte_ptr() + length() - suffix.length(), suffix.data(), + suffix.length()) == 0; +} + +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t +slice::hash_value() const noexcept { + size_t h = length() * 3977471; + for (size_t i = 0; i < length(); ++i) + h = (h ^ static_cast<const uint8_t *>(data())[i]) * 1664525 + 1013904223; + return h ^ 3863194411 * (h >> 11); +} + +inline byte slice::operator[](size_t n) const noexcept { + assert(n < size()); + return byte_ptr()[n]; +} + +inline byte slice::at(size_t n) const { + if (MDBX_UNLIKELY(n >= size())) + MDBX_CXX20_UNLIKELY throw_out_range(); + return byte_ptr()[n]; +} + +inline slice slice::head(size_t n) const noexcept { + assert(n <= size()); + return slice(data(), n); +} + +inline slice slice::tail(size_t n) const noexcept { + assert(n <= size()); + return slice(char_ptr() + size() - n, n); +} + +inline slice slice::middle(size_t from, size_t n) const noexcept { + assert(from + n <= size()); + return slice(char_ptr() + from, n); +} + +inline slice slice::safe_head(size_t n) const { + if (MDBX_UNLIKELY(n > size())) + MDBX_CXX20_UNLIKELY throw_out_range(); + return head(n); +} + +inline slice slice::safe_tail(size_t n) const { + if (MDBX_UNLIKELY(n > size())) + MDBX_CXX20_UNLIKELY throw_out_range(); + return tail(n); +} + +inline slice slice::safe_middle(size_t from, size_t n) const { + if (MDBX_UNLIKELY(n > max_length)) + MDBX_CXX20_UNLIKELY throw_max_length_exceeded(); + if (MDBX_UNLIKELY(from + n > size())) + MDBX_CXX20_UNLIKELY throw_out_range(); + return middle(from, n); +} + +inline intptr_t slice::compare_fast(const slice &a, const slice &b) noexcept { + const intptr_t diff = a.length() - b.length(); + return diff ? diff + : (a.data() == b.data()) + ? 0 + : ::std::memcmp(a.data(), b.data(), a.length()); +} + +inline intptr_t slice::compare_lexicographically(const slice &a, + const slice &b) noexcept { + const intptr_t diff = + ::std::memcmp(a.data(), b.data(), ::std::min(a.length(), b.length())); + return diff ? diff : intptr_t(a.length() - b.length()); +} + +MDBX_NOTHROW_PURE_FUNCTION inline bool operator==(const slice &a, + const slice &b) noexcept { + return slice::compare_fast(a, b) == 0; +} + +MDBX_NOTHROW_PURE_FUNCTION inline bool operator<(const slice &a, + const slice &b) noexcept { + return slice::compare_lexicographically(a, b) < 0; +} + +MDBX_NOTHROW_PURE_FUNCTION inline bool operator>(const slice &a, + const slice &b) noexcept { + return slice::compare_lexicographically(a, b) > 0; +} + +MDBX_NOTHROW_PURE_FUNCTION inline bool operator<=(const slice &a, + const slice &b) noexcept { + return slice::compare_lexicographically(a, b) <= 0; +} + +MDBX_NOTHROW_PURE_FUNCTION inline bool operator>=(const slice &a, + const slice &b) noexcept { + return slice::compare_lexicographically(a, b) >= 0; +} + +MDBX_NOTHROW_PURE_FUNCTION inline bool operator!=(const slice &a, + const slice &b) noexcept { + return slice::compare_fast(a, b) != 0; +} + +template <class ALLOCATOR> +inline ::mdbx::string<ALLOCATOR> +slice::hex_encode(bool uppercase, const ALLOCATOR &allocator) const { + ::mdbx::string<ALLOCATOR> result(allocator); + if (MDBX_LIKELY(length() > 0)) { + result.reserve(to_hex_bytes()); + result.resize(to_hex(const_cast<char *>(result.data()), result.capacity()) - + result.data(), + uppercase); + } + return result; +} + +template <class ALLOCATOR> +inline ::mdbx::string<ALLOCATOR> +slice::hex_decode(const ALLOCATOR &allocator) const { + ::mdbx::string<ALLOCATOR> result(allocator); + if (MDBX_LIKELY(length() > 0)) { + result.reserve(from_hex_bytes()); + result.resize( + from_hex(static_cast<byte *>( + static_cast<void *>(const_cast<char *>(result.data()))), + result.capacity()) - + static_cast<const byte *>(static_cast<const void *>(result.data()))); + } + return result; +} + +template <class ALLOCATOR> +inline ::mdbx::string<ALLOCATOR> +slice::base58_encode(const ALLOCATOR &allocator) const { + ::mdbx::string<ALLOCATOR> result(allocator); + if (MDBX_LIKELY(length() > 0)) { + result.reserve(to_base58_bytes()); + result.resize( + to_base58(const_cast<char *>(result.data()), result.capacity()) - + result.data()); + } + return result; +} + +template <class ALLOCATOR> +inline ::mdbx::string<ALLOCATOR> +slice::base58_decode(const ALLOCATOR &allocator) const { + ::mdbx::string<ALLOCATOR> result(allocator); + if (MDBX_LIKELY(length() > 0)) { + result.reserve(from_base58_bytes()); + result.resize( + from_base58(static_cast<byte *>( + static_cast<void *>(const_cast<char *>(result.data()))), + result.capacity()) - + static_cast<const byte *>(static_cast<const void *>(result.data()))); + } + return result; +} + +template <class ALLOCATOR> +inline ::mdbx::string<ALLOCATOR> +slice::base64_encode(const ALLOCATOR &allocator) const { + ::mdbx::string<ALLOCATOR> result(allocator); + if (MDBX_LIKELY(length() > 0)) { + result.reserve(to_base64_bytes()); + result.resize( + to_base64(const_cast<char *>(result.data()), result.capacity()) - + result.data()); + } + return result; +} + +template <class ALLOCATOR> +inline ::mdbx::string<ALLOCATOR> +slice::base64_decode(const ALLOCATOR &allocator) const { + ::mdbx::string<ALLOCATOR> result(allocator); + if (MDBX_LIKELY(length() > 0)) { + result.reserve(from_base64_bytes()); + result.resize( + from_base64(static_cast<byte *>( + static_cast<void *>(const_cast<char *>(result.data()))), + result.capacity()) - + static_cast<const byte *>(static_cast<const void *>(result.data()))); + } + return result; +} + +//------------------------------------------------------------------------------ + +MDBX_CXX11_CONSTEXPR map_handle::info::info(map_handle::flags flags, + map_handle::state state) noexcept + : flags(flags), state(state) {} + +MDBX_CXX11_CONSTEXPR ::mdbx::key_mode +map_handle::info::key_mode() const noexcept { + return ::mdbx::key_mode(flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)); +} + +MDBX_CXX11_CONSTEXPR ::mdbx::value_mode +map_handle::info::value_mode() const noexcept { + return ::mdbx::value_mode(flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | + MDBX_DUPFIXED | MDBX_INTEGERDUP)); +} + +//------------------------------------------------------------------------------ + +MDBX_CXX11_CONSTEXPR env::env(MDBX_env *ptr) noexcept : handle_(ptr) {} + +inline env &env::operator=(env &&other) noexcept { + handle_ = other.handle_; + other.handle_ = nullptr; + return *this; +} + +inline env::env(env &&other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; +} + +inline env::~env() noexcept { +#ifndef NDEBUG + handle_ = reinterpret_cast<MDBX_env *>(uintptr_t(0xDeadBeef)); +#endif +} + +MDBX_CXX14_CONSTEXPR env::operator bool() const noexcept { + return handle_ != nullptr; +} + +MDBX_CXX14_CONSTEXPR env::operator const MDBX_env *() const { return handle_; } + +MDBX_CXX14_CONSTEXPR env::operator MDBX_env *() { return handle_; } + +MDBX_CXX11_CONSTEXPR bool operator==(const env &a, const env &b) noexcept { + return a.handle_ == b.handle_; +} + +MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, const env &b) noexcept { + return a.handle_ != b.handle_; +} + +inline env::geometry &env::geometry::make_fixed(intptr_t size) noexcept { + size_lower = size_now = size_upper = size; + growth_step = shrink_threshold = 0; + return *this; +} + +inline env::geometry &env::geometry::make_dynamic(intptr_t lower, + intptr_t upper) noexcept { + size_now = size_lower = lower; + size_upper = upper; + growth_step = shrink_threshold = default_value; + return *this; +} + +inline env::reclaiming_options env::operate_parameters::reclaiming_from_flags( + MDBX_env_flags_t flags) noexcept { + return reclaiming_options(flags); +} + +inline env::operate_options +env::operate_parameters::options_from_flags(MDBX_env_flags_t flags) noexcept { + return operate_options(flags); +} + +inline size_t env::limits::pagesize_min() noexcept { return MDBX_MIN_PAGESIZE; } + +inline size_t env::limits::pagesize_max() noexcept { return MDBX_MAX_PAGESIZE; } + +inline size_t env::limits::dbsize_min(intptr_t pagesize) { + const intptr_t result = mdbx_limits_dbsize_min(pagesize); + if (result < 0) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + return static_cast<size_t>(result); +} + +inline size_t env::limits::dbsize_max(intptr_t pagesize) { + const intptr_t result = mdbx_limits_dbsize_max(pagesize); + if (result < 0) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + return static_cast<size_t>(result); +} + +inline size_t env::limits::key_min(MDBX_db_flags_t flags) noexcept { + return (flags & MDBX_INTEGERKEY) ? 4 : 0; +} + +inline size_t env::limits::key_min(key_mode mode) noexcept { + return key_min(MDBX_db_flags_t(mode)); +} + +inline size_t env::limits::key_max(intptr_t pagesize, MDBX_db_flags_t flags) { + const intptr_t result = mdbx_limits_keysize_max(pagesize, flags); + if (result < 0) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + return static_cast<size_t>(result); +} + +inline size_t env::limits::key_max(intptr_t pagesize, key_mode mode) { + return key_max(pagesize, MDBX_db_flags_t(mode)); +} + +inline size_t env::limits::key_max(const env &env, MDBX_db_flags_t flags) { + const intptr_t result = mdbx_env_get_maxkeysize_ex(env, flags); + if (result < 0) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + return static_cast<size_t>(result); +} + +inline size_t env::limits::key_max(const env &env, key_mode mode) { + return key_max(env, MDBX_db_flags_t(mode)); +} + +inline size_t env::limits::value_min(MDBX_db_flags_t flags) noexcept { + return (flags & MDBX_INTEGERDUP) ? 4 : 0; +} + +inline size_t env::limits::value_min(value_mode mode) noexcept { + return value_min(MDBX_db_flags_t(mode)); +} + +inline size_t env::limits::value_max(intptr_t pagesize, MDBX_db_flags_t flags) { + const intptr_t result = mdbx_limits_valsize_max(pagesize, flags); + if (result < 0) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + return static_cast<size_t>(result); +} + +inline size_t env::limits::value_max(intptr_t pagesize, value_mode mode) { + return value_max(pagesize, MDBX_db_flags_t(mode)); +} + +inline size_t env::limits::value_max(const env &env, MDBX_db_flags_t flags) { + const intptr_t result = mdbx_env_get_maxvalsize_ex(env, flags); + if (result < 0) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + return static_cast<size_t>(result); +} + +inline size_t env::limits::value_max(const env &env, value_mode mode) { + return value_max(env, MDBX_db_flags_t(mode)); +} + +inline size_t env::limits::transaction_size_max(intptr_t pagesize) { + const intptr_t result = mdbx_limits_txnsize_max(pagesize); + if (result < 0) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); + return static_cast<size_t>(result); +} + +inline env::operate_parameters env::get_operation_parameters() const { + return env::operate_parameters(*this); +} + +inline env::mode env::get_mode() const { + return operate_parameters::mode_from_flags(get_flags()); +} + +inline env::durability env::get_durability() const { + return env::operate_parameters::durability_from_flags(get_flags()); +} + +inline env::reclaiming_options env::get_reclaiming() const { + return env::operate_parameters::reclaiming_from_flags(get_flags()); +} + +inline env::operate_options env::get_options() const { + return env::operate_parameters::options_from_flags(get_flags()); +} + +inline env::stat env::get_stat() const { + env::stat r; + error::success_or_throw(::mdbx_env_stat_ex(handle_, nullptr, &r, sizeof(r))); + return r; +} + +inline env::stat env::get_stat(const txn &txn) const { + env::stat r; + error::success_or_throw(::mdbx_env_stat_ex(handle_, txn, &r, sizeof(r))); + return r; +} + +inline env::info env::get_info() const { + env::info r; + error::success_or_throw(::mdbx_env_info_ex(handle_, nullptr, &r, sizeof(r))); + return r; +} + +inline env::info env::get_info(const txn &txn) const { + env::info r; + error::success_or_throw(::mdbx_env_info_ex(handle_, txn, &r, sizeof(r))); + return r; +} + +inline filehandle env::get_filehandle() const { + filehandle fd; + error::success_or_throw(::mdbx_env_get_fd(handle_, &fd)); + return fd; +} + +inline MDBX_env_flags_t env::get_flags() const { + unsigned bits; + error::success_or_throw(::mdbx_env_get_flags(handle_, &bits)); + return MDBX_env_flags_t(bits); +} + +inline unsigned env::max_readers() const { + unsigned r; + error::success_or_throw(::mdbx_env_get_maxreaders(handle_, &r)); + return r; +} + +inline unsigned env::max_maps() const { + unsigned r; + error::success_or_throw(::mdbx_env_get_maxdbs(handle_, &r)); + return r; +} + +inline void *env::get_context() const noexcept { + return mdbx_env_get_userctx(handle_); +} + +inline env &env::set_context(void *ptr) { + error::success_or_throw(::mdbx_env_set_userctx(handle_, ptr)); + return *this; +} + +inline env &env::set_sync_threshold(size_t bytes) { + error::success_or_throw(::mdbx_env_set_syncbytes(handle_, bytes)); + return *this; +} + +inline env &env::set_sync_period(unsigned seconds_16dot16) { + error::success_or_throw(::mdbx_env_set_syncperiod(handle_, seconds_16dot16)); + return *this; +} + +inline env &env::set_sync_period(double seconds) { + return set_sync_period(unsigned(seconds * 65536)); +} + +inline env &env::alter_flags(MDBX_env_flags_t flags, bool on_off) { + error::success_or_throw(::mdbx_env_set_flags(handle_, flags, on_off)); + return *this; +} + +inline env &env::set_geometry(const geometry &geo) { + error::success_or_throw(::mdbx_env_set_geometry( + handle_, geo.size_lower, geo.size_now, geo.size_upper, geo.growth_step, + geo.shrink_threshold, geo.pagesize)); + return *this; +} + +inline bool env::sync_to_disk(bool force, bool nonblock) { + const int err = ::mdbx_env_sync_ex(handle_, force, nonblock); + switch (err) { + case MDBX_SUCCESS /* flush done */: + case MDBX_RESULT_TRUE /* no data pending for flush to disk */: + return true; + case MDBX_BUSY /* the environment is used by other thread */: + return false; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline void env::close_map(const map_handle &handle) { + error::success_or_throw(::mdbx_dbi_close(*this, handle.dbi)); +} + +MDBX_CXX11_CONSTEXPR +env::reader_info::reader_info(int slot, mdbx_pid_t pid, mdbx_tid_t thread, + uint64_t txnid, uint64_t lag, size_t used, + size_t retained) noexcept + : slot(slot), pid(pid), thread(thread), transaction_id(txnid), + transaction_lag(lag), bytes_used(used), bytes_retained(retained) {} + +template <typename VISITOR> +inline int env::enumerate_readers(VISITOR &visitor) { + struct reader_visitor_thunk : public exception_thunk { + VISITOR &visitor_; + static int cb(void *ctx, int number, int slot, mdbx_pid_t pid, + mdbx_tid_t thread, uint64_t txnid, uint64_t lag, size_t used, + size_t retained) noexcept { + reader_visitor_thunk *thunk = static_cast<reader_visitor_thunk *>(ctx); + assert(thunk->is_clean()); + try { + const reader_info info(slot, pid, thread, txnid, lag, used, retained); + return loop_control(thunk->visitor_(info, number)); + } catch (... /* capture any exception to rethrow it over C code */) { + thunk->capture(); + return loop_control::exit_loop; + } + } + MDBX_CXX11_CONSTEXPR reader_visitor_thunk(VISITOR &visitor) noexcept + : visitor_(visitor) {} + }; + reader_visitor_thunk thunk(visitor); + const auto rc = ::mdbx_reader_list(*this, thunk.cb, &thunk); + thunk.rethrow_captured(); + return rc; +} + +inline unsigned env::check_readers() { + int dead_count; + error::throw_on_failure(::mdbx_reader_check(*this, &dead_count)); + assert(dead_count >= 0); + return static_cast<unsigned>(dead_count); +} + +inline env &env::set_HandleSlowReaders(MDBX_hsr_func *cb) { + error::success_or_throw(::mdbx_env_set_hsr(handle_, cb)); + return *this; +} + +inline MDBX_hsr_func *env::get_HandleSlowReaders() const noexcept { + return ::mdbx_env_get_hsr(handle_); +} + +inline txn_managed env::start_read() const { + ::MDBX_txn *ptr; + error::success_or_throw( + ::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY, &ptr)); + assert(ptr != nullptr); + return txn_managed(ptr); +} + +inline txn_managed env::prepare_read() const { + ::MDBX_txn *ptr; + error::success_or_throw( + ::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY_PREPARE, &ptr)); + assert(ptr != nullptr); + return txn_managed(ptr); +} + +inline txn_managed env::start_write(bool dont_wait) { + ::MDBX_txn *ptr; + error::success_or_throw(::mdbx_txn_begin( + handle_, nullptr, dont_wait ? MDBX_TXN_TRY : MDBX_TXN_READWRITE, &ptr)); + assert(ptr != nullptr); + return txn_managed(ptr); +} + +inline txn_managed env::try_start_write() { return start_write(true); } + +//------------------------------------------------------------------------------ + +MDBX_CXX11_CONSTEXPR txn::txn(MDBX_txn *ptr) noexcept : handle_(ptr) {} + +inline txn &txn::operator=(txn &&other) noexcept { + handle_ = other.handle_; + other.handle_ = nullptr; + return *this; +} + +inline txn::txn(txn &&other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; +} + +inline txn::~txn() noexcept { +#ifndef NDEBUG + handle_ = reinterpret_cast<MDBX_txn *>(uintptr_t(0xDeadBeef)); +#endif +} + +MDBX_CXX14_CONSTEXPR txn::operator bool() const noexcept { + return handle_ != nullptr; +} + +MDBX_CXX14_CONSTEXPR txn::operator const MDBX_txn *() const { return handle_; } + +MDBX_CXX14_CONSTEXPR txn::operator MDBX_txn *() { return handle_; } + +MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, const txn &b) noexcept { + return a.handle_ == b.handle_; +} + +MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, const txn &b) noexcept { + return a.handle_ != b.handle_; +} + +inline bool txn::is_dirty(const void *ptr) const { + int err = ::mdbx_is_dirty(handle_, ptr); + switch (err) { + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + case MDBX_RESULT_TRUE: + return true; + case MDBX_RESULT_FALSE: + return false; + } +} + +inline ::mdbx::env txn::env() const noexcept { return ::mdbx_txn_env(handle_); } + +inline MDBX_txn_flags_t txn::flags() const { + const int bits = mdbx_txn_flags(handle_); + error::throw_on_failure((bits != -1) ? MDBX_SUCCESS : MDBX_BAD_TXN); + return static_cast<MDBX_txn_flags_t>(bits); +} + +inline uint64_t txn::id() const { + const uint64_t txnid = mdbx_txn_id(handle_); + error::throw_on_failure(txnid ? MDBX_SUCCESS : MDBX_BAD_TXN); + return txnid; +} + +inline void txn::reset_reading() { + error::success_or_throw(::mdbx_txn_reset(handle_)); +} + +inline void txn::renew_reading() { + error::success_or_throw(::mdbx_txn_renew(handle_)); +} + +inline txn::info txn::get_info(bool scan_reader_lock_table) const { + txn::info r; + error::success_or_throw(::mdbx_txn_info(handle_, &r, scan_reader_lock_table)); + return r; +} + +inline cursor_managed txn::open_cursor(map_handle map) { + MDBX_cursor *ptr; + error::success_or_throw(::mdbx_cursor_open(handle_, map.dbi, &ptr)); + return cursor_managed(ptr); +} + +inline ::mdbx::map_handle +txn::open_map(const char *name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { + ::mdbx::map_handle map; + error::success_or_throw(::mdbx_dbi_open( + handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), + &map.dbi)); + assert(map.dbi != 0); + return map; +} + +inline ::mdbx::map_handle +txn::open_map(const ::std::string &name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { + return open_map(name.c_str(), key_mode, value_mode); +} + +inline ::mdbx::map_handle txn::create_map(const char *name, + const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) { + ::mdbx::map_handle map; + error::success_or_throw(::mdbx_dbi_open( + handle_, name, + MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), + &map.dbi)); + assert(map.dbi != 0); + return map; +} + +inline ::mdbx::map_handle txn::create_map(const ::std::string &name, + const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) { + return create_map(name.c_str(), key_mode, value_mode); +} + +inline void txn::drop_map(map_handle map) { + error::success_or_throw(::mdbx_drop(handle_, map.dbi, true)); +} + +inline bool txn::drop_map(const ::std::string &name, bool throw_if_absent) { + return drop_map(name.c_str(), throw_if_absent); +} + +inline void txn::clear_map(map_handle map) { + error::success_or_throw(::mdbx_drop(handle_, map.dbi, false)); +} + +inline bool txn::clear_map(const ::std::string &name, bool throw_if_absent) { + return clear_map(name.c_str(), throw_if_absent); +} + +inline txn::map_stat txn::get_map_stat(map_handle map) const { + txn::map_stat r; + error::success_or_throw(::mdbx_dbi_stat(handle_, map.dbi, &r, sizeof(r))); + return r; +} + +inline uint32_t txn::get_tree_deepmask(map_handle map) const { + uint32_t r; + error::success_or_throw(::mdbx_dbi_dupsort_depthmask(handle_, map.dbi, &r)); + return r; +} + +inline map_handle::info txn::get_handle_info(map_handle map) const { + unsigned flags, state; + error::success_or_throw( + ::mdbx_dbi_flags_ex(handle_, map.dbi, &flags, &state)); + return map_handle::info(MDBX_db_flags_t(flags), MDBX_dbi_state_t(state)); +} + +inline txn &txn::put_canary(const txn::canary &canary) { + error::success_or_throw(::mdbx_canary_put(handle_, &canary)); + return *this; +} + +inline txn::canary txn::get_canary() const { + txn::canary r; + error::success_or_throw(::mdbx_canary_get(handle_, &r)); + return r; +} + +inline uint64_t txn::sequence(map_handle map) const { + uint64_t result; + error::success_or_throw(::mdbx_dbi_sequence(handle_, map.dbi, &result, 0)); + return result; +} + +inline uint64_t txn::sequence(map_handle map, uint64_t increment) { + uint64_t result; + error::success_or_throw( + ::mdbx_dbi_sequence(handle_, map.dbi, &result, increment)); + return result; +} + +inline int txn::compare_keys(map_handle map, const slice &a, + const slice &b) const noexcept { + return ::mdbx_cmp(handle_, map.dbi, &a, &b); +} + +inline int txn::compare_values(map_handle map, const slice &a, + const slice &b) const noexcept { + return ::mdbx_dcmp(handle_, map.dbi, &a, &b); +} + +inline int txn::compare_keys(map_handle map, const pair &a, + const pair &b) const noexcept { + return compare_keys(map, a.key, b.key); +} + +inline int txn::compare_values(map_handle map, const pair &a, + const pair &b) const noexcept { + return compare_values(map, a.value, b.value); +} + +inline slice txn::get(map_handle map, const slice &key) const { + slice result; + error::success_or_throw(::mdbx_get(handle_, map.dbi, &key, &result)); + return result; +} + +inline slice txn::get(map_handle map, slice key, size_t &values_count) const { + slice result; + error::success_or_throw( + ::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count)); + return result; +} + +inline slice txn::get(map_handle map, const slice &key, + const slice &value_at_absence) const { + slice result; + const int err = ::mdbx_get(handle_, map.dbi, &key, &result); + switch (err) { + case MDBX_SUCCESS: + return result; + case MDBX_NOTFOUND: + return value_at_absence; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline slice txn::get(map_handle map, slice key, size_t &values_count, + const slice &value_at_absence) const { + slice result; + const int err = ::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count); + switch (err) { + case MDBX_SUCCESS: + return result; + case MDBX_NOTFOUND: + return value_at_absence; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline pair_result txn::get_equal_or_great(map_handle map, + const slice &key) const { + pair result(key, slice()); + bool exact = !error::boolean_or_throw( + ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value)); + return pair_result(result.key, result.value, exact); +} + +inline pair_result +txn::get_equal_or_great(map_handle map, const slice &key, + const slice &value_at_absence) const { + pair result{key, slice()}; + const int err = + ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value); + switch (err) { + case MDBX_SUCCESS: + return pair_result{result.key, result.value, true}; + case MDBX_RESULT_TRUE: + return pair_result{result.key, result.value, false}; + case MDBX_NOTFOUND: + return pair_result{key, value_at_absence, false}; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline MDBX_error_t txn::put(map_handle map, const slice &key, slice *value, + MDBX_put_flags_t flags) noexcept { + return MDBX_error_t(::mdbx_put(handle_, map.dbi, &key, value, flags)); +} + +inline void txn::put(map_handle map, const slice &key, slice value, + put_mode mode) noexcept { + error::success_or_throw(put(map, key, &value, MDBX_put_flags_t(mode))); +} + +inline void txn::insert(map_handle map, const slice &key, slice value) { + error::success_or_throw( + put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert))); +} + +inline value_result txn::try_insert(map_handle map, const slice &key, + slice value) { + const int err = + put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert)); + switch (err) { + case MDBX_SUCCESS: + return value_result{slice(), true}; + case MDBX_KEYEXIST: + return value_result{value, false}; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline slice txn::insert_reserve(map_handle map, const slice &key, + size_t value_length) { + slice result(nullptr, value_length); + error::success_or_throw( + put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert) | MDBX_RESERVE)); + return result; +} + +inline value_result txn::try_insert_reserve(map_handle map, const slice &key, + size_t value_length) { + slice result(nullptr, value_length); + const int err = + put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert) | MDBX_RESERVE); + switch (err) { + case MDBX_SUCCESS: + return value_result{result, true}; + case MDBX_KEYEXIST: + return value_result{result, false}; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline void txn::upsert(map_handle map, const slice &key, const slice &value) { + error::success_or_throw(put(map, key, const_cast<slice *>(&value), + MDBX_put_flags_t(put_mode::upsert))); +} + +inline slice txn::upsert_reserve(map_handle map, const slice &key, + size_t value_length) { + slice result(nullptr, value_length); + error::success_or_throw(put( + map, key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); + return result; +} + +inline void txn::update(map_handle map, const slice &key, const slice &value) { + error::success_or_throw(put(map, key, const_cast<slice *>(&value), + MDBX_put_flags_t(put_mode::update))); +} + +inline bool txn::try_update(map_handle map, const slice &key, + const slice &value) { + const int err = put(map, key, const_cast<slice *>(&value), + MDBX_put_flags_t(put_mode::update)); + switch (err) { + case MDBX_SUCCESS: + return true; + case MDBX_NOTFOUND: + return false; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline slice txn::update_reserve(map_handle map, const slice &key, + size_t value_length) { + slice result(nullptr, value_length); + error::success_or_throw(put( + map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); + return result; +} + +inline value_result txn::try_update_reserve(map_handle map, const slice &key, + size_t value_length) { + slice result(nullptr, value_length); + const int err = + put(map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); + switch (err) { + case MDBX_SUCCESS: + return value_result{result, true}; + case MDBX_NOTFOUND: + return value_result{slice(), false}; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline bool txn::erase(map_handle map, const slice &key) { + const int err = ::mdbx_del(handle_, map.dbi, &key, nullptr); + switch (err) { + case MDBX_SUCCESS: + return true; + case MDBX_NOTFOUND: + return false; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline bool txn::erase(map_handle map, const slice &key, const slice &value) { + const int err = ::mdbx_del(handle_, map.dbi, &key, &value); + switch (err) { + case MDBX_SUCCESS: + return true; + case MDBX_NOTFOUND: + return false; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline void txn::replace(map_handle map, const slice &key, slice old_value, + const slice &new_value) { + error::success_or_throw(::mdbx_replace_ex( + handle_, map.dbi, &key, const_cast<slice *>(&new_value), &old_value, + MDBX_CURRENT | MDBX_NOOVERWRITE, nullptr, nullptr)); +} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR> txn::extract(map_handle map, const slice &key, + const ALLOCATOR &allocator) { + typename buffer<ALLOCATOR>::data_preserver result(allocator); + error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, nullptr, + &result.slice_, MDBX_CURRENT, + result, &result), + result); + return result; +} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR> txn::replace(map_handle map, const slice &key, + const slice &new_value, + const ALLOCATOR &allocator) { + typename buffer<ALLOCATOR>::data_preserver result(allocator); + error::success_or_throw( + ::mdbx_replace_ex(handle_, map.dbi, &key, const_cast<slice *>(&new_value), + &result.slice_, MDBX_CURRENT, result, &result), + result); + return result; +} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR> txn::replace_reserve(map_handle map, const slice &key, + slice &new_value, + const ALLOCATOR &allocator) { + typename buffer<ALLOCATOR>::data_preserver result(allocator); + error::success_or_throw( + ::mdbx_replace_ex(handle_, map.dbi, &key, &new_value, &result.slice_, + MDBX_CURRENT | MDBX_RESERVE, result, &result), + result); + return result; +} + +inline void txn::append(map_handle map, const slice &key, const slice &value, + bool multivalue_order_preserved) { + error::success_or_throw(::mdbx_put( + handle_, map.dbi, const_cast<slice *>(&key), const_cast<slice *>(&value), + multivalue_order_preserved ? (MDBX_APPEND | MDBX_APPENDDUP) + : MDBX_APPEND)); +} + +inline size_t txn::put_multiple(map_handle map, const slice &key, + const size_t value_length, + const void *values_array, size_t values_count, + put_mode mode, bool allow_partial) { + MDBX_val args[2] = {{const_cast<void *>(values_array), value_length}, + {nullptr, values_count}}; + const int err = ::mdbx_put(handle_, map.dbi, const_cast<slice *>(&key), args, + MDBX_put_flags_t(mode) | MDBX_MULTIPLE); + switch (err) { + case MDBX_SUCCESS: + MDBX_CXX20_LIKELY break; + case MDBX_KEYEXIST: + if (allow_partial) + break; + mdbx_txn_break(handle_); + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } + return args[1].iov_len /* done item count */; +} + +inline ptrdiff_t txn::estimate(map_handle map, pair from, pair to) const { + ptrdiff_t result; + error::success_or_throw(mdbx_estimate_range( + handle_, map.dbi, &from.key, &from.value, &to.key, &to.value, &result)); + return result; +} + +inline ptrdiff_t txn::estimate(map_handle map, slice from, slice to) const { + ptrdiff_t result; + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, + &to, nullptr, &result)); + return result; +} + +inline ptrdiff_t txn::estimate_from_first(map_handle map, slice to) const { + ptrdiff_t result; + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, nullptr, + nullptr, &to, nullptr, &result)); + return result; +} + +inline ptrdiff_t txn::estimate_to_last(map_handle map, slice from) const { + ptrdiff_t result; + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, + nullptr, nullptr, &result)); + return result; +} + +//------------------------------------------------------------------------------ + +MDBX_CXX11_CONSTEXPR cursor::cursor(MDBX_cursor *ptr) noexcept : handle_(ptr) {} + +inline cursor &cursor::operator=(cursor &&other) noexcept { + handle_ = other.handle_; + other.handle_ = nullptr; + return *this; +} + +inline cursor::cursor(cursor &&other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; +} + +inline cursor::~cursor() noexcept { +#ifndef NDEBUG + handle_ = reinterpret_cast<MDBX_cursor *>(uintptr_t(0xDeadBeef)); +#endif +} + +MDBX_CXX14_CONSTEXPR cursor::operator bool() const noexcept { + return handle_ != nullptr; +} + +MDBX_CXX14_CONSTEXPR cursor::operator const MDBX_cursor *() const { + return handle_; +} + +MDBX_CXX14_CONSTEXPR cursor::operator MDBX_cursor *() { return handle_; } + +MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, + const cursor &b) noexcept { + return a.handle_ == b.handle_; +} + +MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, + const cursor &b) noexcept { + return a.handle_ != b.handle_; +} + +inline cursor::move_result::move_result(const cursor &cursor, + bool throw_notfound) + : pair_result(key, value, false) { + done = cursor.move(get_current, &key, &value, throw_notfound); +} + +inline cursor::move_result::move_result(cursor &cursor, + move_operation operation, + bool throw_notfound) + : pair_result(key, value, false) { + done = cursor.move(operation, &key, &value, throw_notfound); +} + +inline cursor::move_result::move_result(cursor &cursor, + move_operation operation, + const slice &key, bool throw_notfound) + : pair_result(key, slice(), false) { + this->done = cursor.move(operation, &this->key, &this->value, throw_notfound); +} + +inline cursor::move_result::move_result(cursor &cursor, + move_operation operation, + const slice &key, const slice &value, + bool throw_notfound) + : pair_result(key, value, false) { + this->done = cursor.move(operation, &this->key, &this->value, throw_notfound); +} + +inline bool cursor::move(move_operation operation, MDBX_val *key, + MDBX_val *value, bool throw_notfound) const { + const int err = + ::mdbx_cursor_get(handle_, key, value, MDBX_cursor_op(operation)); + switch (err) { + case MDBX_SUCCESS: + MDBX_CXX20_LIKELY return true; + case MDBX_NOTFOUND: + if (!throw_notfound) + return false; + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline ptrdiff_t cursor::estimate(move_operation operation, MDBX_val *key, + MDBX_val *value) const { + ptrdiff_t result; + error::success_or_throw(::mdbx_estimate_move( + *this, key, value, MDBX_cursor_op(operation), &result)); + return result; +} + +inline ptrdiff_t estimate(const cursor &from, const cursor &to) { + ptrdiff_t result; + error::success_or_throw(mdbx_estimate_distance(from, to, &result)); + return result; +} + +inline cursor::move_result cursor::move(move_operation operation, + bool throw_notfound) { + return move_result(*this, operation, throw_notfound); +} + +inline cursor::move_result cursor::to_first(bool throw_notfound) { + return move(first, throw_notfound); +} + +inline cursor::move_result cursor::to_previous(bool throw_notfound) { + return move(previous, throw_notfound); +} + +inline cursor::move_result cursor::to_previous_last_multi(bool throw_notfound) { + return move(multi_prevkey_lastvalue, throw_notfound); +} + +inline cursor::move_result cursor::to_current_first_multi(bool throw_notfound) { + return move(multi_currentkey_firstvalue, throw_notfound); +} + +inline cursor::move_result cursor::to_current_prev_multi(bool throw_notfound) { + return move(multi_currentkey_prevvalue, throw_notfound); +} + +inline cursor::move_result cursor::current(bool throw_notfound) const { + return move_result(*this, throw_notfound); +} + +inline cursor::move_result cursor::to_current_next_multi(bool throw_notfound) { + return move(multi_currentkey_nextvalue, throw_notfound); +} + +inline cursor::move_result cursor::to_current_last_multi(bool throw_notfound) { + return move(multi_currentkey_lastvalue, throw_notfound); +} + +inline cursor::move_result cursor::to_next_first_multi(bool throw_notfound) { + return move(multi_nextkey_firstvalue, throw_notfound); +} + +inline cursor::move_result cursor::to_next(bool throw_notfound) { + return move(next, throw_notfound); +} + +inline cursor::move_result cursor::to_last(bool throw_notfound) { + return move(last, throw_notfound); +} + +inline cursor::move_result cursor::move(move_operation operation, + const slice &key, bool throw_notfound) { + return move_result(*this, operation, key, throw_notfound); +} + +inline cursor::move_result cursor::find(const slice &key, bool throw_notfound) { + return move(key_exact, key, throw_notfound); +} + +inline cursor::move_result cursor::lower_bound(const slice &key, + bool throw_notfound) { + return move(key_lowerbound, key, throw_notfound); +} + +inline cursor::move_result cursor::move(move_operation operation, + const slice &key, const slice &value, + bool throw_notfound) { + return move_result(*this, operation, key, value, throw_notfound); +} + +inline cursor::move_result cursor::find_multivalue(const slice &key, + const slice &value, + bool throw_notfound) { + return move(key_exact, key, value, throw_notfound); +} + +inline cursor::move_result cursor::lower_bound_multivalue(const slice &key, + const slice &value, + bool throw_notfound) { + return move(multi_exactkey_lowerboundvalue, key, value, throw_notfound); +} + +inline bool cursor::seek(const slice &key) { + return move(find_key, const_cast<slice *>(&key), nullptr, false); +} + +inline bool cursor::move(move_operation operation, slice &key, slice &value, + bool throw_notfound) { + return move(operation, &key, &value, throw_notfound); +} + +inline size_t cursor::count_multivalue() const { + size_t result; + error::success_or_throw(::mdbx_cursor_count(*this, &result)); + return result; +} + +inline bool cursor::eof() const { + return error::boolean_or_throw(::mdbx_cursor_eof(*this)); +} + +inline bool cursor::on_first() const { + return error::boolean_or_throw(::mdbx_cursor_on_first(*this)); +} + +inline bool cursor::on_last() const { + return error::boolean_or_throw(::mdbx_cursor_on_last(*this)); +} + +inline ptrdiff_t cursor::estimate(slice key, slice value) const { + return estimate(multi_exactkey_lowerboundvalue, &key, &value); +} + +inline ptrdiff_t cursor::estimate(slice key) const { + return estimate(key_lowerbound, &key, nullptr); +} + +inline ptrdiff_t cursor::estimate(move_operation operation) const { + slice unused_key; + return estimate(operation, &unused_key, nullptr); +} + +inline void cursor::renew(::mdbx::txn &txn) { + error::success_or_throw(::mdbx_cursor_renew(txn, handle_)); +} + +inline void cursor::bind(::mdbx::txn &txn, ::mdbx::map_handle map_handle) { + error::success_or_throw(::mdbx_cursor_bind(txn, handle_, map_handle.dbi)); +} + +inline txn cursor::txn() const { + MDBX_txn *txn = ::mdbx_cursor_txn(handle_); + error::throw_on_nullptr(txn, MDBX_EINVAL); + return ::mdbx::txn(txn); +} + +inline map_handle cursor::map() const { + const MDBX_dbi dbi = ::mdbx_cursor_dbi(handle_); + if (MDBX_UNLIKELY(dbi > MDBX_MAX_DBI)) + error::throw_exception(MDBX_EINVAL); + return map_handle(dbi); +} + +inline MDBX_error_t cursor::put(const slice &key, slice *value, + MDBX_put_flags_t flags) noexcept { + return MDBX_error_t(::mdbx_cursor_put(handle_, &key, value, flags)); +} + +inline void cursor::insert(const slice &key, slice value) { + error::success_or_throw( + put(key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert))); +} + +inline value_result cursor::try_insert(const slice &key, slice value) { + const int err = + put(key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert)); + switch (err) { + case MDBX_SUCCESS: + return value_result{slice(), true}; + case MDBX_KEYEXIST: + return value_result{value, false}; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline slice cursor::insert_reserve(const slice &key, size_t value_length) { + slice result(nullptr, value_length); + error::success_or_throw( + put(key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert) | MDBX_RESERVE)); + return result; +} + +inline value_result cursor::try_insert_reserve(const slice &key, + size_t value_length) { + slice result(nullptr, value_length); + const int err = + put(key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert) | MDBX_RESERVE); + switch (err) { + case MDBX_SUCCESS: + return value_result{result, true}; + case MDBX_KEYEXIST: + return value_result{result, false}; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline void cursor::upsert(const slice &key, const slice &value) { + error::success_or_throw(put(key, const_cast<slice *>(&value), + MDBX_put_flags_t(put_mode::upsert))); +} + +inline slice cursor::upsert_reserve(const slice &key, size_t value_length) { + slice result(nullptr, value_length); + error::success_or_throw( + put(key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); + return result; +} + +inline void cursor::update(const slice &key, const slice &value) { + error::success_or_throw(put(key, const_cast<slice *>(&value), + MDBX_put_flags_t(put_mode::update))); +} + +inline bool cursor::try_update(const slice &key, const slice &value) { + const int err = + put(key, const_cast<slice *>(&value), MDBX_put_flags_t(put_mode::update)); + switch (err) { + case MDBX_SUCCESS: + return true; + case MDBX_NOTFOUND: + return false; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline slice cursor::update_reserve(const slice &key, size_t value_length) { + slice result(nullptr, value_length); + error::success_or_throw( + put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); + return result; +} + +inline value_result cursor::try_update_reserve(const slice &key, + size_t value_length) { + slice result(nullptr, value_length); + const int err = + put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); + switch (err) { + case MDBX_SUCCESS: + return value_result{result, true}; + case MDBX_NOTFOUND: + return value_result{slice(), false}; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +inline bool cursor::erase(bool whole_multivalue) { + const int err = ::mdbx_cursor_del(handle_, whole_multivalue ? MDBX_ALLDUPS + : MDBX_CURRENT); + switch (err) { + case MDBX_SUCCESS: + MDBX_CXX20_LIKELY return true; + case MDBX_NOTFOUND: + return false; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +//------------------------------------------------------------------------------ + +inline cursor_managed::cursor_managed() + : cursor_managed(::mdbx_cursor_create(nullptr)) { + if (MDBX_UNLIKELY(!handle_)) + MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_ENOMEM); +} + +//------------------------------------------------------------------------------ + +template <class ALLOCATOR> +inline buffer<ALLOCATOR>::buffer(const txn &txn, const ::mdbx::slice &src, + const ALLOCATOR &allocator) + : buffer(src, !txn.is_dirty(src.data()), allocator) {} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR>::buffer(size_t head_room, size_t tail_room, + const ALLOCATOR &allocator) + : silo_(allocator) { + if (MDBX_UNLIKELY(head_room > max_length || tail_room > max_length || + head_room + tail_room > max_length)) + throw_max_length_exceeded(); + silo_.reserve(head_room + tail_room); + silo_.append(head_room, '\0'); + slice_.iov_base = const_cast<char *>(silo_.data()); + assert(slice_.iov_len == 0); +} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR>::buffer(size_t capacity, const ALLOCATOR &allocator) + : silo_(allocator) { + silo_.reserve(check_length(capacity)); + slice_.iov_base = const_cast<char *>(silo_.data()); + assert(slice_.iov_len == 0); +} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR>::buffer(size_t head_room, const ::mdbx::slice &src, + size_t tail_room, const ALLOCATOR &allocator) + : silo_(allocator) { + if (MDBX_UNLIKELY(head_room > max_length || tail_room > max_length || + head_room + tail_room > max_length - slice_.length())) + throw_max_length_exceeded(); + silo_.reserve(head_room + src.length() + tail_room); + silo_.append(head_room, '\0'); + silo_.append(src.char_ptr(), src.length()); + slice_.iov_base = const_cast<char *>(silo_.data()); + slice_.iov_len = src.length(); +} + +template <class ALLOCATOR> +inline void buffer<ALLOCATOR>::reserve(size_t wanna_headroom, + size_t wanna_tailroom, + size_t shrink_threshold) { + if (MDBX_UNLIKELY( + wanna_headroom > max_length || wanna_tailroom > max_length || + wanna_headroom + wanna_tailroom > max_length - slice_.length())) + throw_max_length_exceeded(); + + wanna_headroom = std::min(std::max(headroom(), wanna_headroom), + wanna_headroom + shrink_threshold); + wanna_tailroom = std::min(std::max(tailroom(), wanna_tailroom), + wanna_tailroom + shrink_threshold); + const auto wanna_capacity = wanna_headroom + slice_.length() + wanna_tailroom; + if (is_reference() || slice_.empty()) { + silo_.reserve(wanna_capacity); + silo_.resize(wanna_headroom); + silo_.append(slice_.char_ptr(), slice_.length()); + } else { + const auto was_headroom = headroom(); + if (was_headroom > wanna_headroom) + silo_.erase(wanna_headroom, was_headroom - wanna_headroom); + silo_.reserve(wanna_capacity); + if (was_headroom < wanna_headroom) + silo_.insert(was_headroom, wanna_headroom - was_headroom, '\0'); + } + slice_.iov_base = const_cast<byte *>(silo_begin()) + wanna_headroom; + assert(headroom() >= wanna_headroom && + headroom() <= wanna_headroom + shrink_threshold); + assert(tailroom() >= wanna_tailroom && + tailroom() <= wanna_tailroom + shrink_threshold); +} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR> &buffer<ALLOCATOR>::append(const void *src, + size_t bytes) { + if (MDBX_UNLIKELY(tailroom() < check_length(bytes))) + reserve(0, bytes); + std::memcpy(static_cast<char *>(slice_.iov_base) + size(), src, bytes); + slice_.iov_len += bytes; + return *this; +} + +template <class ALLOCATOR> +inline buffer<ALLOCATOR> &buffer<ALLOCATOR>::add_header(const void *src, + size_t bytes) { + if (MDBX_UNLIKELY(headroom() < check_length(bytes))) + reserve(bytes, 0); + slice_.iov_base = + std::memcpy(static_cast<char *>(slice_.iov_base) - bytes, src, bytes); + slice_.iov_len += bytes; + return *this; +} + +template <class ALLOCATOR> +inline void buffer<ALLOCATOR>::swap(buffer &other) +#if defined(__cpp_noexcept_function_type) && \ + __cpp_noexcept_function_type >= 201510L + noexcept( + std::allocator_traits<ALLOCATOR>::propagate_on_container_swap::value +#if defined(__cpp_lib_allocator_traits_is_always_equal) && \ + __cpp_lib_allocator_traits_is_always_equal >= 201411L + || std::allocator_traits<ALLOCATOR>::is_always_equal::value +#endif /* __cpp_lib_allocator_traits_is_always_equal */ + ) +#endif /* __cpp_noexcept_function_type */ +{ + if /* checking the equality of allocators to avoid UB */ +#if defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606L + constexpr +#endif + (!std::allocator_traits<ALLOCATOR>::propagate_on_container_swap::value +#if defined(__cpp_lib_allocator_traits_is_always_equal) && \ + __cpp_lib_allocator_traits_is_always_equal >= 201411L + && !std::allocator_traits<ALLOCATOR>::is_always_equal::value +#endif /* __cpp_lib_allocator_traits_is_always_equal */ + ) { + if (MDBX_UNLIKELY(silo_.get_allocator() != other.silo_.get_allocator())) + throw std::bad_alloc(); + } + silo_.swap(other.silo_); + slice_.swap(other.slice_); +} + +template <class ALLOCATOR> +inline int buffer<ALLOCATOR>::data_preserver::callback(void *context, + MDBX_val *target, + const void *src, + size_t bytes) noexcept { + auto self = static_cast<data_preserver *>(context); + assert(self->is_clean()); + assert(&self->data.slice_ == target); + (void)target; + try { + self->data.assign(src, bytes, false); + return MDBX_RESULT_FALSE; + } catch (... /* capture any exception to rethrow it over C code */) { + self->capture(); + return MDBX_RESULT_TRUE; + } +} + +} // namespace mdbx + +//------------------------------------------------------------------------------ + +namespace std { + +inline string to_string(const mdbx::slice &value) { + ostringstream out; + out << value; + return out.str(); +} + +template <class ALLOCATOR> +inline string to_string(const mdbx::buffer<ALLOCATOR> &buffer) { + ostringstream out; + out << buffer; + return out.str(); +} + +inline string to_string(const mdbx::pair &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::env::geometry &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::env::operate_parameters &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::env::mode &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::env::durability &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::env::reclaiming_options &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::env::operate_options &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::env_managed::create_parameters &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const ::MDBX_log_level_t &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const ::MDBX_debug_flags_t &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const mdbx::error &value) { + ostringstream out; + out << value; + return out.str(); +} + +inline string to_string(const ::MDBX_error_t &errcode) { + return to_string(mdbx::error(errcode)); +} + +template <> struct hash<mdbx::slice> { + MDBX_CXX14_CONSTEXPR size_t + operator()(mdbx::slice const &slice) const noexcept { + return slice.hash_value(); + } +}; +} // namespace std + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +/// @} end of C++ API diff --git a/libs/libmdbx/src/mdbx_chk.c b/libs/libmdbx/src/mdbx_chk.c new file mode 100644 index 0000000000..120e4b4048 --- /dev/null +++ b/libs/libmdbx/src/mdbx_chk.c @@ -0,0 +1,4787 @@ +/* mdbx_chk.c - memory-mapped database check tool */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#define MDBX_BUILD_SOURCERY 3b5677a6062b714f1e138b0066c5590ee3c9ebf3bf8cfa3bb9503515ea0d1f02_v0_9_1_18_g1d31ebdc1c +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_FULL_VER < 190024234 + /* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual Studio 2015 Update 3). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for alignment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant aggregate initializer */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ + +#if defined(MDBX_TOOLS) +#undef MDBX_DEPRECATED +#define MDBX_DEPRECATED +#endif /* MDBX_TOOLS */ + +#include "mdbx.h" +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# else +# define __noinline +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(7, 0) && \ + (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +# define __fallthrough __attribute__((__fallthrough__)) +# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ + __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define __fallthrough [[clang::fallthrough]] +# else +# define __fallthrough +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline __maybe_unused const void* MDBX_PURE_FUNCTION + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include <valgrind/memcheck.h> +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include <sanitizer/asan_interface.h> +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template <typename T, size_t N> + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_STATIC_ASSERT) +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif defined(_MSC_VER) +# include <crtdbg.h> +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wnested-anon-types") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wnested-anon-types" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wnested-anon-types" +# else +# pragma warning disable "nested-anon-types" +# endif +#endif /* -Wnested-anon-types */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) && MDBX_AVOID_CRT +#define _NO_CRT_STDIO_INLINE +#endif +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include <assert.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +/* C11 stdalign.h */ +#if __has_include(<stdalign.h>) +#include <stdalign.h> +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif /* Apple OSX & iOS */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) +#include <sys/cdefs.h> +#include <sys/mount.h> +#include <sys/sysctl.h> +#include <sys/types.h> +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <vm/vm_param.h> +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include <uvm/uvm_param.h> +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#ifndef __MACH__ +#include <sys/vmmeter.h> +#endif +#else +#include <malloc.h> +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(_WIN32) || defined(_WIN64)) +#include <mntent.h> +#endif /* !Solaris */ +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) +#include <malloc_np.h> +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) +#include <malloc/malloc.h> +#endif /* MacOS */ + +#if defined(__MACH__) +#include <mach/host_info.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> +#include <uuid/uuid.h> +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include <linux/sysctl.h> +#include <sched.h> +#include <sys/sendfile.h> +#include <sys/statfs.h> +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include <utmpx.h> +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include <kstat.h> +#include <sys/mnttab.h> +/* On Solaris, it's easier to add a missing prototype rather than find a + * combination of #defines that break nothing. */ +__extern_C key_t ftok(const char *, int); +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <tlhelp32.h> +#include <windows.h> +#include <winnt.h> +#include <winternl.h> +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event[2]; +} mdbx_condpair_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <sys/file.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond[2]; +} mdbx_condpair_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +#ifdef __ANDROID_API__ +#include <android/log.h> +#if __ANDROID_API__ >= 21 +#include <sys/sendfile.h> +#endif +#endif /* Android */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) +#include <sys/stat.h> +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) +#include <sys/types.h> +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) +#include <sys/file.h> +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include <intrin.h> +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif /* __ia32__ */ +#if defined(__ia32__) +#include <cpuid.h> +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include <mbarrier.h> +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include <machine/sys/inline.h> +#elif defined(__IBMC__) && defined(__powerpc) +#include <atomic.h> +#elif defined(_AIX) +#include <builtins.h> +#include <sys/atomic_op.h> +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include <c_asm.h> +#include <machine/builtins.h> +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include <machine/inline.h> +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline size_t +mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; +#endif /* Windows */ + +#ifdef __cplusplus +extern void mdbx_osal_jitter(bool tiny); +#else +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +#if __has_include(<sys/cachectl.h>) +#include <sys/cachectl.h> +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include <sys/cachectl.h> +#endif + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC MDBX_PRINTF_ARGS(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +MDBX_INTERNAL_VAR bool + mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; +#endif /* Linux */ + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_NONE = 0, + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + const enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); + +enum mdbx_openfile_purpose { + MDBX_OPEN_DXB_READ = 0, + MDBX_OPEN_DXB_LAZY = 1, + MDBX_OPEN_DXB_DSYNC = 2, + MDBX_OPEN_LCK = 3, + MDBX_OPEN_COPY = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +#define MMAP_OPTION_TRUNCATE 1 +#define MMAP_OPTION_SEMAPHORE 2 +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const unsigned options); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna, const bool may_move); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { + mdbx_tid_t thunk; + STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); +#if defined(_WIN32) || defined(_WIN64) + thunk = GetCurrentThreadId(); +#else + thunk = pthread_self(); +#endif + return (uintptr_t)thunk; +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by argument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (especially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); +MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; + +static __inline bool mdbx_RunningUnderWine(void) { + return !mdbx_NtExtendSection; +} + +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, + LPCSTR lpValue, DWORD dwFlags, + LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData); +MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include <stdatomic.h> +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include <libkern/OSAtomic.h> +#else +#error FIXME atomic-ops +#endif + +#endif /* !__cplusplus */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/******************************************************************************* + ******************************************************************************* + ******************************************************************************* + * + * + * #### ##### ##### # #### # # #### + * # # # # # # # # ## # # + * # # # # # # # # # # # #### + * # # ##### # # # # # # # # + * # # # # # # # # ## # # + * #### # # # #### # # #### + * + * + */ + +/** \defgroup build_option Build options + * The libmdbx build options. + @{ */ + +#ifdef DOXYGEN +/* !!! Actually this is a fake definitions !!! + * !!! for documentation generation by Doxygen !!! */ + +/** Controls enabling of debugging features. + * + * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all, + * including logging and assertion controls. + * Logging level and corresponding debug flags changing + * by \ref mdbx_setup_debug() will not have effect. + * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging, + * assertions checking and internal audit). + * Simultaneously sets the default logging level + * to the `MDBX_DEBUG` value. + * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. + * + * \ingroup build_option */ +#define MDBX_DEBUG 0...7 + +/** Disables using of GNU libc extensions. */ +#define MDBX_DISABLE_GNU_SOURCE 0 or 1 + +#endif /* DOXYGEN */ + +/** Enables support for huge write-transactions */ +#ifndef MDBX_HUGE_TRANSACTIONS +#define MDBX_HUGE_TRANSACTIONS 0 +#endif /* MDBX_HUGE_TRANSACTIONS */ + +/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_DURABILITY 0 +/** Using fsync() with chance of data lost on power failure */ +#define MDBX_OSX_WANNA_SPEED 1 + +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED + * for OSX & iOS */ +#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ + +/** Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_ENV_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be omitted: + * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_ENV_CHECKPID 0 +#else +#define MDBX_ENV_CHECKPID 1 +#endif +#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_ENV_CHECKPID) +#else +#define MDBX_ENV_CHECKPID_CONFIG STRINGIFY(MDBX_ENV_CHECKPID) +#endif /* MDBX_ENV_CHECKPID */ + +/** Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +/** Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +//------------------------------------------------------------------------------ + +/** Win32 File Locking API for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_WIN32FILES -1 + +/** SystemV IPC semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_SYSV 5 + +/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX1988 1988 + +/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2001 2001 + +/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2008 2008 + +/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_BENAPHORE 1995 + +/** Advanced: Choices the locking implementation (autodetection by default). */ +#if defined(_WIN32) || defined(_WIN64) +#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES +#else +#ifndef MDBX_LOCKING +#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ + _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ + _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ + _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 +#else +#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 +#endif +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 +#else +#define MDBX_LOCKING MDBX_LOCKING_SYSV +#endif +#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) +#else +#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) +#endif /* MDBX_LOCKING */ +#endif /* !Windows */ + +/** Advanced: Using POSIX OFD-locks (autodetection by default). */ +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) && \ + !defined(__sun) /* OFD-lock are broken on Solaris */ +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/** Advanced: Using sendfile() syscall (autodetection by default). */ +#ifndef MDBX_USE_SENDFILE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) +#define MDBX_USE_SENDFILE 1 +#else +#define MDBX_USE_SENDFILE 0 +#endif +#endif /* MDBX_USE_SENDFILE */ + +/** Advanced: Using copy_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_COPYFILERANGE +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) +#define MDBX_USE_COPYFILERANGE 1 +#else +#define MDBX_USE_COPYFILERANGE 0 +#endif +#endif /* MDBX_USE_COPYFILERANGE */ + +/** Advanced: Using sync_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_SYNCFILERANGE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) +#define MDBX_USE_SYNCFILERANGE 1 +#else +#define MDBX_USE_SYNCFILERANGE 0 +#endif +#endif /* MDBX_USE_SYNCFILERANGE */ + +//------------------------------------------------------------------------------ + +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) || defined(DOXYGEN) +#define MDBX_CPU_WRITEBACK_INCOHERENT 0 +#else +#define MDBX_CPU_WRITEBACK_INCOHERENT 1 +#endif +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#ifdef __OpenBSD__ +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 +#else +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +/** @} end of build options */ +/******************************************************************************* + ******************************************************************************* + ******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) +#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) +#define INVALID_TXNID UINT64_MAX +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + MDBX_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + bin128_t mm_bootid; + +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + __anonymous_struct_extension__ struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + indx_t mp_ptrs[] /* dynamic size */; +#endif /* C99 */ +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES +#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) +typedef void mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t mdbx_ipclock_t; +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#define MDBX_CLOCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) +typedef sem_t mdbx_ipclock_t; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); +#endif /* MDBX_LOCKING */ + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Write transaction lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_wlock; +#endif /* MDBX_LOCKING > 0 */ + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Readeaders registration lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_rlock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[] /* dynamic size */; +#endif /* C99 */ +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + __anonymous_struct_extension__ struct { + pgno_t pgno; + MDBX_page *ptr; + }; + __anonymous_struct_extension__ struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#if MDBX_HUGE_TRANSACTIONS +#define MDBX_PNL_MAX \ + ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) +#else +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) +#endif /* MDBX_HUGE_TRANSACTIONS */ + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ + size_t md_klen_min, md_klen_max; /* min/max key length for the database */ + size_t md_vlen_min, + md_vlen_max; /* min/max value/data length for the database */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) +#define MDBX_TXN_RW_BEGIN_FLAGS \ + (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + +#define TXN_FLAGS \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ + MDBX_TXN_HAS_CHILD) + +#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ + ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ + MDBX_SHRINK_ALLOWED) +#error "Oops, some flags overlapped or wrong" +#endif + + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + + /* Transaction DBI Flags */ +#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ +#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ +#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ +#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ +#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ +#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* Array of flags for each DB */ + uint8_t *mt_dbistate; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + MDBX_canary mt_canary; + void *mt_userctx; /* User-settable context */ + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbistate for this database */ + uint8_t *mc_dbistate; + unsigned mc_snum; /* number of pushed pages */ + unsigned mc_top; /* index of top page, normally mc_snum-1 */ + + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + + /* Cursor checking flags. */ +#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ +#define C_UPDATING 0x200 /* update/rebalance pending */ +#define C_RETIRING 0x400 /* refs to child pages may be invalid */ +#define C_SKIPORD 0x800 /* don't check keys ordering */ + + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbistate for this Dup DB */ + uint8_t mx_dbistate; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + void *mc_userctx; /* User-settable context */ + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + uint32_t me_signature; + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + /* Legacy MDBX_MAPASYNC (prior v0.9) */ +#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) +#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) + uint32_t me_flags; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_lazy_fd me_dxb_mmap.fd + mdbx_filehandle_t me_dsync_fd; + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + unsigned me_psize; /* DB page size, inited from me_os_psize */ + uint8_t me_psize2log; /* log2 of DB page size */ + int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ + + /* write-txn lock */ +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING > 0 + mdbx_ipclock_t *me_wlock; +#endif /* MDBX_LOCKING > 0 */ + + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + unsigned me_branch_nodemax; /* max size of a branch-node */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ + struct { +#if MDBX_LOCKING > 0 + mdbx_ipclock_t wlock; +#endif /* MDBX_LOCKING > 0 */ + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +#ifndef __cplusplus +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +extern uint8_t mdbx_runtime_flags; +extern uint8_t mdbx_loglevel; +extern MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) + mdbx_debug_log(int level, const char *function, int line, const char *fmt, + ...) MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function, + int line, const char *fmt, + va_list args); + +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#endif /* MDBX_DEBUG */ + +#if !MDBX_DEBUG && defined(__ANDROID_API__) +#define mdbx_assert_fail(env, msg, func, line) \ + __android_log_assert(msg, "mdbx", "%s:%u", func, line) +#else +void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, + int line); +#endif + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ + +#if MDBX_CPU_WRITEBACK_INCOHERENT +#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() +#else +#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +static __maybe_unused __inline void +mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = + (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + mdbx_assert(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#if MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +#endif /* !__cplusplus */ + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) + +/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + }; +#endif /* __BYTE_ORDER__ */ + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + uint8_t mn_data[] /* key and data are appended here */; +#endif /* C99 */ +} MDBX_node; + +#define DB_PERSISTENT_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP) + +/* mdbx_dbi_open() flags */ +#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) + +#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define DB_INTERNAL_FLAGS DB_VALID + +#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif +#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused bool +is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define ENV_CHANGEABLE_FLAGS \ + (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ + MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) +#define ENV_CHANGELESS_FLAGS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) +#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) + +#if !defined(__cplusplus) || defined(__cpp_constexpr) +static __maybe_unused void static_checks(void) { + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, + "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & + (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, + "Oops, some flags overlapped or wrong"); +} +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +#ifdef __cplusplus +} +#endif + +typedef struct flagbit { + int bit; + const char *name; +} flagbit; + +const flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"}, + {MDBX_INTEGERKEY, "integerkey"}, + {MDBX_REVERSEKEY, "reversekey"}, + {MDBX_DUPFIXED, "dupfixed"}, + {MDBX_REVERSEDUP, "reversedup"}, + {MDBX_INTEGERDUP, "integerdup"}, + {0, nullptr}}; + +#if defined(_WIN32) || defined(_WIN64) +/* + * POSIX getopt for Windows + * + * AT&T Public License + * + * Code given out at the 1985 UNIFORUM conference in Dallas. + */ + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#endif /* _MSC_VER (warnings) */ + +#include <stdio.h> +#include <string.h> + +#ifdef _MSC_VER +#pragma warning(pop) +#endif +/*----------------------------------------------------------------------------*/ + +#ifndef NULL +#define NULL 0 +#endif + +#ifndef EOF +#define EOF (-1) +#endif + +int optind = 1; +int optopt; +char *optarg; + +int getopt(int argc, char *const argv[], const char *opts) { + static int sp = 1; + int c; + const char *cp; + + if (sp == 1) { + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return EOF; + else if (strcmp(argv[optind], "--") == 0) { + optind++; + return EOF; + } + } + optopt = c = argv[optind][sp]; + if (c == ':' || (cp = strchr(opts, c)) == NULL) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c); + if (argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return '?'; + } + if (*++cp == ':') { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", + c); + sp = 1; + return '?'; + } else + optarg = argv[optind++]; + sp = 1; + } else { + if (argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return c; +} + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = 1; + return true; +} + +static uint64_t GetMilliseconds(void) { + LARGE_INTEGER Counter, Frequency; + return (QueryPerformanceFrequency(&Frequency) && + QueryPerformanceCounter(&Counter)) + ? Counter.QuadPart * 1000ul / Frequency.QuadPart + : 0; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +#define EXIT_INTERRUPTED (EXIT_FAILURE + 4) +#define EXIT_FAILURE_SYS (EXIT_FAILURE + 3) +#define EXIT_FAILURE_MDBX (EXIT_FAILURE + 2) +#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1) +#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE + +typedef struct { + const char *name; + struct { + uint64_t branch, large_count, large_volume, leaf; + uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed; + uint64_t total, empty, other; + } pages; + uint64_t payload_bytes; + uint64_t lost_bytes; +} walk_dbi_t; + +struct { + short *pagemap; + uint64_t total_payload_bytes; + uint64_t pgcount; + walk_dbi_t + dbi[MDBX_MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1]; +} walk; + +#define dbi_free walk.dbi[FREE_DBI] +#define dbi_main walk.dbi[MAIN_DBI] +#define dbi_meta walk.dbi[CORE_DBS] + +int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE; +MDBX_env *env; +MDBX_txn *txn; +MDBX_envinfo envinfo; +MDBX_stat envstat; +size_t userdb_count, skipped_subdb; +uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages, + unused_pages, backed_pages; +unsigned verbose; +bool ignore_wrong_order, quiet, dont_traversal; +const char *only_subdb; +int stuck_meta = -1; + +struct problem { + struct problem *pr_next; + size_t count; + const char *caption; +}; + +struct problem *problems_list; +uint64_t total_problems; + +static void MDBX_PRINTF_ARGS(1, 2) print(const char *msg, ...) { + if (!quiet) { + va_list args; + + fflush(stderr); + va_start(args, msg); + vfprintf(stdout, msg, args); + va_end(args); + } +} + +static void va_log(MDBX_log_level_t level, const char *msg, va_list args) { + static const char *const prefixes[] = { + "!!!fatal: ", " ! " /* error */, " ~ " /* warning */, + " " /* notice */, " // " /* verbose */, " //// " /* debug */, + " ////// " /* trace */ + }; + + FILE *out = stdout; + if (level <= MDBX_LOG_ERROR) { + total_problems++; + out = stderr; + } + + if (!quiet && verbose + 1 >= (unsigned)level) { + fflush(nullptr); + fputs(prefixes[level], out); + vfprintf(out, msg, args); + if (msg[strlen(msg) - 1] != '\n') + fputc('\n', out); + fflush(nullptr); + } + + if (level == MDBX_LOG_FATAL) { + exit(EXIT_FAILURE_MDBX); + abort(); + } +} + +static void MDBX_PRINTF_ARGS(1, 2) error(const char *msg, ...) { + va_list args; + va_start(args, msg); + va_log(MDBX_LOG_ERROR, msg, args); + va_end(args); +} + +static void logger(MDBX_log_level_t level, const char *function, int line, + const char *msg, va_list args) { + (void)line; + (void)function; + if (level < MDBX_LOG_EXTRA) + va_log(level, msg, args); +} + +static int check_user_break(void) { + switch (user_break) { + case 0: + return MDBX_SUCCESS; + case 1: + print(" - interrupted by signal\n"); + fflush(nullptr); + user_break = 2; + } + return MDBX_EINTR; +} + +static void pagemap_cleanup(void) { + for (size_t i = CORE_DBS + /* account pseudo-entry for meta */ 1; + i < ARRAY_LENGTH(walk.dbi); ++i) { + if (walk.dbi[i].name) { + mdbx_free((void *)walk.dbi[i].name); + walk.dbi[i].name = nullptr; + } + } + + mdbx_free(walk.pagemap); + walk.pagemap = nullptr; +} + +static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) { + static walk_dbi_t *last; + + if (dbi_name == MDBX_PGWALK_MAIN) + return &dbi_main; + if (dbi_name == MDBX_PGWALK_GC) + return &dbi_free; + if (dbi_name == MDBX_PGWALK_META) + return &dbi_meta; + + if (last && strcmp(last->name, dbi_name) == 0) + return last; + + walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1; + for (; dbi < ARRAY_END(walk.dbi) && dbi->name; ++dbi) { + if (strcmp(dbi->name, dbi_name) == 0) + return last = dbi; + } + + if (verbose > 0 && !silent) { + print(" - found '%s' area\n", dbi_name); + fflush(nullptr); + } + + if (dbi == ARRAY_END(walk.dbi)) + return nullptr; + + dbi->name = mdbx_strdup(dbi_name); + return last = dbi; +} + +static void MDBX_PRINTF_ARGS(4, 5) + problem_add(const char *object, uint64_t entry_number, const char *msg, + const char *extra, ...) { + total_problems++; + + if (!quiet) { + int need_fflush = 0; + struct problem *p; + + for (p = problems_list; p; p = p->pr_next) + if (p->caption == msg) + break; + + if (!p) { + p = mdbx_calloc(1, sizeof(*p)); + p->caption = msg; + p->pr_next = problems_list; + problems_list = p; + need_fflush = 1; + } + + p->count++; + if (verbose > 1) { + print(" %s #%" PRIu64 ": %s", object, entry_number, msg); + if (extra) { + va_list args; + printf(" ("); + va_start(args, extra); + vfprintf(stdout, extra, args); + va_end(args); + printf(")"); + } + printf("\n"); + if (need_fflush) + fflush(nullptr); + } + } +} + +static struct problem *problems_push(void) { + struct problem *p = problems_list; + problems_list = nullptr; + return p; +} + +static size_t problems_pop(struct problem *list) { + size_t count = 0; + + if (problems_list) { + int i; + + print(" - problems: "); + for (i = 0; problems_list; ++i) { + struct problem *p = problems_list->pr_next; + count += problems_list->count; + print("%s%s (%" PRIuPTR ")", i ? ", " : "", problems_list->caption, + problems_list->count); + mdbx_free(problems_list); + problems_list = p; + } + print("\n"); + fflush(nullptr); + } + + problems_list = list; + return count; +} + +static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, + void *const ctx, const int deep, + const char *const dbi_name_or_tag, const size_t page_size, + const MDBX_page_type_t pagetype, const MDBX_error_t err, + const size_t nentries, const size_t payload_bytes, + const size_t header_bytes, const size_t unused_bytes) { + (void)ctx; + if (deep > 42) { + problem_add("deep", deep, "too large", nullptr); + return MDBX_CORRUPTED /* avoid infinite loop/recursion */; + } + + walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name_or_tag, false); + if (!dbi) + return MDBX_ENOMEM; + + const size_t page_bytes = payload_bytes + header_bytes + unused_bytes; + walk.pgcount += pgnumber; + + const char *pagetype_caption; + bool branch = false; + switch (pagetype) { + default: + problem_add("page", pgno, "unknown page-type", "type %u, deep %i", + (unsigned)pagetype, deep); + pagetype_caption = "unknown"; + dbi->pages.other += pgnumber; + break; + case MDBX_page_broken: + pagetype_caption = "broken"; + dbi->pages.other += pgnumber; + break; + case MDBX_subpage_broken: + pagetype_caption = "broken-subpage"; + break; + case MDBX_page_meta: + pagetype_caption = "meta"; + dbi->pages.other += pgnumber; + break; + case MDBX_page_large: + pagetype_caption = "large"; + dbi->pages.large_volume += pgnumber; + dbi->pages.large_count += 1; + break; + case MDBX_page_branch: + pagetype_caption = "branch"; + dbi->pages.branch += pgnumber; + branch = true; + break; + case MDBX_page_leaf: + pagetype_caption = "leaf"; + dbi->pages.leaf += pgnumber; + break; + case MDBX_page_dupfixed_leaf: + pagetype_caption = "leaf-dupfixed"; + dbi->pages.leaf_dupfixed += pgnumber; + break; + case MDBX_subpage_leaf: + pagetype_caption = "subleaf-dupsort"; + dbi->pages.subleaf_dupsort += 1; + break; + case MDBX_subpage_dupfixed_leaf: + pagetype_caption = "subleaf-dupfixed"; + dbi->pages.subleaf_dupfixed += 1; + break; + } + + if (pgnumber) { + if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi->name) == 0)) { + if (pgnumber == 1) + print(" %s-page %" PRIu64, pagetype_caption, pgno); + else + print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber); + print(" of %s: header %" PRIiPTR ", payload %" PRIiPTR + ", unused %" PRIiPTR ", deep %i\n", + dbi->name, header_bytes, payload_bytes, unused_bytes, deep); + } + + bool already_used = false; + for (unsigned n = 0; n < pgnumber; ++n) { + uint64_t spanpgno = pgno + n; + if (spanpgno >= alloc_pages) + problem_add("page", spanpgno, "wrong page-no", + "%s-page: %" PRIu64 " > %" PRIu64 ", deep %i", + pagetype_caption, spanpgno, alloc_pages, deep); + else if (walk.pagemap[spanpgno]) { + walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1]; + problem_add("page", spanpgno, + (branch && coll_dbi == dbi) ? "loop" : "already used", + "%s-page: by %s, deep %i", pagetype_caption, coll_dbi->name, + deep); + already_used = true; + } else { + walk.pagemap[spanpgno] = (short)(dbi - walk.dbi + 1); + dbi->pages.total += 1; + } + } + + if (already_used) + return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */ + : MDBX_SUCCESS; + } + + if (MDBX_IS_ERROR(err)) { + problem_add("page", pgno, "invalid/corrupted", "%s-page", pagetype_caption); + } else { + if (unused_bytes > page_size) + problem_add("page", pgno, "illegal unused-bytes", + "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0, + unused_bytes, envstat.ms_psize); + + if (header_bytes < (int)sizeof(long) || + (size_t)header_bytes >= envstat.ms_psize - sizeof(long)) + problem_add("page", pgno, "illegal header-length", + "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR, + pagetype_caption, sizeof(long), header_bytes, + envstat.ms_psize - sizeof(long)); + if (payload_bytes < 1) { + if (nentries > 1) { + problem_add("page", pgno, "zero size-of-entry", + "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries", + pagetype_caption, payload_bytes, nentries); + /* if ((size_t)header_bytes + unused_bytes < page_size) { + // LY: hush a misuse error + page_bytes = page_size; + } */ + } else { + problem_add("page", pgno, "empty", + "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR + " entries, deep %i", + pagetype_caption, payload_bytes, nentries, deep); + dbi->pages.empty += 1; + } + } + + if (pgnumber) { + if (page_bytes != page_size) { + problem_add("page", pgno, "misused", + "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR + "h + %" PRIuPTR "p + %" PRIuPTR "u), deep %i", + pagetype_caption, page_size, page_bytes, header_bytes, + payload_bytes, unused_bytes, deep); + if (page_size > page_bytes) + dbi->lost_bytes += page_size - page_bytes; + } else { + dbi->payload_bytes += payload_bytes + header_bytes; + walk.total_payload_bytes += payload_bytes + header_bytes; + } + } + } + + return check_user_break(); +} + +typedef int(visitor)(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data); +static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler, + bool silent); + +static int handle_userdb(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data) { + (void)record_number; + (void)key; + (void)data; + return check_user_break(); +} + +static int handle_freedb(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data) { + char *bad = ""; + pgno_t *iptr = data->iov_base; + + if (key->iov_len != sizeof(txnid_t)) + problem_add("entry", record_number, "wrong txn-id size", + "key-size %" PRIiPTR, key->iov_len); + else { + txnid_t txnid; + memcpy(&txnid, key->iov_base, sizeof(txnid)); + if (txnid < 1 || txnid > envinfo.mi_recent_txnid) + problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid); + else { + if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t)) + problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR, + data->iov_len); + size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0; + if (number < 1 || number > MDBX_PNL_MAX) + problem_add("entry", txnid, "wrong idl length", "%" PRIuPTR, number); + else if ((number + 1) * sizeof(pgno_t) > data->iov_len) { + problem_add("entry", txnid, "trimmed idl", + "%" PRIuSIZE " > %" PRIuSIZE " (corruption)", + (number + 1) * sizeof(pgno_t), data->iov_len); + number = data->iov_len / sizeof(pgno_t) - 1; + } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >= + /* LY: allow gap up to one page. it is ok + * and better than shink-and-retry inside mdbx_update_gc() */ + envstat.ms_psize) + problem_add("entry", txnid, "extra idl space", + "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", + (number + 1) * sizeof(pgno_t), data->iov_len); + + gc_pages += number; + if (envinfo.mi_latter_reader_txnid > txnid) + reclaimable_pages += number; + + pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : txn->mt_next_pgno; + pgno_t span = 1; + for (unsigned i = 0; i < number; ++i) { + if (check_user_break()) + return MDBX_EINTR; + const pgno_t pgno = iptr[i]; + if (pgno < NUM_METAS) + problem_add("entry", txnid, "wrong idl entry", + "pgno %" PRIaPGNO " < meta-pages %u", pgno, NUM_METAS); + else if (pgno >= backed_pages) + problem_add("entry", txnid, "wrong idl entry", + "pgno %" PRIaPGNO " > backed-pages %" PRIu64, pgno, + backed_pages); + else if (pgno >= alloc_pages) + problem_add("entry", txnid, "wrong idl entry", + "pgno %" PRIaPGNO " > alloc-pages %" PRIu64, pgno, + alloc_pages - 1); + else { + if (MDBX_PNL_DISORDERED(prev, pgno)) { + bad = " [bad sequence]"; + problem_add("entry", txnid, "bad sequence", + "%" PRIaPGNO " %c [%u].%" PRIaPGNO, prev, + (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), + i, pgno); + } + if (walk.pagemap) { + int idx = walk.pagemap[pgno]; + if (idx == 0) + walk.pagemap[pgno] = -1; + else if (idx > 0) + problem_add("page", pgno, "already used", "by %s", + walk.dbi[idx - 1].name); + else + problem_add("page", pgno, "already listed in GC", nullptr); + } + } + prev = pgno; + while (i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) + : pgno_sub(pgno, span))) + ++span; + } + if (verbose > 3 && !only_subdb) { + print(" transaction %" PRIaTXN ", %" PRIuPTR + " pages, maxspan %" PRIaPGNO "%s\n", + txnid, number, span, bad); + if (verbose > 4) { + for (unsigned i = 0; i < number; i += span) { + const pgno_t pgno = iptr[i]; + for (span = 1; + i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) + : pgno_sub(pgno, span)); + ++span) + ; + if (span > 1) { + print(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pgno, span); + } else + print(" %9" PRIaPGNO "\n", pgno); + } + } + } + } + } + + return check_user_break(); +} + +static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { + return (a->iov_len == b->iov_len && + memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) + ? 0 + : 1; +} + +static int handle_maindb(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data) { + char *name; + int rc; + size_t i; + + name = key->iov_base; + for (i = 0; i < key->iov_len; ++i) { + if (name[i] < ' ') + return handle_userdb(record_number, key, data); + } + + name = mdbx_malloc(key->iov_len + 1); + memcpy(name, key->iov_base, key->iov_len); + name[key->iov_len] = '\0'; + userdb_count++; + + rc = process_db(~0u, name, handle_userdb, false); + mdbx_free(name); + if (rc != MDBX_INCOMPATIBLE) + return rc; + + return handle_userdb(record_number, key, data); +} + +static const char *db_flags2keymode(unsigned flags) { + flags &= (MDBX_REVERSEKEY | MDBX_INTEGERKEY); + switch (flags) { + case 0: + return "usual"; + case MDBX_REVERSEKEY: + return "reserve"; + case MDBX_INTEGERKEY: + return "ordinal"; + case MDBX_REVERSEKEY | MDBX_INTEGERKEY: + return "msgpack"; + default: + assert(false); + __unreachable(); + } +} + +static const char *db_flags2valuemode(unsigned flags) { + flags &= (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP); + switch (flags) { + case 0: + return "single"; + case MDBX_DUPSORT: + return "multi"; + case MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_REVERSEDUP: + return "multi-reverse"; + case MDBX_DUPFIXED: + case MDBX_DUPSORT | MDBX_DUPFIXED: + return "multi-samelength"; + case MDBX_DUPFIXED | MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP: + return "multi-reverse-samelength"; + case MDBX_INTEGERDUP: + case MDBX_DUPSORT | MDBX_INTEGERDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP: + case MDBX_DUPFIXED | MDBX_INTEGERDUP: + return "multi-ordinal"; + case MDBX_INTEGERDUP | MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_REVERSEDUP: + return "multi-msgpack"; + case MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: + return "reserved"; + default: + assert(false); + __unreachable(); + } +} + +static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler, + bool silent) { + MDBX_cursor *mc; + MDBX_stat ms; + MDBX_val key, data; + MDBX_val prev_key, prev_data; + unsigned flags; + int rc, i; + struct problem *saved_list; + uint64_t problems_count; + + uint64_t record_count = 0, dups = 0; + uint64_t key_bytes = 0, data_bytes = 0; + + if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & mdbx_txn_flags(txn)) { + print(" ! abort processing '%s' due to a previous error\n", + dbi_name ? dbi_name : "@MAIN"); + return MDBX_BAD_TXN; + } + + if (dbi_handle == ~0u) { + rc = mdbx_dbi_open_ex( + txn, dbi_name, MDBX_DB_ACCEDE, &dbi_handle, + (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr, + (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr); + if (rc) { + if (!dbi_name || + rc != + MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ { + error("mdbx_open '%s' failed, error %d %s\n", + dbi_name ? dbi_name : "main", rc, mdbx_strerror(rc)); + } + return rc; + } + } + + if (dbi_handle >= CORE_DBS && dbi_name && only_subdb && + strcmp(only_subdb, dbi_name) != 0) { + if (verbose) { + print("Skip processing '%s'...\n", dbi_name); + fflush(nullptr); + } + skipped_subdb++; + return MDBX_SUCCESS; + } + + if (!silent && verbose) { + print("Processing '%s'...\n", dbi_name ? dbi_name : "@MAIN"); + fflush(nullptr); + } + + rc = mdbx_dbi_flags(txn, dbi_handle, &flags); + if (rc) { + error("mdbx_dbi_flags failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc; + } + + rc = mdbx_dbi_stat(txn, dbi_handle, &ms, sizeof(ms)); + if (rc) { + error("mdbx_dbi_stat failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc; + } + + if (!silent && verbose) { + print(" - key-value kind: %s-key => %s-value", db_flags2keymode(flags), + db_flags2valuemode(flags)); + if (verbose > 1) { + print(", flags:"); + if (!flags) + print(" none"); + else { + for (i = 0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + print(" %s", dbflags[i].name); + } + if (verbose > 2) + print(" (0x%02X), dbi-id %d", flags, dbi_handle); + } + print("\n"); + if (ms.ms_mod_txnid) + print(" - last modification txn#%" PRIu64 "\n", ms.ms_mod_txnid); + if (verbose > 1) { + print(" - page size %u, entries %" PRIu64 "\n", ms.ms_psize, + ms.ms_entries); + print(" - b-tree depth %u, pages: branch %" PRIu64 ", leaf %" PRIu64 + ", overflow %" PRIu64 "\n", + ms.ms_depth, ms.ms_branch_pages, ms.ms_leaf_pages, + ms.ms_overflow_pages); + } + } + + walk_dbi_t *dbi = (dbi_handle < CORE_DBS) + ? &walk.dbi[dbi_handle] + : pagemap_lookup_dbi(dbi_name, true); + if (!dbi) { + error("too many DBIs or out of memory\n"); + return MDBX_ENOMEM; + } + if (!dont_traversal) { + const uint64_t subtotal_pages = + ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages; + if (subtotal_pages != dbi->pages.total) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", + "subtotal", subtotal_pages, dbi->pages.total); + if (ms.ms_branch_pages != dbi->pages.branch) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch", + ms.ms_branch_pages, dbi->pages.branch); + const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed; + if (ms.ms_leaf_pages != allleaf_pages) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", + "all-leaf", ms.ms_leaf_pages, allleaf_pages); + if (ms.ms_overflow_pages != dbi->pages.large_volume) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", + "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume); + } + rc = mdbx_cursor_open(txn, dbi_handle, &mc); + if (rc) { + error("mdbx_cursor_open failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc; + } + + if (ignore_wrong_order) { /* for debugging with enabled assertions */ + mc->mc_flags |= C_SKIPORD; + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; + } + + const size_t maxkeysize = mdbx_env_get_maxkeysize_ex(env, flags); + saved_list = problems_push(); + prev_key.iov_base = nullptr; + prev_key.iov_len = 0; + prev_data.iov_base = nullptr; + prev_data.iov_len = 0; + rc = mdbx_cursor_get(mc, &key, &data, MDBX_FIRST); + while (rc == MDBX_SUCCESS) { + rc = check_user_break(); + if (rc) + goto bailout; + + bool bad_key = false; + if (key.iov_len > maxkeysize) { + problem_add("entry", record_count, "key length exceeds max-key-size", + "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize); + bad_key = true; + } else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) && + key.iov_len != sizeof(uint32_t)) { + problem_add("entry", record_count, "wrong key length", + "%" PRIuPTR " != 4or8", key.iov_len); + bad_key = true; + } + + bool bad_data = false; + if ((flags & MDBX_INTEGERDUP) && data.iov_len != sizeof(uint64_t) && + data.iov_len != sizeof(uint32_t)) { + problem_add("entry", record_count, "wrong data length", + "%" PRIuPTR " != 4or8", data.iov_len); + bad_data = true; + } + + if (prev_key.iov_base && !bad_data) { + if ((flags & MDBX_DUPFIXED) && prev_data.iov_len != data.iov_len) { + problem_add("entry", record_count, "different data length", + "%" PRIuPTR " != %" PRIuPTR, prev_data.iov_len, + data.iov_len); + bad_data = true; + } + + if (!bad_key) { + int cmp = mdbx_cmp(txn, dbi_handle, &key, &prev_key); + if (cmp == 0) { + ++dups; + if ((flags & MDBX_DUPSORT) == 0) { + problem_add("entry", record_count, "duplicated entries", nullptr); + if (data.iov_len == prev_data.iov_len && + memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0) { + problem_add("entry", record_count, "complete duplicate", nullptr); + } + } else if (!bad_data) { + cmp = mdbx_dcmp(txn, dbi_handle, &data, &prev_data); + if (cmp == 0) { + problem_add("entry", record_count, "complete duplicate", nullptr); + } else if (cmp < 0 && !ignore_wrong_order) { + problem_add("entry", record_count, "wrong order of multi-values", + nullptr); + } + } + } else if (cmp < 0 && !ignore_wrong_order) { + problem_add("entry", record_count, "wrong order of entries", nullptr); + } + } + } else if (verbose) { + if (flags & MDBX_INTEGERKEY) + print(" - fixed key-size %" PRIuPTR "\n", key.iov_len); + if (flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) + print(" - fixed data-size %" PRIuPTR "\n", data.iov_len); + } + + if (handler) { + rc = handler(record_count, &key, &data); + if (MDBX_IS_ERROR(rc)) + goto bailout; + } + + record_count++; + key_bytes += key.iov_len; + data_bytes += data.iov_len; + + if (!bad_key) + prev_key = key; + if (!bad_data) + prev_data = data; + rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT); + } + if (rc != MDBX_NOTFOUND) + error("mdbx_cursor_get failed, error %d %s\n", rc, mdbx_strerror(rc)); + else + rc = 0; + + if (record_count != ms.ms_entries) + problem_add("entry", record_count, "different number of entries", + "%" PRIu64 " != %" PRIu64, record_count, ms.ms_entries); +bailout: + problems_count = problems_pop(saved_list); + if (!silent && verbose) { + print(" - summary: %" PRIu64 " records, %" PRIu64 " dups, %" PRIu64 + " key's bytes, %" PRIu64 " data's " + "bytes, %" PRIu64 " problems\n", + record_count, dups, key_bytes, data_bytes, problems_count); + fflush(nullptr); + } + + mdbx_cursor_close(mc); + return (rc || problems_count) ? MDBX_RESULT_TRUE : MDBX_SUCCESS; +} + +static void usage(char *prog) { + fprintf(stderr, + "usage: %s [-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s subdb] " + "dbpath\n" + " -V\t\tprint version and exit\n" + " -v\t\tmore verbose, could be used multiple times\n" + " -q\t\tbe quiet\n" + " -c\t\tforce cooperative mode (don't try exclusive)\n" + " -w\t\twrite-mode checking\n" + " -d\t\tdisable page-by-page traversal of B-tree\n" + " -i\t\tignore wrong order errors (for custom comparators case)\n" + " -s subdb\tprocess a specific subdatabase only\n" + " -0|1|2\tforce using specific meta-page 0, or 2 for checking\n" + " -t\t\tturn to a specified meta-page on successful check\n" + " -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n", + prog); + exit(EXIT_INTERRUPTED); +} + +static __inline bool meta_ot(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, + uint64_t sign_b, const bool wanna_steady) { + if (txn_a == txn_b) + return SIGN_IS_STEADY(sign_b); + + if (wanna_steady && SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) + return SIGN_IS_STEADY(sign_b); + + return txn_a < txn_b; +} + +static __inline bool meta_eq(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, + uint64_t sign_b) { + if (txn_a != txn_b) + return false; + + if (SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) + return false; + + return true; +} + +static __inline int meta_recent(const bool wanna_steady) { + if (meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady)) + return meta_ot(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady) + ? 1 + : 2; + else + return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, wanna_steady) + ? 2 + : 0; +} + +static __inline int meta_tail(int head) { + switch (head) { + case 0: + return meta_ot(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) + ? 1 + : 2; + case 1: + return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) + ? 0 + : 2; + case 2: + return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, true) + ? 0 + : 1; + default: + assert(false); + return -1; + } +} + +static int meta_head(void) { return meta_recent(false); } + +void verbose_meta(int num, txnid_t txnid, uint64_t sign, uint64_t bootid_x, + uint64_t bootid_y) { + const bool have_bootid = (bootid_x | bootid_y) != 0; + const bool bootid_match = bootid_x == envinfo.mi_bootid.current.x && + bootid_y == envinfo.mi_bootid.current.y; + + print(" - meta-%d: ", num); + switch (sign) { + case MDBX_DATASIGN_NONE: + print("no-sync/legacy"); + break; + case MDBX_DATASIGN_WEAK: + print("weak-%s", bootid_match ? (have_bootid ? "intact (same boot-id)" + : "unknown (no boot-id") + : "dead"); + break; + default: + print("steady"); + break; + } + print(" txn#%" PRIu64, txnid); + + const int head = meta_head(); + if (num == head) + print(", head"); + else if (num == meta_tail(head)) + print(", tail"); + else + print(", stay"); + + if (stuck_meta >= 0) { + if (num == stuck_meta) + print(", forced for checking"); + } else if (txnid > envinfo.mi_recent_txnid && + (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE) + print(", rolled-back %" PRIu64 " (%" PRIu64 " >>> %" PRIu64 ")", + txnid - envinfo.mi_recent_txnid, txnid, envinfo.mi_recent_txnid); + print("\n"); +} + +static uint64_t get_meta_txnid(const unsigned meta_id) { + switch (meta_id) { + default: + assert(false); + error("unexpected meta_id %u\n", meta_id); + return 0; + case 0: + return envinfo.mi_meta0_txnid; + case 1: + return envinfo.mi_meta1_txnid; + case 2: + return envinfo.mi_meta2_txnid; + } +} + +static void print_size(const char *prefix, const uint64_t value, + const char *suffix) { + const char sf[] = + "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */ + double k = 1024.0; + size_t i; + for (i = 0; sf[i + 1] && value / k > 1000.0; ++i) + k *= 1024; + print("%s%" PRIu64 " (%.2f %cb)%s", prefix, value, value / k, sf[i], suffix); +} + +int main(int argc, char *argv[]) { + int rc; + char *prog = argv[0]; + char *envname; + int problems_maindb = 0, problems_freedb = 0, problems_meta = 0; + bool write_locked = false; + bool turn_meta = false; + bool force_turn_meta = false; + + double elapsed; +#if defined(_WIN32) || defined(_WIN64) + uint64_t timestamp_start, timestamp_finish; + timestamp_start = GetMilliseconds(); +#else + struct timespec timestamp_start, timestamp_finish; + if (clock_gettime(CLOCK_MONOTONIC, ×tamp_start)) { + rc = errno; + error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc)); + return EXIT_FAILURE_SYS; + } +#endif + + dbi_meta.name = "@META"; + dbi_free.name = "@GC"; + dbi_main.name = "@MAIN"; + atexit(pagemap_cleanup); + + if (argc < 2) + usage(prog); + + for (int i; (i = getopt(argc, argv, "012TVvqnwctdis:")) != EOF;) { + switch (i) { + case 'V': + printf("mdbx_chk version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'v': + verbose++; + break; + case '0': + stuck_meta = 0; + break; + case '1': + stuck_meta = 1; + break; + case '2': + stuck_meta = 2; + break; + case 't': + turn_meta = true; + break; + case 'T': + turn_meta = force_turn_meta = true; + quiet = false; + if (verbose < 2) + verbose = 2; + break; + case 'q': + quiet = true; + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 'w': + envflags &= ~MDBX_RDONLY; +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + /* Temporary `workaround` for OpenBSD kernel's flaw. + * See https://github.com/erthink/libmdbx/issues/67 */ + envflags |= MDBX_WRITEMAP; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + break; + case 'c': + envflags = (envflags & ~MDBX_EXCLUSIVE) | MDBX_ACCEDE; + break; + case 'd': + dont_traversal = true; + break; + case 's': + if (only_subdb && strcmp(only_subdb, optarg)) + usage(prog); + only_subdb = optarg; + break; + case 'i': + ignore_wrong_order = true; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + rc = MDBX_SUCCESS; + if (stuck_meta >= 0 && (envflags & MDBX_EXCLUSIVE) == 0) { + error("exclusive mode is required to using specific meta-page(%d) for " + "checking.\n", + stuck_meta); + rc = EXIT_INTERRUPTED; + } + if (turn_meta) { + if (stuck_meta < 0) { + error("meta-page must be specified (by -0, -1 or -2 options) to turn to " + "it.\n"); + rc = EXIT_INTERRUPTED; + } + if (envflags & MDBX_RDONLY) { + error("write-mode must be enabled to turn to the specified meta-page.\n"); + rc = EXIT_INTERRUPTED; + } + if (only_subdb || dont_traversal) { + error("whole database checking with tree-traversal are required to turn " + "to the specified meta-page.\n"); + rc = EXIT_INTERRUPTED; + } + } + if (rc) + exit(rc); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname, + (envflags & MDBX_RDONLY) ? "only" : "write"); + fflush(nullptr); + mdbx_setup_debug((verbose < MDBX_LOG_TRACE - 1) + ? (MDBX_log_level_t)(verbose + 1) + : MDBX_LOG_TRACE, + MDBX_DBG_LEGACY_OVERLAP, logger); + + rc = mdbx_env_create(&env); + if (rc) { + error("mdbx_env_create failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc < 0 ? EXIT_FAILURE_MDBX : EXIT_FAILURE_SYS; + } + + rc = mdbx_env_set_maxdbs(env, MDBX_MAX_DBI); + if (rc) { + error("mdbx_env_set_maxdbs failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + if (stuck_meta >= 0) { + rc = mdbx_env_open_for_recovery(env, envname, stuck_meta, + (envflags & MDBX_RDONLY) ? false : true); + } else { + rc = mdbx_env_open(env, envname, envflags, 0); + if ((envflags & MDBX_EXCLUSIVE) && + (rc == MDBX_BUSY || +#if defined(_WIN32) || defined(_WIN64) + rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION +#else + rc == EBUSY || rc == EAGAIN +#endif + )) { + envflags &= ~MDBX_EXCLUSIVE; + rc = mdbx_env_open(env, envname, envflags | MDBX_ACCEDE, 0); + } + } + + if (rc) { + error("mdbx_env_open failed, error %d %s\n", rc, mdbx_strerror(rc)); + if (rc == MDBX_WANNA_RECOVERY && (envflags & MDBX_RDONLY)) + print("Please run %s in the read-write mode (with '-w' option).\n", prog); + goto bailout; + } + if (verbose) + print(" - %s mode\n", + (envflags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative"); + + if ((envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0) { + rc = mdbx_txn_lock(env, false); + if (rc != MDBX_SUCCESS) { + error("mdbx_txn_lock failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + write_locked = true; + } + + rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); + if (rc) { + error("mdbx_txn_begin() failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + rc = mdbx_env_info_ex(env, txn, &envinfo, sizeof(envinfo)); + if (rc) { + error("mdbx_env_info failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + if (verbose) { + print(" - current boot-id "); + if (envinfo.mi_bootid.current.x | envinfo.mi_bootid.current.y) + print("%016" PRIx64 "-%016" PRIx64 "\n", envinfo.mi_bootid.current.x, + envinfo.mi_bootid.current.y); + else + print("unavailable\n"); + } + + rc = mdbx_env_stat_ex(env, txn, &envstat, sizeof(envstat)); + if (rc) { + error("mdbx_env_stat failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + mdbx_filehandle_t dxb_fd; + rc = mdbx_env_get_fd(env, &dxb_fd); + if (rc) { + error("mdbx_env_get_fd failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + uint64_t dxb_filesize = 0; +#if defined(_WIN32) || defined(_WIN64) + { + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(dxb_fd, &info)) + rc = GetLastError(); + else + dxb_filesize = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; + } +#else + { + struct stat st; + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + if (fstat(dxb_fd, &st)) + rc = errno; + else + dxb_filesize = st.st_size; + } +#endif + if (rc) { + error("mdbx_filesize failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + errno = 0; + const uint64_t dxbfile_pages = dxb_filesize / envinfo.mi_dxb_pagesize; + alloc_pages = txn->mt_next_pgno; + backed_pages = envinfo.mi_geo.current / envinfo.mi_dxb_pagesize; + if (backed_pages > dxbfile_pages) { + print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", + backed_pages, dxbfile_pages); + ++problems_meta; + } + if (dxbfile_pages < NUM_METAS) + print(" ! file-pages %" PRIu64 " < %u\n", dxbfile_pages, NUM_METAS); + if (backed_pages < NUM_METAS) + print(" ! backed-pages %" PRIu64 " < %u\n", backed_pages, NUM_METAS); + if (backed_pages < NUM_METAS || dxbfile_pages < NUM_METAS) + goto bailout; + if (backed_pages > MAX_PAGENO) { + print(" ! backed-pages %" PRIu64 " > max-pages %" PRIaPGNO "\n", + backed_pages, MAX_PAGENO); + ++problems_meta; + backed_pages = MAX_PAGENO; + } + + if ((envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { + if (backed_pages > dxbfile_pages) { + print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", + backed_pages, dxbfile_pages); + ++problems_meta; + backed_pages = dxbfile_pages; + } + if (alloc_pages > backed_pages) { + print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", + alloc_pages, backed_pages); + ++problems_meta; + alloc_pages = backed_pages; + } + } else { + /* LY: DB may be shrinked by writer down to the allocated pages. */ + if (alloc_pages > backed_pages) { + print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", + alloc_pages, backed_pages); + ++problems_meta; + alloc_pages = backed_pages; + } + if (alloc_pages > dxbfile_pages) { + print(" ! alloc-pages %" PRIu64 " > file-pages %" PRIu64 "\n", + alloc_pages, dxbfile_pages); + ++problems_meta; + alloc_pages = dxbfile_pages; + } + if (backed_pages > dxbfile_pages) + backed_pages = dxbfile_pages; + } + + if (verbose) { + print(" - pagesize %u (%u system), max keysize %d..%d" + ", max readers %u\n", + envinfo.mi_dxb_pagesize, envinfo.mi_sys_pagesize, + mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT), + mdbx_env_get_maxkeysize_ex(env, 0), envinfo.mi_maxreaders); + print_size(" - mapsize ", envinfo.mi_mapsize, "\n"); + if (envinfo.mi_geo.lower == envinfo.mi_geo.upper) + print_size(" - fixed datafile: ", envinfo.mi_geo.current, ""); + else { + print_size(" - dynamic datafile: ", envinfo.mi_geo.lower, ""); + print_size(" .. ", envinfo.mi_geo.upper, ", "); + print_size("+", envinfo.mi_geo.grow, ", "); + print_size("-", envinfo.mi_geo.shrink, "\n"); + print_size(" - current datafile: ", envinfo.mi_geo.current, ""); + } + printf(", %" PRIu64 " pages\n", + envinfo.mi_geo.current / envinfo.mi_dxb_pagesize); +#if defined(_WIN32) || defined(_WIN64) + if (envinfo.mi_geo.shrink && envinfo.mi_geo.current != envinfo.mi_geo.upper) + print( + " WARNING: Due Windows system limitations a " + "file couldn't\n be truncated while the database " + "is opened. So, the size\n database file " + "of may by large than the database itself,\n " + "until it will be closed or reopened in read-write mode.\n"); +#endif + print(" - transactions: recent %" PRIu64 ", latter reader %" PRIu64 + ", lag %" PRIi64 "\n", + envinfo.mi_recent_txnid, envinfo.mi_latter_reader_txnid, + envinfo.mi_recent_txnid - envinfo.mi_latter_reader_txnid); + + verbose_meta(0, envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_bootid.meta0.x, envinfo.mi_bootid.meta0.y); + verbose_meta(1, envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, + envinfo.mi_bootid.meta1.x, envinfo.mi_bootid.meta1.y); + verbose_meta(2, envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, + envinfo.mi_bootid.meta2.x, envinfo.mi_bootid.meta2.y); + } + + if (verbose > 1) + print(" - performs check for meta-pages clashes\n"); + if (meta_eq(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign)) { + print(" ! meta-%d and meta-%d are clashed\n", 0, 1); + ++problems_meta; + } + if (meta_eq(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign)) { + print(" ! meta-%d and meta-%d are clashed\n", 1, 2); + ++problems_meta; + } + if (meta_eq(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, + envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign)) { + print(" ! meta-%d and meta-%d are clashed\n", 2, 0); + ++problems_meta; + } + + const unsigned steady_meta_id = meta_recent(true); + const uint64_t steady_meta_txnid = get_meta_txnid(steady_meta_id); + const unsigned weak_meta_id = meta_recent(false); + const uint64_t weak_meta_txnid = get_meta_txnid(weak_meta_id); + if (envflags & MDBX_EXCLUSIVE) { + if (verbose > 1) + print(" - performs full check recent-txn-id with meta-pages\n"); + if (steady_meta_txnid != envinfo.mi_recent_txnid) { + print(" ! steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64 + " != %" PRIi64 ")\n", + steady_meta_id, steady_meta_txnid, envinfo.mi_recent_txnid); + ++problems_meta; + } + } else if (write_locked) { + if (verbose > 1) + print(" - performs lite check recent-txn-id with meta-pages (not a " + "monopolistic mode)\n"); + if (weak_meta_txnid != envinfo.mi_recent_txnid) { + print(" ! weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64 + " != %" PRIi64 ")\n", + weak_meta_id, weak_meta_txnid, envinfo.mi_recent_txnid); + ++problems_meta; + } + } else if (verbose) { + print(" - skip check recent-txn-id with meta-pages (monopolistic or " + "read-write mode only)\n"); + } + total_problems += problems_meta; + + if (!dont_traversal) { + struct problem *saved_list; + size_t traversal_problems; + uint64_t empty_pages, lost_bytes; + + print("Traversal b-tree by txn#%" PRIaTXN "...\n", txn->mt_txnid); + fflush(nullptr); + walk.pagemap = mdbx_calloc((size_t)backed_pages, sizeof(*walk.pagemap)); + if (!walk.pagemap) { + rc = errno ? errno : MDBX_ENOMEM; + error("calloc failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + saved_list = problems_push(); + rc = mdbx_env_pgwalk(txn, pgvisitor, nullptr, + true /* always skip key ordering checking to avoid + MDBX_CORRUPTED when using custom comparators */); + traversal_problems = problems_pop(saved_list); + + if (rc) { + if (rc != MDBX_EINTR || !check_user_break()) + error("mdbx_env_pgwalk failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + for (uint64_t n = 0; n < alloc_pages; ++n) + if (!walk.pagemap[n]) + unused_pages += 1; + + empty_pages = lost_bytes = 0; + for (walk_dbi_t *dbi = &dbi_main; dbi < ARRAY_END(walk.dbi) && dbi->name; + ++dbi) { + empty_pages += dbi->pages.empty; + lost_bytes += dbi->lost_bytes; + } + + if (verbose) { + uint64_t total_page_bytes = walk.pgcount * envstat.ms_psize; + print(" - pages: walked %" PRIu64 ", left/unused %" PRIu64 "\n", + walk.pgcount, unused_pages); + if (verbose > 1) { + for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; + ++dbi) { + print(" %s: subtotal %" PRIu64, dbi->name, dbi->pages.total); + if (dbi->pages.other && dbi->pages.other != dbi->pages.total) + print(", other %" PRIu64, dbi->pages.other); + if (dbi->pages.branch) + print(", branch %" PRIu64, dbi->pages.branch); + if (dbi->pages.large_count) + print(", large %" PRIu64, dbi->pages.large_count); + uint64_t all_leaf = dbi->pages.leaf + dbi->pages.leaf_dupfixed; + if (all_leaf) { + print(", leaf %" PRIu64, all_leaf); + if (verbose > 2 && + (dbi->pages.leaf_dupfixed | dbi->pages.subleaf_dupsort | + dbi->pages.subleaf_dupsort)) + print(" (usual %" PRIu64 ", sub-dupsort %" PRIu64 + ", dupfixed %" PRIu64 ", sub-dupfixed %" PRIu64 ")", + dbi->pages.leaf, dbi->pages.subleaf_dupsort, + dbi->pages.leaf_dupfixed, dbi->pages.subleaf_dupfixed); + } + print("\n"); + } + } + + if (verbose > 1) + print(" - usage: total %" PRIu64 " bytes, payload %" PRIu64 + " (%.1f%%), unused " + "%" PRIu64 " (%.1f%%)\n", + total_page_bytes, walk.total_payload_bytes, + walk.total_payload_bytes * 100.0 / total_page_bytes, + total_page_bytes - walk.total_payload_bytes, + (total_page_bytes - walk.total_payload_bytes) * 100.0 / + total_page_bytes); + if (verbose > 2) { + for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; + ++dbi) + if (dbi->pages.total) { + uint64_t dbi_bytes = dbi->pages.total * envstat.ms_psize; + print(" %s: subtotal %" PRIu64 " bytes (%.1f%%)," + " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)", + dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes, + dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes, + dbi_bytes - dbi->payload_bytes, + (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes); + if (dbi->pages.empty) + print(", %" PRIu64 " empty pages", dbi->pages.empty); + if (dbi->lost_bytes) + print(", %" PRIu64 " bytes lost", dbi->lost_bytes); + print("\n"); + } else + print(" %s: empty\n", dbi->name); + } + print(" - summary: average fill %.1f%%", + walk.total_payload_bytes * 100.0 / total_page_bytes); + if (empty_pages) + print(", %" PRIu64 " empty pages", empty_pages); + if (lost_bytes) + print(", %" PRIu64 " bytes lost", lost_bytes); + print(", %" PRIuPTR " problems\n", traversal_problems); + } + } else if (verbose) { + print("Skipping b-tree walk...\n"); + fflush(nullptr); + } + + if (!verbose) + print("Iterating DBIs...\n"); + problems_maindb = process_db(~0u, /* MAIN_DBI */ nullptr, nullptr, false); + problems_freedb = process_db(FREE_DBI, "@GC", handle_freedb, false); + + if (verbose) { + uint64_t value = envinfo.mi_mapsize / envstat.ms_psize; + double percent = value / 100.0; + print(" - space: %" PRIu64 " total pages", value); + print(", backed %" PRIu64 " (%.1f%%)", backed_pages, + backed_pages / percent); + print(", allocated %" PRIu64 " (%.1f%%)", alloc_pages, + alloc_pages / percent); + + if (verbose > 1) { + value = envinfo.mi_mapsize / envstat.ms_psize - alloc_pages; + print(", remained %" PRIu64 " (%.1f%%)", value, value / percent); + + value = alloc_pages - gc_pages; + print(", used %" PRIu64 " (%.1f%%)", value, value / percent); + + print(", gc %" PRIu64 " (%.1f%%)", gc_pages, gc_pages / percent); + + value = gc_pages - reclaimable_pages; + print(", detained %" PRIu64 " (%.1f%%)", value, value / percent); + + print(", reclaimable %" PRIu64 " (%.1f%%)", reclaimable_pages, + reclaimable_pages / percent); + } + + value = + envinfo.mi_mapsize / envstat.ms_psize - alloc_pages + reclaimable_pages; + print(", available %" PRIu64 " (%.1f%%)\n", value, value / percent); + } + + if (problems_maindb == 0 && problems_freedb == 0) { + if (!dont_traversal && + (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { + if (walk.pgcount != alloc_pages - gc_pages) { + error("used pages mismatch (%" PRIu64 "(walked) != %" PRIu64 + "(allocated - GC))\n", + walk.pgcount, alloc_pages - gc_pages); + } + if (unused_pages != gc_pages) { + error("gc pages mismatch (%" PRIu64 "(walked) != %" PRIu64 "(GC))\n", + unused_pages, gc_pages); + } + } else if (verbose) { + print(" - skip check used and gc pages (btree-traversal with " + "monopolistic or read-write mode only)\n"); + } + + if (!process_db(MAIN_DBI, nullptr, handle_maindb, true)) { + if (!userdb_count && verbose) + print(" - does not contain multiple databases\n"); + } + } + + if (rc == 0 && total_problems == 1 && problems_meta == 1 && !dont_traversal && + (envflags & MDBX_RDONLY) == 0 && !only_subdb && stuck_meta < 0 && + steady_meta_txnid < envinfo.mi_recent_txnid) { + print("Perform sync-to-disk for make steady checkpoint at txn-id #%" PRIi64 + "\n", + envinfo.mi_recent_txnid); + fflush(nullptr); + if (write_locked) { + mdbx_txn_unlock(env); + write_locked = false; + } + rc = mdbx_env_sync_ex(env, true, false); + if (rc != MDBX_SUCCESS) + error("mdbx_env_pgwalk failed, error %d %s\n", rc, mdbx_strerror(rc)); + else { + total_problems -= 1; + problems_meta -= 1; + } + } + + if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb && + (envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) { + const bool successful_check = (rc | total_problems | problems_meta) == 0; + if (successful_check || force_turn_meta) { + fflush(nullptr); + print(" = Performing turn to the specified meta-page (%d) due to %s!\n", + stuck_meta, + successful_check ? "successful check" : "the -T option was given"); + fflush(nullptr); + rc = mdbx_env_turn_for_recovery(env, stuck_meta); + if (rc != MDBX_SUCCESS) + error("mdbx_env_turn_for_recovery failed, error %d %s\n", rc, + mdbx_strerror(rc)); + } else { + print(" = Skipping turn to the specified meta-page (%d) due to " + "unsuccessful check!\n", + stuck_meta); + } + } + +bailout: + if (txn) + mdbx_txn_abort(txn); + if (write_locked) { + mdbx_txn_unlock(env); + write_locked = false; + } + if (env) { + const bool dont_sync = rc != 0 || total_problems; + mdbx_env_close_ex(env, dont_sync); + } + fflush(nullptr); + if (rc) { + if (rc < 0) + return user_break ? EXIT_INTERRUPTED : EXIT_FAILURE_SYS; + return EXIT_FAILURE_MDBX; + } + +#if defined(_WIN32) || defined(_WIN64) + timestamp_finish = GetMilliseconds(); + elapsed = (timestamp_finish - timestamp_start) * 1e-3; +#else + if (clock_gettime(CLOCK_MONOTONIC, ×tamp_finish)) { + rc = errno; + error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc)); + return EXIT_FAILURE_SYS; + } + elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec + + (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9; +#endif /* !WINDOWS */ + + if (total_problems) { + print("Total %" PRIu64 " error%s detected, elapsed %.3f seconds.\n", + total_problems, (total_problems > 1) ? "s are" : " is", elapsed); + if (problems_meta || problems_maindb || problems_freedb) + return EXIT_FAILURE_CHECK_MAJOR; + return EXIT_FAILURE_CHECK_MINOR; + } + print("No error is detected, elapsed %.3f seconds\n", elapsed); + return EXIT_SUCCESS; +} diff --git a/libs/libmdbx/src/mdbx_copy.c b/libs/libmdbx/src/mdbx_copy.c new file mode 100644 index 0000000000..71f6a05527 --- /dev/null +++ b/libs/libmdbx/src/mdbx_copy.c @@ -0,0 +1,3253 @@ +/* mdbx_copy.c - memory-mapped database backup tool */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#define MDBX_BUILD_SOURCERY 3b5677a6062b714f1e138b0066c5590ee3c9ebf3bf8cfa3bb9503515ea0d1f02_v0_9_1_18_g1d31ebdc1c +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_FULL_VER < 190024234 + /* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual Studio 2015 Update 3). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for alignment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant aggregate initializer */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ + +#if defined(MDBX_TOOLS) +#undef MDBX_DEPRECATED +#define MDBX_DEPRECATED +#endif /* MDBX_TOOLS */ + +#include "mdbx.h" +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# else +# define __noinline +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(7, 0) && \ + (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +# define __fallthrough __attribute__((__fallthrough__)) +# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ + __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define __fallthrough [[clang::fallthrough]] +# else +# define __fallthrough +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline __maybe_unused const void* MDBX_PURE_FUNCTION + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include <valgrind/memcheck.h> +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include <sanitizer/asan_interface.h> +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template <typename T, size_t N> + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_STATIC_ASSERT) +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif defined(_MSC_VER) +# include <crtdbg.h> +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wnested-anon-types") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wnested-anon-types" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wnested-anon-types" +# else +# pragma warning disable "nested-anon-types" +# endif +#endif /* -Wnested-anon-types */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) && MDBX_AVOID_CRT +#define _NO_CRT_STDIO_INLINE +#endif +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include <assert.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +/* C11 stdalign.h */ +#if __has_include(<stdalign.h>) +#include <stdalign.h> +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif /* Apple OSX & iOS */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) +#include <sys/cdefs.h> +#include <sys/mount.h> +#include <sys/sysctl.h> +#include <sys/types.h> +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <vm/vm_param.h> +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include <uvm/uvm_param.h> +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#ifndef __MACH__ +#include <sys/vmmeter.h> +#endif +#else +#include <malloc.h> +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(_WIN32) || defined(_WIN64)) +#include <mntent.h> +#endif /* !Solaris */ +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) +#include <malloc_np.h> +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) +#include <malloc/malloc.h> +#endif /* MacOS */ + +#if defined(__MACH__) +#include <mach/host_info.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> +#include <uuid/uuid.h> +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include <linux/sysctl.h> +#include <sched.h> +#include <sys/sendfile.h> +#include <sys/statfs.h> +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include <utmpx.h> +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include <kstat.h> +#include <sys/mnttab.h> +/* On Solaris, it's easier to add a missing prototype rather than find a + * combination of #defines that break nothing. */ +__extern_C key_t ftok(const char *, int); +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <tlhelp32.h> +#include <windows.h> +#include <winnt.h> +#include <winternl.h> +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event[2]; +} mdbx_condpair_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <sys/file.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond[2]; +} mdbx_condpair_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +#ifdef __ANDROID_API__ +#include <android/log.h> +#if __ANDROID_API__ >= 21 +#include <sys/sendfile.h> +#endif +#endif /* Android */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) +#include <sys/stat.h> +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) +#include <sys/types.h> +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) +#include <sys/file.h> +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include <intrin.h> +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif /* __ia32__ */ +#if defined(__ia32__) +#include <cpuid.h> +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include <mbarrier.h> +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include <machine/sys/inline.h> +#elif defined(__IBMC__) && defined(__powerpc) +#include <atomic.h> +#elif defined(_AIX) +#include <builtins.h> +#include <sys/atomic_op.h> +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include <c_asm.h> +#include <machine/builtins.h> +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include <machine/inline.h> +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline size_t +mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; +#endif /* Windows */ + +#ifdef __cplusplus +extern void mdbx_osal_jitter(bool tiny); +#else +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +#if __has_include(<sys/cachectl.h>) +#include <sys/cachectl.h> +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include <sys/cachectl.h> +#endif + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC MDBX_PRINTF_ARGS(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +MDBX_INTERNAL_VAR bool + mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; +#endif /* Linux */ + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_NONE = 0, + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + const enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); + +enum mdbx_openfile_purpose { + MDBX_OPEN_DXB_READ = 0, + MDBX_OPEN_DXB_LAZY = 1, + MDBX_OPEN_DXB_DSYNC = 2, + MDBX_OPEN_LCK = 3, + MDBX_OPEN_COPY = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +#define MMAP_OPTION_TRUNCATE 1 +#define MMAP_OPTION_SEMAPHORE 2 +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const unsigned options); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna, const bool may_move); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { + mdbx_tid_t thunk; + STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); +#if defined(_WIN32) || defined(_WIN64) + thunk = GetCurrentThreadId(); +#else + thunk = pthread_self(); +#endif + return (uintptr_t)thunk; +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by argument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (especially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); +MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; + +static __inline bool mdbx_RunningUnderWine(void) { + return !mdbx_NtExtendSection; +} + +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, + LPCSTR lpValue, DWORD dwFlags, + LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData); +MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include <stdatomic.h> +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include <libkern/OSAtomic.h> +#else +#error FIXME atomic-ops +#endif + +#endif /* !__cplusplus */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/******************************************************************************* + ******************************************************************************* + ******************************************************************************* + * + * + * #### ##### ##### # #### # # #### + * # # # # # # # # ## # # + * # # # # # # # # # # # #### + * # # ##### # # # # # # # # + * # # # # # # # # ## # # + * #### # # # #### # # #### + * + * + */ + +/** \defgroup build_option Build options + * The libmdbx build options. + @{ */ + +#ifdef DOXYGEN +/* !!! Actually this is a fake definitions !!! + * !!! for documentation generation by Doxygen !!! */ + +/** Controls enabling of debugging features. + * + * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all, + * including logging and assertion controls. + * Logging level and corresponding debug flags changing + * by \ref mdbx_setup_debug() will not have effect. + * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging, + * assertions checking and internal audit). + * Simultaneously sets the default logging level + * to the `MDBX_DEBUG` value. + * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. + * + * \ingroup build_option */ +#define MDBX_DEBUG 0...7 + +/** Disables using of GNU libc extensions. */ +#define MDBX_DISABLE_GNU_SOURCE 0 or 1 + +#endif /* DOXYGEN */ + +/** Enables support for huge write-transactions */ +#ifndef MDBX_HUGE_TRANSACTIONS +#define MDBX_HUGE_TRANSACTIONS 0 +#endif /* MDBX_HUGE_TRANSACTIONS */ + +/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_DURABILITY 0 +/** Using fsync() with chance of data lost on power failure */ +#define MDBX_OSX_WANNA_SPEED 1 + +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED + * for OSX & iOS */ +#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ + +/** Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_ENV_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be omitted: + * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_ENV_CHECKPID 0 +#else +#define MDBX_ENV_CHECKPID 1 +#endif +#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_ENV_CHECKPID) +#else +#define MDBX_ENV_CHECKPID_CONFIG STRINGIFY(MDBX_ENV_CHECKPID) +#endif /* MDBX_ENV_CHECKPID */ + +/** Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +/** Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +//------------------------------------------------------------------------------ + +/** Win32 File Locking API for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_WIN32FILES -1 + +/** SystemV IPC semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_SYSV 5 + +/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX1988 1988 + +/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2001 2001 + +/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2008 2008 + +/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_BENAPHORE 1995 + +/** Advanced: Choices the locking implementation (autodetection by default). */ +#if defined(_WIN32) || defined(_WIN64) +#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES +#else +#ifndef MDBX_LOCKING +#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ + _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ + _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ + _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 +#else +#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 +#endif +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 +#else +#define MDBX_LOCKING MDBX_LOCKING_SYSV +#endif +#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) +#else +#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) +#endif /* MDBX_LOCKING */ +#endif /* !Windows */ + +/** Advanced: Using POSIX OFD-locks (autodetection by default). */ +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) && \ + !defined(__sun) /* OFD-lock are broken on Solaris */ +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/** Advanced: Using sendfile() syscall (autodetection by default). */ +#ifndef MDBX_USE_SENDFILE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) +#define MDBX_USE_SENDFILE 1 +#else +#define MDBX_USE_SENDFILE 0 +#endif +#endif /* MDBX_USE_SENDFILE */ + +/** Advanced: Using copy_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_COPYFILERANGE +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) +#define MDBX_USE_COPYFILERANGE 1 +#else +#define MDBX_USE_COPYFILERANGE 0 +#endif +#endif /* MDBX_USE_COPYFILERANGE */ + +/** Advanced: Using sync_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_SYNCFILERANGE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) +#define MDBX_USE_SYNCFILERANGE 1 +#else +#define MDBX_USE_SYNCFILERANGE 0 +#endif +#endif /* MDBX_USE_SYNCFILERANGE */ + +//------------------------------------------------------------------------------ + +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) || defined(DOXYGEN) +#define MDBX_CPU_WRITEBACK_INCOHERENT 0 +#else +#define MDBX_CPU_WRITEBACK_INCOHERENT 1 +#endif +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#ifdef __OpenBSD__ +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 +#else +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +/** @} end of build options */ +/******************************************************************************* + ******************************************************************************* + ******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) +#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) +#define INVALID_TXNID UINT64_MAX +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + MDBX_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + bin128_t mm_bootid; + +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + __anonymous_struct_extension__ struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + indx_t mp_ptrs[] /* dynamic size */; +#endif /* C99 */ +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES +#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) +typedef void mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t mdbx_ipclock_t; +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#define MDBX_CLOCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) +typedef sem_t mdbx_ipclock_t; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); +#endif /* MDBX_LOCKING */ + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Write transaction lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_wlock; +#endif /* MDBX_LOCKING > 0 */ + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Readeaders registration lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_rlock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[] /* dynamic size */; +#endif /* C99 */ +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + __anonymous_struct_extension__ struct { + pgno_t pgno; + MDBX_page *ptr; + }; + __anonymous_struct_extension__ struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#if MDBX_HUGE_TRANSACTIONS +#define MDBX_PNL_MAX \ + ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) +#else +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) +#endif /* MDBX_HUGE_TRANSACTIONS */ + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ + size_t md_klen_min, md_klen_max; /* min/max key length for the database */ + size_t md_vlen_min, + md_vlen_max; /* min/max value/data length for the database */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) +#define MDBX_TXN_RW_BEGIN_FLAGS \ + (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + +#define TXN_FLAGS \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ + MDBX_TXN_HAS_CHILD) + +#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ + ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ + MDBX_SHRINK_ALLOWED) +#error "Oops, some flags overlapped or wrong" +#endif + + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + + /* Transaction DBI Flags */ +#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ +#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ +#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ +#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ +#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ +#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* Array of flags for each DB */ + uint8_t *mt_dbistate; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + MDBX_canary mt_canary; + void *mt_userctx; /* User-settable context */ + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbistate for this database */ + uint8_t *mc_dbistate; + unsigned mc_snum; /* number of pushed pages */ + unsigned mc_top; /* index of top page, normally mc_snum-1 */ + + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + + /* Cursor checking flags. */ +#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ +#define C_UPDATING 0x200 /* update/rebalance pending */ +#define C_RETIRING 0x400 /* refs to child pages may be invalid */ +#define C_SKIPORD 0x800 /* don't check keys ordering */ + + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbistate for this Dup DB */ + uint8_t mx_dbistate; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + void *mc_userctx; /* User-settable context */ + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + uint32_t me_signature; + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + /* Legacy MDBX_MAPASYNC (prior v0.9) */ +#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) +#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) + uint32_t me_flags; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_lazy_fd me_dxb_mmap.fd + mdbx_filehandle_t me_dsync_fd; + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + unsigned me_psize; /* DB page size, inited from me_os_psize */ + uint8_t me_psize2log; /* log2 of DB page size */ + int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ + + /* write-txn lock */ +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING > 0 + mdbx_ipclock_t *me_wlock; +#endif /* MDBX_LOCKING > 0 */ + + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + unsigned me_branch_nodemax; /* max size of a branch-node */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ + struct { +#if MDBX_LOCKING > 0 + mdbx_ipclock_t wlock; +#endif /* MDBX_LOCKING > 0 */ + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +#ifndef __cplusplus +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +extern uint8_t mdbx_runtime_flags; +extern uint8_t mdbx_loglevel; +extern MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) + mdbx_debug_log(int level, const char *function, int line, const char *fmt, + ...) MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function, + int line, const char *fmt, + va_list args); + +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#endif /* MDBX_DEBUG */ + +#if !MDBX_DEBUG && defined(__ANDROID_API__) +#define mdbx_assert_fail(env, msg, func, line) \ + __android_log_assert(msg, "mdbx", "%s:%u", func, line) +#else +void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, + int line); +#endif + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ + +#if MDBX_CPU_WRITEBACK_INCOHERENT +#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() +#else +#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +static __maybe_unused __inline void +mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = + (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + mdbx_assert(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#if MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +#endif /* !__cplusplus */ + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) + +/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + }; +#endif /* __BYTE_ORDER__ */ + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + uint8_t mn_data[] /* key and data are appended here */; +#endif /* C99 */ +} MDBX_node; + +#define DB_PERSISTENT_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP) + +/* mdbx_dbi_open() flags */ +#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) + +#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define DB_INTERNAL_FLAGS DB_VALID + +#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif +#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused bool +is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define ENV_CHANGEABLE_FLAGS \ + (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ + MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) +#define ENV_CHANGELESS_FLAGS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) +#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) + +#if !defined(__cplusplus) || defined(__cpp_constexpr) +static __maybe_unused void static_checks(void) { + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, + "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & + (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, + "Oops, some flags overlapped or wrong"); +} +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +#ifdef __cplusplus +} +#endif + +#if defined(_WIN32) || defined(_WIN64) +/* + * POSIX getopt for Windows + * + * AT&T Public License + * + * Code given out at the 1985 UNIFORUM conference in Dallas. + */ + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#endif /* _MSC_VER (warnings) */ + +#include <stdio.h> +#include <string.h> + +#ifdef _MSC_VER +#pragma warning(pop) +#endif +/*----------------------------------------------------------------------------*/ + +#ifndef NULL +#define NULL 0 +#endif + +#ifndef EOF +#define EOF (-1) +#endif + +int optind = 1; +int optopt; +char *optarg; + +int getopt(int argc, char *const argv[], const char *opts) { + static int sp = 1; + int c; + const char *cp; + + if (sp == 1) { + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return EOF; + else if (strcmp(argv[optind], "--") == 0) { + optind++; + return EOF; + } + } + optopt = c = argv[optind][sp]; + if (c == ':' || (cp = strchr(opts, c)) == NULL) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c); + if (argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return '?'; + } + if (*++cp == ':') { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", + c); + sp = 1; + return '?'; + } else + optarg = argv[optind++]; + sp = 1; + } else { + if (argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return c; +} + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +static void usage(const char *prog) { + fprintf(stderr, + "usage: %s [-V] [-q] [-c] src_path [dest_path]\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -c\t\tenable compactification (skip unused pages)\n" + " src_path\tsource database\n" + " dest_path\tdestination (stdout if not specified)\n", + prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) { + int rc; + MDBX_env *env = NULL; + const char *progname = argv[0], *act; + unsigned flags = MDBX_RDONLY; + unsigned cpflags = 0; + bool quiet = false; + + for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { + if (argv[1][1] == 'n' && argv[1][2] == '\0') + flags |= MDBX_NOSUBDIR; + else if (argv[1][1] == 'c' && argv[1][2] == '\0') + cpflags |= MDBX_CP_COMPACT; + else if (argv[1][1] == 'q' && argv[1][2] == '\0') + quiet = true; + else if ((argv[1][1] == 'h' && argv[1][2] == '\0') || + strcmp(argv[1], "--help") == 0) + usage(progname); + else if (argv[1][1] == 'V' && argv[1][2] == '\0') { + printf("mdbx_copy version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + } else + argc = 0; + } + + if (argc < 2 || argc > 3) + usage(progname); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + if (!quiet) { + fprintf((argc == 2) ? stderr : stdout, + "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]); + fflush(NULL); + } + + act = "opening environment"; + rc = mdbx_env_create(&env); + if (rc == MDBX_SUCCESS) { + rc = mdbx_env_open(env, argv[1], flags, 0640); + } + if (rc == MDBX_SUCCESS) { + act = "copying"; + if (argc == 2) { + mdbx_filehandle_t fd; +#if defined(_WIN32) || defined(_WIN64) + fd = GetStdHandle(STD_OUTPUT_HANDLE); +#else + fd = fileno(stdout); +#endif + rc = mdbx_env_copy2fd(env, fd, cpflags); + } else + rc = mdbx_env_copy(env, argv[2], cpflags); + } + if (rc) + fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, + mdbx_strerror(rc)); + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libs/libmdbx/src/mdbx_dump.c b/libs/libmdbx/src/mdbx_dump.c new file mode 100644 index 0000000000..f338498c7c --- /dev/null +++ b/libs/libmdbx/src/mdbx_dump.c @@ -0,0 +1,3592 @@ +/* mdbx_dump.c - memory-mapped database dump tool */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#define MDBX_BUILD_SOURCERY 3b5677a6062b714f1e138b0066c5590ee3c9ebf3bf8cfa3bb9503515ea0d1f02_v0_9_1_18_g1d31ebdc1c +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_FULL_VER < 190024234 + /* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual Studio 2015 Update 3). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for alignment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant aggregate initializer */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ + +#if defined(MDBX_TOOLS) +#undef MDBX_DEPRECATED +#define MDBX_DEPRECATED +#endif /* MDBX_TOOLS */ + +#include "mdbx.h" +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# else +# define __noinline +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(7, 0) && \ + (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +# define __fallthrough __attribute__((__fallthrough__)) +# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ + __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define __fallthrough [[clang::fallthrough]] +# else +# define __fallthrough +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline __maybe_unused const void* MDBX_PURE_FUNCTION + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include <valgrind/memcheck.h> +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include <sanitizer/asan_interface.h> +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template <typename T, size_t N> + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_STATIC_ASSERT) +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif defined(_MSC_VER) +# include <crtdbg.h> +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wnested-anon-types") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wnested-anon-types" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wnested-anon-types" +# else +# pragma warning disable "nested-anon-types" +# endif +#endif /* -Wnested-anon-types */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) && MDBX_AVOID_CRT +#define _NO_CRT_STDIO_INLINE +#endif +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include <assert.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +/* C11 stdalign.h */ +#if __has_include(<stdalign.h>) +#include <stdalign.h> +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif /* Apple OSX & iOS */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) +#include <sys/cdefs.h> +#include <sys/mount.h> +#include <sys/sysctl.h> +#include <sys/types.h> +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <vm/vm_param.h> +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include <uvm/uvm_param.h> +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#ifndef __MACH__ +#include <sys/vmmeter.h> +#endif +#else +#include <malloc.h> +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(_WIN32) || defined(_WIN64)) +#include <mntent.h> +#endif /* !Solaris */ +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) +#include <malloc_np.h> +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) +#include <malloc/malloc.h> +#endif /* MacOS */ + +#if defined(__MACH__) +#include <mach/host_info.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> +#include <uuid/uuid.h> +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include <linux/sysctl.h> +#include <sched.h> +#include <sys/sendfile.h> +#include <sys/statfs.h> +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include <utmpx.h> +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include <kstat.h> +#include <sys/mnttab.h> +/* On Solaris, it's easier to add a missing prototype rather than find a + * combination of #defines that break nothing. */ +__extern_C key_t ftok(const char *, int); +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <tlhelp32.h> +#include <windows.h> +#include <winnt.h> +#include <winternl.h> +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event[2]; +} mdbx_condpair_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <sys/file.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond[2]; +} mdbx_condpair_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +#ifdef __ANDROID_API__ +#include <android/log.h> +#if __ANDROID_API__ >= 21 +#include <sys/sendfile.h> +#endif +#endif /* Android */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) +#include <sys/stat.h> +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) +#include <sys/types.h> +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) +#include <sys/file.h> +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include <intrin.h> +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif /* __ia32__ */ +#if defined(__ia32__) +#include <cpuid.h> +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include <mbarrier.h> +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include <machine/sys/inline.h> +#elif defined(__IBMC__) && defined(__powerpc) +#include <atomic.h> +#elif defined(_AIX) +#include <builtins.h> +#include <sys/atomic_op.h> +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include <c_asm.h> +#include <machine/builtins.h> +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include <machine/inline.h> +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline size_t +mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; +#endif /* Windows */ + +#ifdef __cplusplus +extern void mdbx_osal_jitter(bool tiny); +#else +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +#if __has_include(<sys/cachectl.h>) +#include <sys/cachectl.h> +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include <sys/cachectl.h> +#endif + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC MDBX_PRINTF_ARGS(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +MDBX_INTERNAL_VAR bool + mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; +#endif /* Linux */ + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_NONE = 0, + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + const enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); + +enum mdbx_openfile_purpose { + MDBX_OPEN_DXB_READ = 0, + MDBX_OPEN_DXB_LAZY = 1, + MDBX_OPEN_DXB_DSYNC = 2, + MDBX_OPEN_LCK = 3, + MDBX_OPEN_COPY = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +#define MMAP_OPTION_TRUNCATE 1 +#define MMAP_OPTION_SEMAPHORE 2 +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const unsigned options); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna, const bool may_move); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { + mdbx_tid_t thunk; + STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); +#if defined(_WIN32) || defined(_WIN64) + thunk = GetCurrentThreadId(); +#else + thunk = pthread_self(); +#endif + return (uintptr_t)thunk; +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by argument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (especially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); +MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; + +static __inline bool mdbx_RunningUnderWine(void) { + return !mdbx_NtExtendSection; +} + +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, + LPCSTR lpValue, DWORD dwFlags, + LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData); +MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include <stdatomic.h> +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include <libkern/OSAtomic.h> +#else +#error FIXME atomic-ops +#endif + +#endif /* !__cplusplus */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/******************************************************************************* + ******************************************************************************* + ******************************************************************************* + * + * + * #### ##### ##### # #### # # #### + * # # # # # # # # ## # # + * # # # # # # # # # # # #### + * # # ##### # # # # # # # # + * # # # # # # # # ## # # + * #### # # # #### # # #### + * + * + */ + +/** \defgroup build_option Build options + * The libmdbx build options. + @{ */ + +#ifdef DOXYGEN +/* !!! Actually this is a fake definitions !!! + * !!! for documentation generation by Doxygen !!! */ + +/** Controls enabling of debugging features. + * + * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all, + * including logging and assertion controls. + * Logging level and corresponding debug flags changing + * by \ref mdbx_setup_debug() will not have effect. + * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging, + * assertions checking and internal audit). + * Simultaneously sets the default logging level + * to the `MDBX_DEBUG` value. + * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. + * + * \ingroup build_option */ +#define MDBX_DEBUG 0...7 + +/** Disables using of GNU libc extensions. */ +#define MDBX_DISABLE_GNU_SOURCE 0 or 1 + +#endif /* DOXYGEN */ + +/** Enables support for huge write-transactions */ +#ifndef MDBX_HUGE_TRANSACTIONS +#define MDBX_HUGE_TRANSACTIONS 0 +#endif /* MDBX_HUGE_TRANSACTIONS */ + +/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_DURABILITY 0 +/** Using fsync() with chance of data lost on power failure */ +#define MDBX_OSX_WANNA_SPEED 1 + +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED + * for OSX & iOS */ +#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ + +/** Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_ENV_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be omitted: + * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_ENV_CHECKPID 0 +#else +#define MDBX_ENV_CHECKPID 1 +#endif +#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_ENV_CHECKPID) +#else +#define MDBX_ENV_CHECKPID_CONFIG STRINGIFY(MDBX_ENV_CHECKPID) +#endif /* MDBX_ENV_CHECKPID */ + +/** Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +/** Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +//------------------------------------------------------------------------------ + +/** Win32 File Locking API for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_WIN32FILES -1 + +/** SystemV IPC semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_SYSV 5 + +/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX1988 1988 + +/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2001 2001 + +/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2008 2008 + +/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_BENAPHORE 1995 + +/** Advanced: Choices the locking implementation (autodetection by default). */ +#if defined(_WIN32) || defined(_WIN64) +#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES +#else +#ifndef MDBX_LOCKING +#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ + _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ + _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ + _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 +#else +#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 +#endif +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 +#else +#define MDBX_LOCKING MDBX_LOCKING_SYSV +#endif +#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) +#else +#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) +#endif /* MDBX_LOCKING */ +#endif /* !Windows */ + +/** Advanced: Using POSIX OFD-locks (autodetection by default). */ +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) && \ + !defined(__sun) /* OFD-lock are broken on Solaris */ +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/** Advanced: Using sendfile() syscall (autodetection by default). */ +#ifndef MDBX_USE_SENDFILE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) +#define MDBX_USE_SENDFILE 1 +#else +#define MDBX_USE_SENDFILE 0 +#endif +#endif /* MDBX_USE_SENDFILE */ + +/** Advanced: Using copy_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_COPYFILERANGE +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) +#define MDBX_USE_COPYFILERANGE 1 +#else +#define MDBX_USE_COPYFILERANGE 0 +#endif +#endif /* MDBX_USE_COPYFILERANGE */ + +/** Advanced: Using sync_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_SYNCFILERANGE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) +#define MDBX_USE_SYNCFILERANGE 1 +#else +#define MDBX_USE_SYNCFILERANGE 0 +#endif +#endif /* MDBX_USE_SYNCFILERANGE */ + +//------------------------------------------------------------------------------ + +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) || defined(DOXYGEN) +#define MDBX_CPU_WRITEBACK_INCOHERENT 0 +#else +#define MDBX_CPU_WRITEBACK_INCOHERENT 1 +#endif +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#ifdef __OpenBSD__ +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 +#else +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +/** @} end of build options */ +/******************************************************************************* + ******************************************************************************* + ******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) +#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) +#define INVALID_TXNID UINT64_MAX +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + MDBX_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + bin128_t mm_bootid; + +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + __anonymous_struct_extension__ struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + indx_t mp_ptrs[] /* dynamic size */; +#endif /* C99 */ +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES +#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) +typedef void mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t mdbx_ipclock_t; +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#define MDBX_CLOCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) +typedef sem_t mdbx_ipclock_t; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); +#endif /* MDBX_LOCKING */ + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Write transaction lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_wlock; +#endif /* MDBX_LOCKING > 0 */ + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Readeaders registration lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_rlock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[] /* dynamic size */; +#endif /* C99 */ +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + __anonymous_struct_extension__ struct { + pgno_t pgno; + MDBX_page *ptr; + }; + __anonymous_struct_extension__ struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#if MDBX_HUGE_TRANSACTIONS +#define MDBX_PNL_MAX \ + ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) +#else +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) +#endif /* MDBX_HUGE_TRANSACTIONS */ + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ + size_t md_klen_min, md_klen_max; /* min/max key length for the database */ + size_t md_vlen_min, + md_vlen_max; /* min/max value/data length for the database */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) +#define MDBX_TXN_RW_BEGIN_FLAGS \ + (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + +#define TXN_FLAGS \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ + MDBX_TXN_HAS_CHILD) + +#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ + ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ + MDBX_SHRINK_ALLOWED) +#error "Oops, some flags overlapped or wrong" +#endif + + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + + /* Transaction DBI Flags */ +#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ +#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ +#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ +#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ +#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ +#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* Array of flags for each DB */ + uint8_t *mt_dbistate; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + MDBX_canary mt_canary; + void *mt_userctx; /* User-settable context */ + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbistate for this database */ + uint8_t *mc_dbistate; + unsigned mc_snum; /* number of pushed pages */ + unsigned mc_top; /* index of top page, normally mc_snum-1 */ + + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + + /* Cursor checking flags. */ +#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ +#define C_UPDATING 0x200 /* update/rebalance pending */ +#define C_RETIRING 0x400 /* refs to child pages may be invalid */ +#define C_SKIPORD 0x800 /* don't check keys ordering */ + + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbistate for this Dup DB */ + uint8_t mx_dbistate; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + void *mc_userctx; /* User-settable context */ + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + uint32_t me_signature; + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + /* Legacy MDBX_MAPASYNC (prior v0.9) */ +#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) +#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) + uint32_t me_flags; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_lazy_fd me_dxb_mmap.fd + mdbx_filehandle_t me_dsync_fd; + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + unsigned me_psize; /* DB page size, inited from me_os_psize */ + uint8_t me_psize2log; /* log2 of DB page size */ + int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ + + /* write-txn lock */ +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING > 0 + mdbx_ipclock_t *me_wlock; +#endif /* MDBX_LOCKING > 0 */ + + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + unsigned me_branch_nodemax; /* max size of a branch-node */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ + struct { +#if MDBX_LOCKING > 0 + mdbx_ipclock_t wlock; +#endif /* MDBX_LOCKING > 0 */ + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +#ifndef __cplusplus +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +extern uint8_t mdbx_runtime_flags; +extern uint8_t mdbx_loglevel; +extern MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) + mdbx_debug_log(int level, const char *function, int line, const char *fmt, + ...) MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function, + int line, const char *fmt, + va_list args); + +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#endif /* MDBX_DEBUG */ + +#if !MDBX_DEBUG && defined(__ANDROID_API__) +#define mdbx_assert_fail(env, msg, func, line) \ + __android_log_assert(msg, "mdbx", "%s:%u", func, line) +#else +void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, + int line); +#endif + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ + +#if MDBX_CPU_WRITEBACK_INCOHERENT +#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() +#else +#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +static __maybe_unused __inline void +mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = + (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + mdbx_assert(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#if MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +#endif /* !__cplusplus */ + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) + +/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + }; +#endif /* __BYTE_ORDER__ */ + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + uint8_t mn_data[] /* key and data are appended here */; +#endif /* C99 */ +} MDBX_node; + +#define DB_PERSISTENT_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP) + +/* mdbx_dbi_open() flags */ +#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) + +#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define DB_INTERNAL_FLAGS DB_VALID + +#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif +#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused bool +is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define ENV_CHANGEABLE_FLAGS \ + (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ + MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) +#define ENV_CHANGELESS_FLAGS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) +#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) + +#if !defined(__cplusplus) || defined(__cpp_constexpr) +static __maybe_unused void static_checks(void) { + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, + "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & + (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, + "Oops, some flags overlapped or wrong"); +} +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +#ifdef __cplusplus +} +#endif + +#include <ctype.h> + +#define PRINT 1 +#define GLOBAL 2 +static int mode = GLOBAL; + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"}, + {MDBX_DUPSORT, "dupsort"}, + {MDBX_INTEGERKEY, "integerkey"}, + {MDBX_DUPFIXED, "dupfixed"}, + {MDBX_INTEGERDUP, "integerdup"}, + {MDBX_REVERSEDUP, "reversedup"}, + {0, nullptr}}; + +#if defined(_WIN32) || defined(_WIN64) +/* + * POSIX getopt for Windows + * + * AT&T Public License + * + * Code given out at the 1985 UNIFORUM conference in Dallas. + */ + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#endif /* _MSC_VER (warnings) */ + +#include <stdio.h> +#include <string.h> + +#ifdef _MSC_VER +#pragma warning(pop) +#endif +/*----------------------------------------------------------------------------*/ + +#ifndef NULL +#define NULL 0 +#endif + +#ifndef EOF +#define EOF (-1) +#endif + +int optind = 1; +int optopt; +char *optarg; + +int getopt(int argc, char *const argv[], const char *opts) { + static int sp = 1; + int c; + const char *cp; + + if (sp == 1) { + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return EOF; + else if (strcmp(argv[optind], "--") == 0) { + optind++; + return EOF; + } + } + optopt = c = argv[optind][sp]; + if (c == ':' || (cp = strchr(opts, c)) == NULL) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c); + if (argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return '?'; + } + if (*++cp == ':') { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", + c); + sp = 1; + return '?'; + } else + optarg = argv[optind++]; + sp = 1; + } else { + if (argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return c; +} + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +static const char hexc[] = "0123456789abcdef"; + +static void dumpbyte(unsigned char c) { + putchar(hexc[c >> 4]); + putchar(hexc[c & 0xf]); +} + +static void text(MDBX_val *v) { + unsigned char *c, *end; + + putchar(' '); + c = v->iov_base; + end = c + v->iov_len; + while (c < end) { + if (isprint(*c) && *c != '\\') { + putchar(*c); + } else { + putchar('\\'); + dumpbyte(*c); + } + c++; + } + putchar('\n'); +} + +static void dumpval(MDBX_val *v) { + unsigned char *c, *end; + + putchar(' '); + c = v->iov_base; + end = c + v->iov_len; + while (c < end) + dumpbyte(*c++); + putchar('\n'); +} + +bool quiet = false, rescue = false; +const char *prog; +static void error(const char *func, int rc) { + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); +} + +/* Dump in BDB-compatible format */ +static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { + unsigned int flags; + int rc = mdbx_dbi_flags(txn, dbi, &flags); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_flags", rc); + return rc; + } + + MDBX_stat ms; + rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms)); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_stat", rc); + return rc; + } + + MDBX_envinfo info; + rc = mdbx_env_info_ex(mdbx_txn_env(txn), txn, &info, sizeof(info)); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_info_ex", rc); + return rc; + } + + printf("VERSION=3\n"); + if (mode & GLOBAL) { + mode -= GLOBAL; + if (info.mi_geo.upper != info.mi_geo.lower) + printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 + ",g%" PRIu64 "\n", + info.mi_geo.lower, info.mi_geo.current, info.mi_geo.upper, + info.mi_geo.shrink, info.mi_geo.grow); + printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper); + printf("maxreaders=%u\n", info.mi_maxreaders); + + MDBX_canary canary; + rc = mdbx_canary_get(txn, &canary); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_canary_get", rc); + return rc; + } + if (canary.v) + printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n", + canary.v, canary.x, canary.y, canary.z); + } + printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); + if (name) + printf("database=%s\n", name); + printf("type=btree\n"); + printf("db_pagesize=%u\n", ms.ms_psize); + /* if (ms.ms_mod_txnid) + printf("txnid=%" PRIaTXN "\n", ms.ms_mod_txnid); + else if (!name) + printf("txnid=%" PRIaTXN "\n", mdbx_txn_id(txn)); */ + + printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | + MDBX_INTEGERDUP | MDBX_REVERSEDUP)) + ? 1 + : 0); + for (int i = 0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + printf("%s=1\n", dbflags[i].name); + + uint64_t sequence; + rc = mdbx_dbi_sequence(txn, dbi, &sequence, 0); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_sequence", rc); + return rc; + } + if (sequence) + printf("sequence=%" PRIu64 "\n", sequence); + + printf("HEADER=END\n"); /*-------------------------------------------------*/ + + MDBX_cursor *cursor; + MDBX_val key, data; + rc = mdbx_cursor_open(txn, dbi, &cursor); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_open", rc); + return rc; + } + if (MDBX_DEBUG > 0 && rescue) { + cursor->mc_flags |= C_SKIPORD; + if (cursor->mc_xcursor) + cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; + } + + while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == + MDBX_SUCCESS) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + if (mode & PRINT) { + text(&key); + text(&data); + } else { + dumpval(&key); + dumpval(&data); + } + } + printf("DATA=END\n"); + if (rc == MDBX_NOTFOUND) + rc = MDBX_SUCCESS; + if (unlikely(rc != MDBX_SUCCESS)) + error("mdbx_cursor_get", rc); + return rc; +} + +static void usage(void) { + fprintf(stderr, + "usage: %s [-V] [-q] [-f file] [-l] [-p] [-a|-s subdb] [-r] " + "dbpath\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -f\t\twrite to file instead of stdout\n" + " -l\t\tlist subDBs and exit\n" + " -p\t\tuse printable characters\n" + " -a\t\tdump main DB and all subDBs,\n" + " \t\tby default dump only the main DB\n" + " -s\t\tdump only the named subDB\n" + " -r\t\trescue mode (ignore errors to dump corrupted DB)\n", + prog); + exit(EXIT_FAILURE); +} + +static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { + return (a->iov_len == b->iov_len && + memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) + ? 0 + : 1; +} + +int main(int argc, char *argv[]) { + int i, rc; + MDBX_env *env; + MDBX_txn *txn; + MDBX_dbi dbi; + prog = argv[0]; + char *envname; + char *subname = nullptr; + unsigned envflags = 0; + bool alldbs = false, list = false; + + if (argc < 2) + usage(); + + while ((i = getopt(argc, argv, "af:lnps:Vrq")) != EOF) { + switch (i) { + case 'V': + printf("mdbx_dump version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'l': + list = true; + /*FALLTHROUGH*/; + __fallthrough; + case 'a': + if (subname) + usage(); + alldbs = true; + break; + case 'f': + if (freopen(optarg, "w", stdout) == nullptr) { + fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, + mdbx_strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 'p': + mode |= PRINT; + break; + case 's': + if (alldbs) + usage(); + subname = optarg; + break; + case 'q': + quiet = true; + break; + case 'r': + rescue = true; + break; + default: + usage(); + } + } + + if (optind != argc - 1) + usage(); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + if (!quiet) { + fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname); + fflush(nullptr); + } + + rc = mdbx_env_create(&env); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_create", rc); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + rc = mdbx_env_set_maxdbs(env, 2); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_set_maxdbs", rc); + goto env_close; + } + } + + rc = mdbx_env_open( + env, envname, + envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE : MDBX_RDONLY), 0); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_open", rc); + goto env_close; + } + + rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_begin", rc); + goto env_close; + } + + rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_open", rc); + goto txn_abort; + } + + if (alldbs) { + assert(dbi == MAIN_DBI); + + MDBX_cursor *cursor; + rc = mdbx_cursor_open(txn, MAIN_DBI, &cursor); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_open", rc); + goto txn_abort; + } + if (MDBX_DEBUG > 0 && rescue) { + cursor->mc_flags |= C_SKIPORD; + if (cursor->mc_xcursor) + cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; + } + + bool have_raw = false; + int count = 0; + MDBX_val key; + while (MDBX_SUCCESS == + (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + + if (memchr(key.iov_base, '\0', key.iov_len)) + continue; + subname = mdbx_malloc(key.iov_len + 1); + memcpy(subname, key.iov_base, key.iov_len); + subname[key.iov_len] = '\0'; + + MDBX_dbi sub_dbi; + rc = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi, + rescue ? equal_or_greater : nullptr, + rescue ? equal_or_greater : nullptr); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_INCOMPATIBLE) { + have_raw = true; + continue; + } + error("mdbx_dbi_open", rc); + if (!rescue) + break; + } else { + count++; + if (list) { + printf("%s\n", subname); + } else { + rc = dump_sdb(txn, sub_dbi, subname); + if (unlikely(rc != MDBX_SUCCESS)) { + if (!rescue) + break; + fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, + envname, mdbx_strerror(rc), subname); + /* Here is a hack for rescue mode, don't do that: + * - we should restart transaction in case error due + * database corruption; + * - but we won't close cursor, reopen and re-positioning it + * for new a transaction; + * - this is possible since DB is opened in read-only exclusive + * mode and transaction is the same, i.e. has the same address + * and so on. */ + rc = mdbx_txn_reset(txn); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_reset", rc); + goto env_close; + } + rc = mdbx_txn_renew(txn); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_renew", rc); + goto env_close; + } + } + } + rc = mdbx_dbi_close(env, sub_dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_close", rc); + break; + } + } + mdbx_free(subname); + } + mdbx_cursor_close(cursor); + cursor = nullptr; + + if (have_raw && (!count /* || rescue */)) + rc = dump_sdb(txn, MAIN_DBI, nullptr); + else if (!count) { + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, + envname); + rc = MDBX_NOTFOUND; + } + } else { + rc = dump_sdb(txn, dbi, subname); + } + + switch (rc) { + case MDBX_NOTFOUND: + rc = MDBX_SUCCESS; + case MDBX_SUCCESS: + break; + case MDBX_EINTR: + fprintf(stderr, "Interrupted by signal/user\n"); + break; + default: + if (unlikely(rc != MDBX_SUCCESS)) + error("mdbx_cursor_get", rc); + } + + mdbx_dbi_close(env, dbi); +txn_abort: + mdbx_txn_abort(txn); +env_close: + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libs/libmdbx/src/mdbx_load.c b/libs/libmdbx/src/mdbx_load.c new file mode 100644 index 0000000000..6832e0f73a --- /dev/null +++ b/libs/libmdbx/src/mdbx_load.c @@ -0,0 +1,3931 @@ +/* mdbx_load.c - memory-mapped database load tool */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#define MDBX_BUILD_SOURCERY 3b5677a6062b714f1e138b0066c5590ee3c9ebf3bf8cfa3bb9503515ea0d1f02_v0_9_1_18_g1d31ebdc1c +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_FULL_VER < 190024234 + /* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual Studio 2015 Update 3). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for alignment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant aggregate initializer */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ + +#if defined(MDBX_TOOLS) +#undef MDBX_DEPRECATED +#define MDBX_DEPRECATED +#endif /* MDBX_TOOLS */ + +#include "mdbx.h" +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# else +# define __noinline +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(7, 0) && \ + (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +# define __fallthrough __attribute__((__fallthrough__)) +# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ + __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define __fallthrough [[clang::fallthrough]] +# else +# define __fallthrough +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline __maybe_unused const void* MDBX_PURE_FUNCTION + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include <valgrind/memcheck.h> +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include <sanitizer/asan_interface.h> +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template <typename T, size_t N> + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_STATIC_ASSERT) +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif defined(_MSC_VER) +# include <crtdbg.h> +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wnested-anon-types") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wnested-anon-types" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wnested-anon-types" +# else +# pragma warning disable "nested-anon-types" +# endif +#endif /* -Wnested-anon-types */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) && MDBX_AVOID_CRT +#define _NO_CRT_STDIO_INLINE +#endif +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include <assert.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +/* C11 stdalign.h */ +#if __has_include(<stdalign.h>) +#include <stdalign.h> +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif /* Apple OSX & iOS */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) +#include <sys/cdefs.h> +#include <sys/mount.h> +#include <sys/sysctl.h> +#include <sys/types.h> +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <vm/vm_param.h> +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include <uvm/uvm_param.h> +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#ifndef __MACH__ +#include <sys/vmmeter.h> +#endif +#else +#include <malloc.h> +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(_WIN32) || defined(_WIN64)) +#include <mntent.h> +#endif /* !Solaris */ +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) +#include <malloc_np.h> +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) +#include <malloc/malloc.h> +#endif /* MacOS */ + +#if defined(__MACH__) +#include <mach/host_info.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> +#include <uuid/uuid.h> +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include <linux/sysctl.h> +#include <sched.h> +#include <sys/sendfile.h> +#include <sys/statfs.h> +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include <utmpx.h> +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include <kstat.h> +#include <sys/mnttab.h> +/* On Solaris, it's easier to add a missing prototype rather than find a + * combination of #defines that break nothing. */ +__extern_C key_t ftok(const char *, int); +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <tlhelp32.h> +#include <windows.h> +#include <winnt.h> +#include <winternl.h> +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event[2]; +} mdbx_condpair_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <sys/file.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond[2]; +} mdbx_condpair_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +#ifdef __ANDROID_API__ +#include <android/log.h> +#if __ANDROID_API__ >= 21 +#include <sys/sendfile.h> +#endif +#endif /* Android */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) +#include <sys/stat.h> +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) +#include <sys/types.h> +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) +#include <sys/file.h> +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include <intrin.h> +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif /* __ia32__ */ +#if defined(__ia32__) +#include <cpuid.h> +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include <mbarrier.h> +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include <machine/sys/inline.h> +#elif defined(__IBMC__) && defined(__powerpc) +#include <atomic.h> +#elif defined(_AIX) +#include <builtins.h> +#include <sys/atomic_op.h> +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include <c_asm.h> +#include <machine/builtins.h> +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include <machine/inline.h> +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline size_t +mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; +#endif /* Windows */ + +#ifdef __cplusplus +extern void mdbx_osal_jitter(bool tiny); +#else +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +#if __has_include(<sys/cachectl.h>) +#include <sys/cachectl.h> +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include <sys/cachectl.h> +#endif + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC MDBX_PRINTF_ARGS(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +MDBX_INTERNAL_VAR bool + mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; +#endif /* Linux */ + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_NONE = 0, + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + const enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); + +enum mdbx_openfile_purpose { + MDBX_OPEN_DXB_READ = 0, + MDBX_OPEN_DXB_LAZY = 1, + MDBX_OPEN_DXB_DSYNC = 2, + MDBX_OPEN_LCK = 3, + MDBX_OPEN_COPY = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +#define MMAP_OPTION_TRUNCATE 1 +#define MMAP_OPTION_SEMAPHORE 2 +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const unsigned options); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna, const bool may_move); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { + mdbx_tid_t thunk; + STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); +#if defined(_WIN32) || defined(_WIN64) + thunk = GetCurrentThreadId(); +#else + thunk = pthread_self(); +#endif + return (uintptr_t)thunk; +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by argument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (especially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); +MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; + +static __inline bool mdbx_RunningUnderWine(void) { + return !mdbx_NtExtendSection; +} + +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, + LPCSTR lpValue, DWORD dwFlags, + LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData); +MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include <stdatomic.h> +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include <libkern/OSAtomic.h> +#else +#error FIXME atomic-ops +#endif + +#endif /* !__cplusplus */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/******************************************************************************* + ******************************************************************************* + ******************************************************************************* + * + * + * #### ##### ##### # #### # # #### + * # # # # # # # # ## # # + * # # # # # # # # # # # #### + * # # ##### # # # # # # # # + * # # # # # # # # ## # # + * #### # # # #### # # #### + * + * + */ + +/** \defgroup build_option Build options + * The libmdbx build options. + @{ */ + +#ifdef DOXYGEN +/* !!! Actually this is a fake definitions !!! + * !!! for documentation generation by Doxygen !!! */ + +/** Controls enabling of debugging features. + * + * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all, + * including logging and assertion controls. + * Logging level and corresponding debug flags changing + * by \ref mdbx_setup_debug() will not have effect. + * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging, + * assertions checking and internal audit). + * Simultaneously sets the default logging level + * to the `MDBX_DEBUG` value. + * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. + * + * \ingroup build_option */ +#define MDBX_DEBUG 0...7 + +/** Disables using of GNU libc extensions. */ +#define MDBX_DISABLE_GNU_SOURCE 0 or 1 + +#endif /* DOXYGEN */ + +/** Enables support for huge write-transactions */ +#ifndef MDBX_HUGE_TRANSACTIONS +#define MDBX_HUGE_TRANSACTIONS 0 +#endif /* MDBX_HUGE_TRANSACTIONS */ + +/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_DURABILITY 0 +/** Using fsync() with chance of data lost on power failure */ +#define MDBX_OSX_WANNA_SPEED 1 + +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED + * for OSX & iOS */ +#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ + +/** Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_ENV_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be omitted: + * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_ENV_CHECKPID 0 +#else +#define MDBX_ENV_CHECKPID 1 +#endif +#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_ENV_CHECKPID) +#else +#define MDBX_ENV_CHECKPID_CONFIG STRINGIFY(MDBX_ENV_CHECKPID) +#endif /* MDBX_ENV_CHECKPID */ + +/** Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +/** Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +//------------------------------------------------------------------------------ + +/** Win32 File Locking API for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_WIN32FILES -1 + +/** SystemV IPC semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_SYSV 5 + +/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX1988 1988 + +/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2001 2001 + +/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2008 2008 + +/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_BENAPHORE 1995 + +/** Advanced: Choices the locking implementation (autodetection by default). */ +#if defined(_WIN32) || defined(_WIN64) +#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES +#else +#ifndef MDBX_LOCKING +#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ + _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ + _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ + _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 +#else +#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 +#endif +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 +#else +#define MDBX_LOCKING MDBX_LOCKING_SYSV +#endif +#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) +#else +#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) +#endif /* MDBX_LOCKING */ +#endif /* !Windows */ + +/** Advanced: Using POSIX OFD-locks (autodetection by default). */ +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) && \ + !defined(__sun) /* OFD-lock are broken on Solaris */ +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/** Advanced: Using sendfile() syscall (autodetection by default). */ +#ifndef MDBX_USE_SENDFILE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) +#define MDBX_USE_SENDFILE 1 +#else +#define MDBX_USE_SENDFILE 0 +#endif +#endif /* MDBX_USE_SENDFILE */ + +/** Advanced: Using copy_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_COPYFILERANGE +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) +#define MDBX_USE_COPYFILERANGE 1 +#else +#define MDBX_USE_COPYFILERANGE 0 +#endif +#endif /* MDBX_USE_COPYFILERANGE */ + +/** Advanced: Using sync_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_SYNCFILERANGE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) +#define MDBX_USE_SYNCFILERANGE 1 +#else +#define MDBX_USE_SYNCFILERANGE 0 +#endif +#endif /* MDBX_USE_SYNCFILERANGE */ + +//------------------------------------------------------------------------------ + +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) || defined(DOXYGEN) +#define MDBX_CPU_WRITEBACK_INCOHERENT 0 +#else +#define MDBX_CPU_WRITEBACK_INCOHERENT 1 +#endif +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#ifdef __OpenBSD__ +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 +#else +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +/** @} end of build options */ +/******************************************************************************* + ******************************************************************************* + ******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) +#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) +#define INVALID_TXNID UINT64_MAX +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + MDBX_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + bin128_t mm_bootid; + +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + __anonymous_struct_extension__ struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + indx_t mp_ptrs[] /* dynamic size */; +#endif /* C99 */ +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES +#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) +typedef void mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t mdbx_ipclock_t; +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#define MDBX_CLOCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) +typedef sem_t mdbx_ipclock_t; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); +#endif /* MDBX_LOCKING */ + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Write transaction lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_wlock; +#endif /* MDBX_LOCKING > 0 */ + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Readeaders registration lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_rlock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[] /* dynamic size */; +#endif /* C99 */ +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + __anonymous_struct_extension__ struct { + pgno_t pgno; + MDBX_page *ptr; + }; + __anonymous_struct_extension__ struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#if MDBX_HUGE_TRANSACTIONS +#define MDBX_PNL_MAX \ + ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) +#else +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) +#endif /* MDBX_HUGE_TRANSACTIONS */ + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ + size_t md_klen_min, md_klen_max; /* min/max key length for the database */ + size_t md_vlen_min, + md_vlen_max; /* min/max value/data length for the database */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) +#define MDBX_TXN_RW_BEGIN_FLAGS \ + (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + +#define TXN_FLAGS \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ + MDBX_TXN_HAS_CHILD) + +#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ + ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ + MDBX_SHRINK_ALLOWED) +#error "Oops, some flags overlapped or wrong" +#endif + + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + + /* Transaction DBI Flags */ +#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ +#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ +#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ +#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ +#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ +#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* Array of flags for each DB */ + uint8_t *mt_dbistate; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + MDBX_canary mt_canary; + void *mt_userctx; /* User-settable context */ + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbistate for this database */ + uint8_t *mc_dbistate; + unsigned mc_snum; /* number of pushed pages */ + unsigned mc_top; /* index of top page, normally mc_snum-1 */ + + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + + /* Cursor checking flags. */ +#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ +#define C_UPDATING 0x200 /* update/rebalance pending */ +#define C_RETIRING 0x400 /* refs to child pages may be invalid */ +#define C_SKIPORD 0x800 /* don't check keys ordering */ + + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbistate for this Dup DB */ + uint8_t mx_dbistate; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + void *mc_userctx; /* User-settable context */ + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + uint32_t me_signature; + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + /* Legacy MDBX_MAPASYNC (prior v0.9) */ +#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) +#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) + uint32_t me_flags; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_lazy_fd me_dxb_mmap.fd + mdbx_filehandle_t me_dsync_fd; + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + unsigned me_psize; /* DB page size, inited from me_os_psize */ + uint8_t me_psize2log; /* log2 of DB page size */ + int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ + + /* write-txn lock */ +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING > 0 + mdbx_ipclock_t *me_wlock; +#endif /* MDBX_LOCKING > 0 */ + + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + unsigned me_branch_nodemax; /* max size of a branch-node */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ + struct { +#if MDBX_LOCKING > 0 + mdbx_ipclock_t wlock; +#endif /* MDBX_LOCKING > 0 */ + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +#ifndef __cplusplus +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +extern uint8_t mdbx_runtime_flags; +extern uint8_t mdbx_loglevel; +extern MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) + mdbx_debug_log(int level, const char *function, int line, const char *fmt, + ...) MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function, + int line, const char *fmt, + va_list args); + +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#endif /* MDBX_DEBUG */ + +#if !MDBX_DEBUG && defined(__ANDROID_API__) +#define mdbx_assert_fail(env, msg, func, line) \ + __android_log_assert(msg, "mdbx", "%s:%u", func, line) +#else +void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, + int line); +#endif + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ + +#if MDBX_CPU_WRITEBACK_INCOHERENT +#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() +#else +#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +static __maybe_unused __inline void +mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = + (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + mdbx_assert(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#if MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +#endif /* !__cplusplus */ + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) + +/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + }; +#endif /* __BYTE_ORDER__ */ + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + uint8_t mn_data[] /* key and data are appended here */; +#endif /* C99 */ +} MDBX_node; + +#define DB_PERSISTENT_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP) + +/* mdbx_dbi_open() flags */ +#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) + +#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define DB_INTERNAL_FLAGS DB_VALID + +#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif +#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused bool +is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define ENV_CHANGEABLE_FLAGS \ + (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ + MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) +#define ENV_CHANGELESS_FLAGS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) +#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) + +#if !defined(__cplusplus) || defined(__cpp_constexpr) +static __maybe_unused void static_checks(void) { + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, + "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & + (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, + "Oops, some flags overlapped or wrong"); +} +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +#ifdef __cplusplus +} +#endif + +#include <ctype.h> + +#if defined(_WIN32) || defined(_WIN64) +/* + * POSIX getopt for Windows + * + * AT&T Public License + * + * Code given out at the 1985 UNIFORUM conference in Dallas. + */ + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#endif /* _MSC_VER (warnings) */ + +#include <stdio.h> +#include <string.h> + +#ifdef _MSC_VER +#pragma warning(pop) +#endif +/*----------------------------------------------------------------------------*/ + +#ifndef NULL +#define NULL 0 +#endif + +#ifndef EOF +#define EOF (-1) +#endif + +int optind = 1; +int optopt; +char *optarg; + +int getopt(int argc, char *const argv[], const char *opts) { + static int sp = 1; + int c; + const char *cp; + + if (sp == 1) { + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return EOF; + else if (strcmp(argv[optind], "--") == 0) { + optind++; + return EOF; + } + } + optopt = c = argv[optind][sp]; + if (c == ':' || (cp = strchr(opts, c)) == NULL) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c); + if (argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return '?'; + } + if (*++cp == ':') { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", + c); + sp = 1; + return '?'; + } else + optarg = argv[optind++]; + sp = 1; + } else { + if (argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return c; +} + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +static char *prog; +static size_t lineno; +static void error(const char *func, int rc) { + if (lineno) + fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n", + prog, lineno, func, rc, mdbx_strerror(rc)); + else + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, + mdbx_strerror(rc)); +} + +static char *valstr(char *line, const char *item) { + const size_t len = strlen(item); + if (strncmp(line, item, len) != 0) + return nullptr; + if (line[len] != '=') { + if (line[len] > ' ') + return nullptr; + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", + prog, lineno, item); + exit(EXIT_FAILURE); + } + char *ptr = strchr(line, '\n'); + if (ptr) + *ptr = '\0'; + return line + len + 1; +} + +static bool valnum(char *line, const char *item, uint64_t *value) { + char *str = valstr(line, item); + if (!str) + return false; + + char *end = nullptr; + *value = strtoull(str, &end, 0); + if (end && *end) { + fprintf(stderr, + "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n", prog, + lineno, item); + exit(EXIT_FAILURE); + } + return true; +} + +static bool valbool(char *line, const char *item, bool *value) { + uint64_t u64; + if (!valnum(line, item, &u64)) + return false; + + if (u64 > 1) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n", prog, + lineno, item); + exit(EXIT_FAILURE); + } + *value = u64 != 0; + return true; +} + +/*----------------------------------------------------------------------------*/ + +static char *subname = nullptr; +static int dbi_flags; +static txnid_t txnid; +static uint64_t sequence; +static MDBX_canary canary; +static MDBX_envinfo envinfo; + +#define PRINT 1 +#define NOHDR 2 +#define GLOBAL 4 +static int mode = GLOBAL; + +static MDBX_val kbuf, dbuf; +static MDBX_val k0buf; + +#define STRLENOF(s) (sizeof(s) - 1) + +typedef struct flagbit { + unsigned bit; + char *name; + unsigned len; +} flagbit; + +#define S(s) s, STRLENOF(s) + +flagbit dbflags[] = { + {MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")}, + {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")}, + {MDBX_DUPFIXED, S("dupfixed")}, {MDBX_INTEGERDUP, S("integerdup")}, + {MDBX_REVERSEDUP, S("reversedup")}, {0, nullptr, 0}}; + +static int readhdr(void) { + /* reset parameters */ + if (subname) { + free(subname); + subname = nullptr; + } + dbi_flags = 0; + txnid = 0; + sequence = 0; + + while (true) { + errno = 0; + if (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) == nullptr) + return errno ? errno : EOF; + if (user_break) + return MDBX_EINTR; + + lineno++; + uint64_t u64; + + if (valnum(dbuf.iov_base, "VERSION", &u64)) { + if (u64 != 3) { + fprintf(stderr, + "%s: line %" PRIiSIZE ": unsupported value %" PRIu64 + " for %s\n", + prog, lineno, u64, "VERSION"); + exit(EXIT_FAILURE); + } + continue; + } + + if (valnum(dbuf.iov_base, "db_pagesize", &u64)) { + if (!(mode & GLOBAL) && envinfo.mi_dxb_pagesize != u64) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore value %" PRIu64 + " for '%s' in non-global context\n", + prog, lineno, u64, "db_pagesize"); + else if (u64 < MDBX_MIN_PAGESIZE || u64 > MDBX_MAX_PAGESIZE) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64 + " for %s\n", + prog, lineno, u64, "db_pagesize"); + else + envinfo.mi_dxb_pagesize = (uint32_t)u64; + continue; + } + + char *str = valstr(dbuf.iov_base, "format"); + if (str) { + if (strcmp(str, "print") == 0) { + mode |= PRINT; + continue; + } + if (strcmp(str, "bytevalue") == 0) { + mode &= ~PRINT; + continue; + } + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", + prog, lineno, str, "format"); + exit(EXIT_FAILURE); + } + + str = valstr(dbuf.iov_base, "database"); + if (str) { + if (*str) { + free(subname); + subname = mdbx_strdup(str); + if (!subname) { + perror("strdup()"); + exit(EXIT_FAILURE); + } + } + continue; + } + + str = valstr(dbuf.iov_base, "type"); + if (str) { + if (strcmp(str, "btree") != 0) { + fprintf(stderr, + "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, + lineno, str, "type"); + free(subname); + exit(EXIT_FAILURE); + } + continue; + } + + if (valnum(dbuf.iov_base, "mapaddr", &u64)) { + if (u64) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 + " for %s\n", + prog, lineno, u64, "mapaddr"); + continue; + } + + if (valnum(dbuf.iov_base, "mapsize", &u64)) { + if (!(mode & GLOBAL)) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore value %" PRIu64 + " for '%s' in non-global context\n", + prog, lineno, u64, "mapsize"); + else if (u64 < MIN_MAPSIZE || u64 > MAX_MAPSIZE64) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 + " for %s\n", + prog, lineno, u64, "mapsize"); + else + envinfo.mi_mapsize = (size_t)u64; + continue; + } + + if (valnum(dbuf.iov_base, "maxreaders", &u64)) { + if (!(mode & GLOBAL)) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore value %" PRIu64 + " for '%s' in non-global context\n", + prog, lineno, u64, "maxreaders"); + else if (u64 < 1 || u64 > MDBX_READERS_LIMIT) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 + " for %s\n", + prog, lineno, u64, "maxreaders"); + else + envinfo.mi_maxreaders = (int)u64; + continue; + } + + if (valnum(dbuf.iov_base, "txnid", &u64)) { + if (u64 < MIN_TXNID || u64 > MAX_TXNID) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 + " for %s\n", + prog, lineno, u64, "txnid"); + txnid = u64; + continue; + } + + if (valnum(dbuf.iov_base, "sequence", &u64)) { + sequence = u64; + continue; + } + + str = valstr(dbuf.iov_base, "geometry"); + if (str) { + if (!(mode & GLOBAL)) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore values %s" + " for '%s' in non-global context\n", + prog, lineno, str, "geometry"); + else if (sscanf(str, + "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 + ",g%" PRIu64, + &envinfo.mi_geo.lower, &envinfo.mi_geo.current, + &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink, + &envinfo.mi_geo.grow) != 5) { + fprintf(stderr, + "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", + prog, lineno, "geometry"); + exit(EXIT_FAILURE); + } + continue; + } + + str = valstr(dbuf.iov_base, "canary"); + if (str) { + if (!(mode & GLOBAL)) + fprintf(stderr, + "%s: line %" PRIiSIZE ": ignore values %s" + " for '%s' in non-global context\n", + prog, lineno, str, "canary"); + else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64, + &canary.v, &canary.x, &canary.y, &canary.z) != 4) { + fprintf(stderr, + "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", + prog, lineno, "canary"); + exit(EXIT_FAILURE); + } + continue; + } + + for (int i = 0; dbflags[i].bit; i++) { + bool value = false; + if (valbool(dbuf.iov_base, dbflags[i].name, &value)) { + if (value) + dbi_flags |= dbflags[i].bit; + else + dbi_flags &= ~dbflags[i].bit; + goto next; + } + } + + str = valstr(dbuf.iov_base, "HEADER"); + if (str) { + if (strcmp(str, "END") == 0) + return MDBX_SUCCESS; + } + + fprintf(stderr, + "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", prog, + lineno, (char *)dbuf.iov_base); + next:; + } + return EOF; +} + +static int badend(void) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, + lineno); + return errno ? errno : MDBX_ENODATA; +} + +static int unhex(unsigned char *c2) { + int x, c; + x = *c2++ & 0x4f; + if (x & 0x40) + x -= 55; + c = x << 4; + x = *c2 & 0x4f; + if (x & 0x40) + x -= 55; + c |= x; + return c; +} + +static int readline(MDBX_val *out, MDBX_val *buf) { + unsigned char *c1, *c2, *end; + size_t len, l2; + int c; + + if (user_break) + return MDBX_EINTR; + + errno = 0; + if (!(mode & NOHDR)) { + c = fgetc(stdin); + if (c == EOF) + return errno ? errno : EOF; + if (c != ' ') { + lineno++; + errno = 0; + if (fgets(buf->iov_base, (int)buf->iov_len, stdin)) { + if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END"))) + return EOF; + } + return badend(); + } + } + if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == nullptr) + return errno ? errno : EOF; + lineno++; + + c1 = buf->iov_base; + len = strlen((char *)c1); + l2 = len; + + /* Is buffer too short? */ + while (c1[len - 1] != '\n') { + buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2); + if (!buf->iov_base) { + fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", + prog, lineno); + return MDBX_ENOMEM; + } + c1 = buf->iov_base; + c1 += l2; + errno = 0; + if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == nullptr) + return errno ? errno : EOF; + buf->iov_len *= 2; + len = strlen((char *)c1); + l2 += len; + } + c1 = c2 = buf->iov_base; + len = l2; + c1[--len] = '\0'; + end = c1 + len; + + if (mode & PRINT) { + while (c2 < end) { + if (unlikely(*c2 == '\\')) { + if (c2[1] == '\\') { + *c1++ = '\\'; + } else { + if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) + return badend(); + *c1++ = (char)unhex(++c2); + } + c2 += 2; + } else { + /* copies are redundant when no escapes were used */ + *c1++ = *c2++; + } + } + } else { + /* odd length not allowed */ + if (len & 1) + return badend(); + while (c2 < end) { + if (!isxdigit(*c2) || !isxdigit(c2[1])) + return badend(); + *c1++ = (char)unhex(c2); + c2 += 2; + } + } + c2 = out->iov_base = buf->iov_base; + out->iov_len = c1 - c2; + + return MDBX_SUCCESS; +} + +static void usage(void) { + fprintf(stderr, + "usage: %s [-V] [-q] [-a] [-f file] [-s name] [-N] [-T] [-r] [-n]" + "dbpath\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -a\t\tappend records in input order (required for custom " + "comparators)\n" + " -f file\tread from file instead of stdin\n" + " -s name\tload into named subDB\n" + " -N\t\tdon't overwrite existing records when loading (), just skip " + "them\n" + " -T\t\tread plaintext\n" + " -r\t\trescue mode (ignore errors to load corrupted DB dump)\n" + " -n\t\tdon't use subdirectory for newly created database " + "(MDBX_NOSUBDIR)\n", + prog); + exit(EXIT_FAILURE); +} + +static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { + return (a->iov_len == b->iov_len && + memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) + ? 0 + : 1; +} + +int main(int argc, char *argv[]) { + int i, rc; + MDBX_env *env = nullptr; + MDBX_txn *txn = nullptr; + MDBX_cursor *mc = nullptr; + MDBX_dbi dbi; + char *envname = nullptr; + int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0; + bool append = false; + bool quiet = false; + bool rescue = false; + MDBX_val prevk; + + prog = argv[0]; + if (argc < 2) + usage(); + + while ((i = getopt(argc, argv, "af:ns:NTVrq")) != EOF) { + switch (i) { + case 'V': + printf("mdbx_load version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'a': + append = true; + break; + case 'f': + if (freopen(optarg, "r", stdin) == nullptr) { + fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, + mdbx_strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 's': + subname = mdbx_strdup(optarg); + break; + case 'N': + putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA; + break; + case 'T': + mode |= NOHDR | PRINT; + break; + case 'q': + quiet = true; + break; + case 'r': + rescue = true; + break; + default: + usage(); + } + } + + if (optind != argc - 1) + usage(); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + if (!quiet) + printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname); + fflush(nullptr); + + dbuf.iov_len = 4096; + dbuf.iov_base = mdbx_malloc(dbuf.iov_len); + + /* read first header for mapsize= */ + if (!(mode & NOHDR)) { + rc = readhdr(); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == EOF) + rc = MDBX_ENODATA; + error("readheader", rc); + goto env_close; + } + } + + rc = mdbx_env_create(&env); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_create", rc); + return EXIT_FAILURE; + } + + mdbx_env_set_maxdbs(env, 2); + if (envinfo.mi_maxreaders) { + rc = mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_set_maxreaders", rc); + goto env_close; + } + } + + if (envinfo.mi_mapsize) { + if (envinfo.mi_geo.current) { + rc = mdbx_env_set_geometry( + env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current, + (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink, + (intptr_t)envinfo.mi_geo.grow, + envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); + } else { + if (envinfo.mi_mapsize > MAX_MAPSIZE) { + fprintf( + stderr, + "Database size is too large for current system (mapsize=%" PRIu64 + " is great than system-limit %zu)\n", + envinfo.mi_mapsize, (size_t)MAX_MAPSIZE); + goto env_close; + } + rc = mdbx_env_set_geometry( + env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize, + (intptr_t)envinfo.mi_mapsize, 0, 0, + envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); + } + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_set_geometry", rc); + goto env_close; + } + } + + rc = mdbx_env_open(env, envname, envflags, 0664); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_open", rc); + goto env_close; + } + + kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, MDBX_DUPSORT); + if (kbuf.iov_len >= INTPTR_MAX / 4) { + fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n", + kbuf.iov_len); + goto env_close; + } + kbuf.iov_len = (kbuf.iov_len + 1) * 2; + kbuf.iov_base = malloc(kbuf.iov_len * 2); + k0buf.iov_len = kbuf.iov_len; + k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len; + prevk.iov_base = k0buf.iov_base; + + while (rc == MDBX_SUCCESS) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + + rc = mdbx_txn_begin(env, nullptr, 0, &txn); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_begin", rc); + goto env_close; + } + + if (mode & GLOBAL) { + mode -= GLOBAL; + if (canary.v | canary.x | canary.y | canary.z) { + rc = mdbx_canary_put(txn, &canary); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_canary_put", rc); + goto txn_abort; + } + } + } + + const char *const dbi_name = subname ? subname : "@MAIN"; + rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, + append ? equal_or_greater : nullptr, + append ? equal_or_greater : nullptr); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_open_ex", rc); + goto txn_abort; + } + + uint64_t present_sequence; + rc = mdbx_dbi_sequence(txn, dbi, &present_sequence, 0); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_sequence", rc); + goto txn_abort; + } + if (present_sequence > sequence) { + fprintf(stderr, + "present sequence for '%s' value (%" PRIu64 + ") is greated than loaded (%" PRIu64 ")\n", + dbi_name, present_sequence, sequence); + rc = MDBX_RESULT_TRUE; + goto txn_abort; + } + if (present_sequence < sequence) { + rc = mdbx_dbi_sequence(txn, dbi, nullptr, sequence - present_sequence); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_sequence", rc); + goto txn_abort; + } + } + + rc = mdbx_cursor_open(txn, dbi, &mc); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_open", rc); + goto txn_abort; + } + /* if (append) { + mc->mc_flags |= C_SKIPORD; + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; + } */ + + int batch = 0; + prevk.iov_len = 0; + while (rc == MDBX_SUCCESS) { + MDBX_val key, data; + rc = readline(&key, &kbuf); + if (rc == EOF) + break; + + if (rc == MDBX_SUCCESS) + rc = readline(&data, &dbuf); + if (rc) { + fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", + prog, lineno); + goto txn_abort; + } + + int appflag = 0; + if (append) { + appflag = MDBX_APPEND; + if (dbi_flags & MDBX_DUPSORT) { + if (prevk.iov_len == key.iov_len && + memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0) + appflag = MDBX_APPEND | MDBX_APPENDDUP; + else + memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len); + } + } + rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag); + if (rc == MDBX_KEYEXIST && putflags) + continue; + if (rc == MDBX_BAD_VALSIZE && rescue) { + fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, + mdbx_strerror(rc)); + continue; + } + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_put", rc); + goto txn_abort; + } + batch++; + + MDBX_txn_info txn_info; + rc = mdbx_txn_info(txn, &txn_info, false); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_info", rc); + goto txn_abort; + } + + if (batch == 10000 || txn_info.txn_space_dirty > MEGABYTE * 16) { + mdbx_cursor_close(mc); + mc = nullptr; + rc = mdbx_txn_commit(txn); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_commit", rc); + goto env_close; + } + batch = 0; + + rc = mdbx_txn_begin(env, nullptr, 0, &txn); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_begin", rc); + goto env_close; + } + rc = mdbx_cursor_open(txn, dbi, &mc); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_open", rc); + goto txn_abort; + } + /* if (append) { + mc->mc_flags |= C_SKIPORD; + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; + } */ + } + } + + mdbx_cursor_close(mc); + mc = nullptr; + rc = mdbx_txn_commit(txn); + txn = nullptr; + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_commit", rc); + goto env_close; + } + rc = mdbx_dbi_close(env, dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_close", rc); + goto env_close; + } + + /* try read next header */ + if (!(mode & NOHDR)) + rc = readhdr(); + } + + switch (rc) { + case EOF: + rc = MDBX_SUCCESS; + case MDBX_SUCCESS: + break; + case MDBX_EINTR: + fprintf(stderr, "Interrupted by signal/user\n"); + break; + default: + if (unlikely(rc != MDBX_SUCCESS)) + error("readline", rc); + } + +txn_abort: + mdbx_cursor_close(mc); + mdbx_txn_abort(txn); +env_close: + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libs/libmdbx/src/mdbx_stat.c b/libs/libmdbx/src/mdbx_stat.c new file mode 100644 index 0000000000..37ebc55a00 --- /dev/null +++ b/libs/libmdbx/src/mdbx_stat.c @@ -0,0 +1,3587 @@ +/* mdbx_stat.c - memory-mapped database status tool */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. */ + +#define MDBX_BUILD_SOURCERY 3b5677a6062b714f1e138b0066c5590ee3c9ebf3bf8cfa3bb9503515ea0d1f02_v0_9_1_18_g1d31ebdc1c +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif (defined(__linux__) || defined(__gnu_linux__)) && !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_FULL_VER < 190024234 + /* Actually libmdbx was not tested with compilers older than 19.00.24234 (Visual Studio 2015 Update 3). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for alignment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant aggregate initializer */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ + +#if defined(MDBX_TOOLS) +#undef MDBX_DEPRECATED +#define MDBX_DEPRECATED +#endif /* MDBX_TOOLS */ + +#include "mdbx.h" +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# else +# define __noinline +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ + (!defined(__clang__) || __clang__ > 4)) || __cplusplus >= 201703L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L +# define __fallthrough [[fallthrough]] +# elif __GNUC_PREREQ(7, 0) && \ + (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +# define __fallthrough __attribute__((__fallthrough__)) +# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ + __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define __fallthrough [[clang::fallthrough]] +# else +# define __fallthrough +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline __maybe_unused const void* MDBX_PURE_FUNCTION + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include <valgrind/memcheck.h> +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include <sanitizer/asan_interface.h> +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template <typename T, size_t N> + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_STATIC_ASSERT) +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif defined(_MSC_VER) +# include <crtdbg.h> +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wnested-anon-types") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wnested-anon-types" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wnested-anon-types" +# else +# pragma warning disable "nested-anon-types" +# endif +#endif /* -Wnested-anon-types */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) && MDBX_AVOID_CRT +#define _NO_CRT_STDIO_INLINE +#endif +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> + +#include <assert.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <string.h> +#include <time.h> + +/* C11 stdalign.h */ +#if __has_include(<stdalign.h>) +#include <stdalign.h> +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif /* Apple OSX & iOS */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) +#include <sys/cdefs.h> +#include <sys/mount.h> +#include <sys/sysctl.h> +#include <sys/types.h> +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <vm/vm_param.h> +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include <uvm/uvm_param.h> +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#ifndef __MACH__ +#include <sys/vmmeter.h> +#endif +#else +#include <malloc.h> +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ + defined(_WIN32) || defined(_WIN64)) +#include <mntent.h> +#endif /* !Solaris */ +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) +#include <malloc_np.h> +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) +#include <malloc/malloc.h> +#endif /* MacOS */ + +#if defined(__MACH__) +#include <mach/host_info.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> +#include <uuid/uuid.h> +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include <linux/sysctl.h> +#include <sched.h> +#include <sys/sendfile.h> +#include <sys/statfs.h> +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include <utmpx.h> +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include <kstat.h> +#include <sys/mnttab.h> +/* On Solaris, it's easier to add a missing prototype rather than find a + * combination of #defines that break nothing. */ +__extern_C key_t ftok(const char *, int); +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <tlhelp32.h> +#include <windows.h> +#include <winnt.h> +#include <winternl.h> +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event[2]; +} mdbx_condpair_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <sys/file.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/uio.h> +#include <unistd.h> +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond[2]; +} mdbx_condpair_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +#ifdef __ANDROID_API__ +#include <android/log.h> +#if __ANDROID_API__ >= 21 +#include <sys/sendfile.h> +#endif +#endif /* Android */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) +#include <sys/stat.h> +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) +#include <sys/types.h> +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) +#include <sys/file.h> +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include <intrin.h> +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include <x86intrin.h> +#endif /* __ia32__ */ +#if defined(__ia32__) +#include <cpuid.h> +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include <mbarrier.h> +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include <machine/sys/inline.h> +#elif defined(__IBMC__) && defined(__powerpc) +#include <atomic.h> +#elif defined(_AIX) +#include <builtins.h> +#include <sys/atomic_op.h> +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include <c_asm.h> +#include <machine/builtins.h> +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include <machine/inline.h> +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline size_t +mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; +#endif /* Windows */ + +#ifdef __cplusplus +extern void mdbx_osal_jitter(bool tiny); +#else +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +#if __has_include(<sys/cachectl.h>) +#include <sys/cachectl.h> +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include <sys/cachectl.h> +#endif + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC MDBX_PRINTF_ARGS(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +MDBX_INTERNAL_VAR bool + mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; +#endif /* Linux */ + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); +MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, + bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); +MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_NONE = 0, + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_fsync(mdbx_filehandle_t fd, + const enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); + +enum mdbx_openfile_purpose { + MDBX_OPEN_DXB_READ = 0, + MDBX_OPEN_DXB_LAZY = 1, + MDBX_OPEN_DXB_DSYNC = 2, + MDBX_OPEN_LCK = 3, + MDBX_OPEN_COPY = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, + const MDBX_env *env, const char *pathname, + mdbx_filehandle_t *fd, + mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +#define MMAP_OPTION_TRUNCATE 1 +#define MMAP_OPTION_SEMAPHORE 2 +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const unsigned options); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna, const bool may_move); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, + const char *pathname, int err); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { + mdbx_tid_t thunk; + STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); +#if defined(_WIN32) || defined(_WIN64) + thunk = GetCurrentThreadId(); +#else + thunk = pthread_self(); +#endif + return (uintptr_t)thunk; +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by argument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (especially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); +MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; + +static __inline bool mdbx_RunningUnderWine(void) { + return !mdbx_NtExtendSection; +} + +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, + LPCSTR lpValue, DWORD dwFlags, + LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData); +MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include <stdatomic.h> +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include <libkern/OSAtomic.h> +#else +#error FIXME atomic-ops +#endif + +#endif /* !__cplusplus */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/******************************************************************************* + ******************************************************************************* + ******************************************************************************* + * + * + * #### ##### ##### # #### # # #### + * # # # # # # # # ## # # + * # # # # # # # # # # # #### + * # # ##### # # # # # # # # + * # # # # # # # # ## # # + * #### # # # #### # # #### + * + * + */ + +/** \defgroup build_option Build options + * The libmdbx build options. + @{ */ + +#ifdef DOXYGEN +/* !!! Actually this is a fake definitions !!! + * !!! for documentation generation by Doxygen !!! */ + +/** Controls enabling of debugging features. + * + * - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all, + * including logging and assertion controls. + * Logging level and corresponding debug flags changing + * by \ref mdbx_setup_debug() will not have effect. + * - `MDBX_DEBUG > 0` Enables code for the debugging features (logging, + * assertions checking and internal audit). + * Simultaneously sets the default logging level + * to the `MDBX_DEBUG` value. + * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. + * + * \ingroup build_option */ +#define MDBX_DEBUG 0...7 + +/** Disables using of GNU libc extensions. */ +#define MDBX_DISABLE_GNU_SOURCE 0 or 1 + +#endif /* DOXYGEN */ + +/** Enables support for huge write-transactions */ +#ifndef MDBX_HUGE_TRANSACTIONS +#define MDBX_HUGE_TRANSACTIONS 0 +#endif /* MDBX_HUGE_TRANSACTIONS */ + +/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_DURABILITY 0 +/** Using fsync() with chance of data lost on power failure */ +#define MDBX_OSX_WANNA_SPEED 1 + +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED + * for OSX & iOS */ +#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ + +/** Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_ENV_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be omitted: + * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_ENV_CHECKPID 0 +#else +#define MDBX_ENV_CHECKPID 1 +#endif +#define MDBX_ENV_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_ENV_CHECKPID) +#else +#define MDBX_ENV_CHECKPID_CONFIG STRINGIFY(MDBX_ENV_CHECKPID) +#endif /* MDBX_ENV_CHECKPID */ + +/** Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +/** Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +//------------------------------------------------------------------------------ + +/** Win32 File Locking API for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_WIN32FILES -1 + +/** SystemV IPC semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_SYSV 5 + +/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX1988 1988 + +/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2001 2001 + +/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_POSIX2008 2008 + +/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ +#define MDBX_LOCKING_BENAPHORE 1995 + +/** Advanced: Choices the locking implementation (autodetection by default). */ +#if defined(_WIN32) || defined(_WIN64) +#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES +#else +#ifndef MDBX_LOCKING +#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ + _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ + _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ + _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 +#else +#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 +#endif +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 +#else +#define MDBX_LOCKING MDBX_LOCKING_SYSV +#endif +#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) +#else +#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) +#endif /* MDBX_LOCKING */ +#endif /* !Windows */ + +/** Advanced: Using POSIX OFD-locks (autodetection by default). */ +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) && \ + !defined(__sun) /* OFD-lock are broken on Solaris */ +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/** Advanced: Using sendfile() syscall (autodetection by default). */ +#ifndef MDBX_USE_SENDFILE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) +#define MDBX_USE_SENDFILE 1 +#else +#define MDBX_USE_SENDFILE 0 +#endif +#endif /* MDBX_USE_SENDFILE */ + +/** Advanced: Using copy_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_COPYFILERANGE +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) +#define MDBX_USE_COPYFILERANGE 1 +#else +#define MDBX_USE_COPYFILERANGE 0 +#endif +#endif /* MDBX_USE_COPYFILERANGE */ + +/** Advanced: Using sync_file_range() syscall (autodetection by default). */ +#ifndef MDBX_USE_SYNCFILERANGE +#if ((defined(__linux__) || defined(__gnu_linux__)) && \ + defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ + (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) +#define MDBX_USE_SYNCFILERANGE 1 +#else +#define MDBX_USE_SYNCFILERANGE 0 +#endif +#endif /* MDBX_USE_SYNCFILERANGE */ + +//------------------------------------------------------------------------------ + +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) || defined(DOXYGEN) +#define MDBX_CPU_WRITEBACK_INCOHERENT 0 +#else +#define MDBX_CPU_WRITEBACK_INCOHERENT 1 +#endif +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#ifdef __OpenBSD__ +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 +#else +#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 +#endif +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +/** @} end of build options */ +/******************************************************************************* + ******************************************************************************* + ******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) +#define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) +#define INVALID_TXNID UINT64_MAX +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + MDBX_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + bin128_t mm_bootid; + +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + __anonymous_struct_extension__ struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + indx_t mp_ptrs[] /* dynamic size */; +#endif /* C99 */ +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES +#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) +typedef void mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t mdbx_ipclock_t; +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#define MDBX_CLOCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t mdbx_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) +typedef sem_t mdbx_ipclock_t; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) +MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); +MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); +#endif /* MDBX_LOCKING */ + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Write transaction lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_wlock; +#endif /* MDBX_LOCKING > 0 */ + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + + /* Readeaders registration lock. */ +#if MDBX_LOCKING > 0 + mdbx_ipclock_t mti_rlock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[] /* dynamic size */; +#endif /* C99 */ +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + __anonymous_struct_extension__ struct { + pgno_t pgno; + MDBX_page *ptr; + }; + __anonymous_struct_extension__ struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#if MDBX_HUGE_TRANSACTIONS +#define MDBX_PNL_MAX \ + ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) +#else +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) +#endif /* MDBX_HUGE_TRANSACTIONS */ + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ + size_t md_klen_min, md_klen_max; /* min/max key length for the database */ + size_t md_vlen_min, + md_vlen_max; /* min/max value/data length for the database */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) +#define MDBX_TXN_RW_BEGIN_FLAGS \ + (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + +#define TXN_FLAGS \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ + MDBX_TXN_HAS_CHILD) + +#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ + ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ + MDBX_SHRINK_ALLOWED) +#error "Oops, some flags overlapped or wrong" +#endif + + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + + /* Transaction DBI Flags */ +#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ +#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ +#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ +#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ +#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ +#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* Array of flags for each DB */ + uint8_t *mt_dbistate; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + MDBX_canary mt_canary; + void *mt_userctx; /* User-settable context */ + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbistate for this database */ + uint8_t *mc_dbistate; + unsigned mc_snum; /* number of pushed pages */ + unsigned mc_top; /* index of top page, normally mc_snum-1 */ + + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + + /* Cursor checking flags. */ +#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ +#define C_UPDATING 0x200 /* update/rebalance pending */ +#define C_RETIRING 0x400 /* refs to child pages may be invalid */ +#define C_SKIPORD 0x800 /* don't check keys ordering */ + + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbistate for this Dup DB */ + uint8_t mx_dbistate; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + void *mc_userctx; /* User-settable context */ + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + uint32_t me_signature; + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + /* Legacy MDBX_MAPASYNC (prior v0.9) */ +#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) +#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) + uint32_t me_flags; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_lazy_fd me_dxb_mmap.fd + mdbx_filehandle_t me_dsync_fd; + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + unsigned me_psize; /* DB page size, inited from me_os_psize */ + uint8_t me_psize2log; /* log2 of DB page size */ + int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ + + /* write-txn lock */ +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING > 0 + mdbx_ipclock_t *me_wlock; +#endif /* MDBX_LOCKING > 0 */ + + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + unsigned me_branch_nodemax; /* max size of a branch-node */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ + struct { +#if MDBX_LOCKING > 0 + mdbx_ipclock_t wlock; +#endif /* MDBX_LOCKING > 0 */ + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +#ifndef __cplusplus +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +extern uint8_t mdbx_runtime_flags; +extern uint8_t mdbx_loglevel; +extern MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) + mdbx_debug_log(int level, const char *function, int line, const char *fmt, + ...) MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL_FUNC void mdbx_debug_log_va(int level, const char *function, + int line, const char *fmt, + va_list args); + +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#endif /* MDBX_DEBUG */ + +#if !MDBX_DEBUG && defined(__ANDROID_API__) +#define mdbx_assert_fail(env, msg, func, line) \ + __android_log_assert(msg, "mdbx", "%s:%u", func, line) +#else +void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, + int line); +#endif + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (MDBX_DEBUG && mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ + +#if MDBX_CPU_WRITEBACK_INCOHERENT +#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() +#else +#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +static __maybe_unused __inline void +mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = + (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + mdbx_assert(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#if MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_cleanup_dead_readers(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +#endif /* !__cplusplus */ + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) + +/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + }; +#endif /* __BYTE_ORDER__ */ + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ + (!defined(__cplusplus) && defined(_MSC_VER)) + uint8_t mn_data[] /* key and data are appended here */; +#endif /* C99 */ +} MDBX_node; + +#define DB_PERSISTENT_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP) + +/* mdbx_dbi_open() flags */ +#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) + +#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define DB_INTERNAL_FLAGS DB_VALID + +#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif +#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS +#error "Oops, some flags overlapped or wrong" +#endif + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __maybe_unused __inline pgno_t +pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused bool +is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} + +MDBX_NOTHROW_CONST_FUNCTION static __always_inline __maybe_unused size_t +ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define ENV_CHANGEABLE_FLAGS \ + (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ + MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE) +#define ENV_CHANGELESS_FLAGS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) +#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) + +#if !defined(__cplusplus) || defined(__cpp_constexpr) +static __maybe_unused void static_checks(void) { + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, + "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & + (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, + "Oops, some flags overlapped or wrong"); +} +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +#ifdef __cplusplus +} +#endif + +#if defined(_WIN32) || defined(_WIN64) +/* + * POSIX getopt for Windows + * + * AT&T Public License + * + * Code given out at the 1985 UNIFORUM conference in Dallas. + */ + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#endif /* _MSC_VER (warnings) */ + +#include <stdio.h> +#include <string.h> + +#ifdef _MSC_VER +#pragma warning(pop) +#endif +/*----------------------------------------------------------------------------*/ + +#ifndef NULL +#define NULL 0 +#endif + +#ifndef EOF +#define EOF (-1) +#endif + +int optind = 1; +int optopt; +char *optarg; + +int getopt(int argc, char *const argv[], const char *opts) { + static int sp = 1; + int c; + const char *cp; + + if (sp == 1) { + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return EOF; + else if (strcmp(argv[optind], "--") == 0) { + optind++; + return EOF; + } + } + optopt = c = argv[optind][sp]; + if (c == ':' || (cp = strchr(opts, c)) == NULL) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "illegal option", c); + if (argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return '?'; + } + if (*++cp == ':') { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) { + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", + c); + sp = 1; + return '?'; + } else + optarg = argv[optind++]; + sp = 1; + } else { + if (argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return c; +} + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +static void print_stat(MDBX_stat *ms) { + printf(" Pagesize: %u\n", ms->ms_psize); + printf(" Tree depth: %u\n", ms->ms_depth); + printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages); + printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages); + printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages); + printf(" Entries: %" PRIu64 "\n", ms->ms_entries); +} + +static void usage(const char *prog) { + fprintf(stderr, + "usage: %s [-V] [-e] [-f[f[f]]] [-r[r]] [-a|-s name] dbpath\n" + " -V\t\tprint version and exit\n" + " -e\t\tshow whole DB info\n" + " -f\t\tshow GC info\n" + " -r\t\tshow readers\n" + " -a\t\tprint stat of main DB and all subDBs\n" + " \t\t(default) print stat of only the main DB\n" + " -s name\tprint stat of only the named subDB\n", + prog); + exit(EXIT_FAILURE); +} + +static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, + mdbx_tid_t thread, uint64_t txnid, uint64_t lag, + size_t bytes_used, size_t bytes_retained) { + (void)ctx; + if (num == 1) + printf("Reader Table\n" + " #\tslot\t%6s %*s %20s %10s %13s %13s\n", + "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", + "retained"); + + printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid, + (int)sizeof(size_t) * 2, (uintptr_t)thread); + if (txnid) + printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, + bytes_used / 1048576.0, bytes_retained / 1048576.0); + else + printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0"); + + return user_break ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; +} + +const char *prog; +static void error(const char *func, int rc) { + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); +} + +int main(int argc, char *argv[]) { + int o, rc; + MDBX_env *env; + MDBX_txn *txn; + MDBX_dbi dbi; + MDBX_stat mst; + MDBX_envinfo mei; + prog = argv[0]; + char *envname; + char *subname = nullptr; + int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; + + if (argc < 2) + usage(prog); + + while ((o = getopt(argc, argv, "Vaefnrs:")) != EOF) { + switch (o) { + case 'V': + printf("mdbx_stat version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'e': + envinfo++; + break; + case 'f': + freinfo++; + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 'r': + rdrinfo++; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + envname = argv[optind]; + printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname); + fflush(nullptr); + + rc = mdbx_env_create(&env); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_create", rc); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + rc = mdbx_env_set_maxdbs(env, 2); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_set_maxdbs", rc); + goto env_close; + } + } + + rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_open", rc); + goto env_close; + } + + rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_txn_begin", rc); + goto txn_abort; + } + + if (envinfo || freinfo) { + rc = mdbx_env_info_ex(env, txn, &mei, sizeof(mei)); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_info_ex", rc); + goto txn_abort; + } + } else { + /* LY: zap warnings from gcc */ + memset(&mei, 0, sizeof(mei)); + } + + if (envinfo) { + rc = mdbx_env_stat_ex(env, txn, &mst, sizeof(mst)); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_env_stat_ex", rc); + goto txn_abort; + } + printf("Environment Info\n"); + printf(" Pagesize: %u\n", mst.ms_psize); + if (mei.mi_geo.lower != mei.mi_geo.upper) { + printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 + "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 + "/-%" PRIu64 ")\n", + mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, + mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize, + mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize, + mei.mi_geo.shrink / mst.ms_psize); + printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", + mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize); + printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", + mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); +#if defined(_WIN32) || defined(_WIN64) + if (mei.mi_geo.shrink && mei.mi_geo.current != mei.mi_geo.upper) + printf(" WARNING: Due Windows system limitations a " + "file couldn't\n be truncated while database " + "is opened. So, the size of\n database file " + "may by large than the database itself,\n " + "until it will be closed or reopened in read-write mode.\n"); +#endif + } else { + printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", + mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); + } + printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); + printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n", + mei.mi_latter_reader_txnid, + mei.mi_latter_reader_txnid - mei.mi_recent_txnid); + printf(" Max readers: %u\n", mei.mi_maxreaders); + printf(" Number of reader slots uses: %u\n", mei.mi_numreaders); + } else { + /* LY: zap warnings from gcc */ + memset(&mst, 0, sizeof(mst)); + } + + if (rdrinfo) { + rc = mdbx_reader_list(env, reader_list_func, nullptr); + if (MDBX_IS_ERROR(rc)) { + error("mdbx_reader_list", rc); + goto txn_abort; + } + if (rc == MDBX_RESULT_TRUE) + printf("Reader Table is empty\n"); + else if (rc == MDBX_SUCCESS && rdrinfo > 1) { + int dead; + rc = mdbx_reader_check(env, &dead); + if (MDBX_IS_ERROR(rc)) { + error("mdbx_reader_check", rc); + goto txn_abort; + } + if (rc == MDBX_RESULT_TRUE) { + printf(" %d stale readers cleared.\n", dead); + rc = mdbx_reader_list(env, reader_list_func, nullptr); + if (rc == MDBX_RESULT_TRUE) + printf(" Now Reader Table is empty\n"); + } else + printf(" No stale readers.\n"); + } + if (!(subname || alldbs || freinfo)) + goto txn_abort; + } + + if (freinfo) { + printf("Garbage Collection\n"); + dbi = 0; + MDBX_cursor *cursor; + rc = mdbx_cursor_open(txn, dbi, &cursor); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_open", rc); + goto txn_abort; + } + rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst)); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_stat", rc); + goto txn_abort; + } + print_stat(&mst); + + pgno_t pages = 0, *iptr; + pgno_t reclaimable = 0; + MDBX_val key, data; + while (MDBX_SUCCESS == + (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + iptr = data.iov_base; + const pgno_t number = *iptr++; + + pages += number; + if (envinfo && mei.mi_latter_reader_txnid > *(txnid_t *)key.iov_base) + reclaimable += number; + + if (freinfo > 1) { + char *bad = ""; + pgno_t prev = + MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1; + pgno_t span = 1; + for (unsigned i = 0; i < number; ++i) { + pgno_t pg = iptr[i]; + if (MDBX_PNL_DISORDERED(prev, pg)) + bad = " [bad sequence]"; + prev = pg; + while (i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) + : pgno_sub(pg, span))) + ++span; + } + printf(" Transaction %" PRIaTXN ", %" PRIaPGNO + " pages, maxspan %" PRIaPGNO "%s\n", + *(txnid_t *)key.iov_base, number, span, bad); + if (freinfo > 2) { + for (unsigned i = 0; i < number; i += span) { + const pgno_t pg = iptr[i]; + for (span = 1; + i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) + : pgno_sub(pg, span)); + ++span) + ; + if (span > 1) + printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span); + else + printf(" %9" PRIaPGNO "\n", pg); + } + } + } + } + mdbx_cursor_close(cursor); + cursor = nullptr; + + switch (rc) { + case MDBX_SUCCESS: + case MDBX_NOTFOUND: + break; + case MDBX_EINTR: + fprintf(stderr, "Interrupted by signal/user\n"); + goto txn_abort; + default: + error("mdbx_cursor_get", rc); + goto txn_abort; + } + + if (envinfo) { + uint64_t value = mei.mi_mapsize / mst.ms_psize; + double percent = value / 100.0; + printf("Page Usage\n"); + printf(" Total: %" PRIu64 " 100%%\n", value); + + value = mei.mi_geo.current / mst.ms_psize; + printf(" Backed: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = mei.mi_last_pgno + 1; + printf(" Allocated: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1); + printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = mei.mi_last_pgno + 1 - pages; + printf(" Used: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = pages; + printf(" GC: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = pages - reclaimable; + printf(" Retained: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = reclaimable; + printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = + mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1) + reclaimable; + printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent); + } else + printf(" GC: %" PRIaPGNO " pages\n", pages); + } + + rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_open", rc); + goto txn_abort; + } + rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst)); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_stat", rc); + goto txn_abort; + } + printf("Status of %s\n", subname ? subname : "Main DB"); + print_stat(&mst); + + if (alldbs) { + MDBX_cursor *cursor; + rc = mdbx_cursor_open(txn, dbi, &cursor); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_open", rc); + goto txn_abort; + } + + MDBX_val key; + while (MDBX_SUCCESS == + (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { + MDBX_dbi subdbi; + if (memchr(key.iov_base, '\0', key.iov_len)) + continue; + subname = mdbx_malloc(key.iov_len + 1); + memcpy(subname, key.iov_base, key.iov_len); + subname[key.iov_len] = '\0'; + rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &subdbi); + if (rc == MDBX_SUCCESS) + printf("Status of %s\n", subname); + mdbx_free(subname); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_INCOMPATIBLE) + continue; + error("mdbx_dbi_open", rc); + goto txn_abort; + } + + rc = mdbx_dbi_stat(txn, subdbi, &mst, sizeof(mst)); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_stat", rc); + goto txn_abort; + } + print_stat(&mst); + + rc = mdbx_dbi_close(env, subdbi); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_dbi_close", rc); + goto txn_abort; + } + } + mdbx_cursor_close(cursor); + cursor = nullptr; + } + + switch (rc) { + case MDBX_SUCCESS: + case MDBX_NOTFOUND: + break; + case MDBX_EINTR: + fprintf(stderr, "Interrupted by signal/user\n"); + break; + default: + if (unlikely(rc != MDBX_SUCCESS)) + error("mdbx_cursor_get", rc); + } + + mdbx_dbi_close(env, dbi); +txn_abort: + mdbx_txn_abort(txn); +env_close: + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libs/libmdbx/src/src/ntdll.def b/libs/libmdbx/src/ntdll.def index e3a6e33c94..e3a6e33c94 100644 --- a/libs/libmdbx/src/src/ntdll.def +++ b/libs/libmdbx/src/ntdll.def diff --git a/libs/libmdbx/src/packages/rpm/CMakeLists.txt b/libs/libmdbx/src/packages/rpm/CMakeLists.txt deleted file mode 100644 index 5949e9f0ac..0000000000 --- a/libs/libmdbx/src/packages/rpm/CMakeLists.txt +++ /dev/null @@ -1,184 +0,0 @@ -cmake_minimum_required(VERSION 2.8.7) -set(TARGET mdbx) -project(${TARGET}) - -set(MDBX_VERSION_MAJOR 0) -set(MDBX_VERSION_MINOR 3) -set(MDBX_VERSION_RELEASE 1) -set(MDBX_VERSION_REVISION 0) - -set(MDBX_VERSION_STRING ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE}) - -enable_language(C) -enable_language(CXX) - -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED on) - -add_definitions(-DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1 -D_GNU_SOURCE=1) - -find_package(Threads REQUIRED) - -get_directory_property(hasParent PARENT_DIRECTORY) -if(hasParent) - set(STANDALONE_BUILD 0) -else() - set(STANDALONE_BUILD 1) - enable_testing() - - if (CMAKE_C_COMPILER_ID MATCHES GNU) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g3") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wextra") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread") - endif() - - if (CMAKE_CXX_COMPILER_ID MATCHES GNU) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpointer-arith") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-security") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Woverloaded-virtual") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wwrite-strings") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=20") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wunused-function -Wunused-variable -Wunused-value -Wmissing-declarations") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-qual") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finline-functions-called-once") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat") - - set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g3") - endif() - - if (COVERAGE) - if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") - message(FATAL_ERROR "Coverage requires -DCMAKE_BUILD_TYPE=Debug Current value=${CMAKE_BUILD_TYPE}") - endif() - - message(STATUS "Setting coverage compiler flags") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage") - add_definitions(-DCOVERAGE_TEST) - endif() - - if (NOT TRAVIS) - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fstack-protector-strong -static-libasan") - endif() -endif() - -set(${TARGET}_SRC - mdbx.h - src/bits.h - src/defs.h - src/lck-linux.c - src/mdbx.c - src/osal.c - src/osal.h - src/version.c - ) - -add_library(${TARGET}_STATIC STATIC - ${${TARGET}_SRC} - ) - -add_library(${TARGET} ALIAS ${TARGET}_STATIC) - -add_library(${TARGET}_SHARED SHARED - ${${TARGET}_SRC} - ) - -set_target_properties(${TARGET}_SHARED PROPERTIES - VERSION ${MDBX_VERSION_STRING} - SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR} - OUTPUT_NAME ${TARGET} - CLEAN_DIRECT_OUTPUT 1 - ) - -set_target_properties(${TARGET}_STATIC PROPERTIES - VERSION ${MDBX_VERSION_STRING} - SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR} - OUTPUT_NAME ${TARGET} - CLEAN_DIRECT_OUTPUT 1 - ) - -target_include_directories(${TARGET}_STATIC PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}) -target_include_directories(${TARGET}_SHARED PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}) - -target_link_libraries(${TARGET}_STATIC ${CMAKE_THREAD_LIBS_INIT}) -target_link_libraries(${TARGET}_SHARED ${CMAKE_THREAD_LIBS_INIT}) -if(UNIX AND NOT APPLE) - target_link_libraries(${TARGET}_STATIC rt) - target_link_libraries(${TARGET}_SHARED rt) -endif() - -install(TARGETS ${TARGET}_STATIC DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx) -install(TARGETS ${TARGET}_SHARED DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx) -install(FILES mdbx.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT mdbx-devel) - -add_subdirectory(src/tools) -add_subdirectory(test) -add_subdirectory(test/pcrf) -add_subdirectory(tutorial) - -############################################################################## - -set(CPACK_GENERATOR "RPM") -set(CPACK_RPM_COMPONENT_INSTALL ON) - -# Version -if (NOT "$ENV{BUILD_NUMBER}" STREQUAL "") - set(CPACK_PACKAGE_RELEASE $ENV{BUILD_NUMBER}) -else() - if (NOT "$ENV{CI_PIPELINE_ID}" STREQUAL "") - set(CPACK_PACKAGE_RELEASE $ENV{CI_PIPELINE_ID}) - else() - set(CPACK_PACKAGE_RELEASE 1) - endif() -endif() -set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE}) - -set(CPACK_PACKAGE_VERSION ${MDBX_VERSION_STRING}) -set(CPACK_PACKAGE_VERSION_FULL ${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}) - -set(CPACK_RPM_mdbx-devel_PACKAGE_REQUIRES "mdbx = ${CPACK_PACKAGE_VERSION}") - -set(CPACK_RPM_SPEC_INSTALL_POST "/bin/true") -set(CPACK_RPM_mdbx_PACKAGE_NAME mdbx) -set(CPACK_RPM_mdbx-devel_PACKAGE_NAME mdbx-devel) -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "The revised and extended descendant of Symas LMDB") - -set(CPACK_PACKAGE_VENDOR "???") -set(CPACK_PACKAGE_CONTACT "Vladimir Romanov") -set(CPACK_PACKAGE_RELOCATABLE false) -set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64") -set(CPACK_RPM_PACKAGE_REQUIRES "") -set(CPACK_RPM_PACKAGE_GROUP "Applications/Database") - -set(CPACK_RPM_mdbx_FILE_NAME "${CPACK_RPM_mdbx_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm") -set(CPACK_RPM_mdbx-devel_FILE_NAME "${CPACK_RPM_mdbx-devel_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm") - -set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION - /usr/local - /usr/local/bin - /usr/local/lib64 - /usr/local/include - /usr/local/man - /usr/local/man/man1 - ) - -include(CPack) diff --git a/libs/libmdbx/src/packages/rpm/build.sh b/libs/libmdbx/src/packages/rpm/build.sh deleted file mode 100644 index 5170882265..0000000000 --- a/libs/libmdbx/src/packages/rpm/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -e -CONFIG=$1 - -if [[ -z "${CONFIG}" ]]; then - CONFIG=Debug -fi -if [[ -r /opt/rh/devtoolset-6/enable ]]; then - source /opt/rh/devtoolset-6/enable -fi -#rm -f -r build || true -mkdir -p cmake-build-${CONFIG} -pushd cmake-build-${CONFIG} &> /dev/null -if [[ ! -r Makefile ]]; then - cmake .. -DCMAKE_BUILD_TYPE=${CONFIG} -fi -make -j8 || exit 1 -popd &> /dev/null diff --git a/libs/libmdbx/src/packages/rpm/package.sh b/libs/libmdbx/src/packages/rpm/package.sh deleted file mode 100644 index d7f9ab297a..0000000000 --- a/libs/libmdbx/src/packages/rpm/package.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e - -CONFIG=$1 - -if [[ -z "${CONFIG}" ]]; then - CONFIG=Debug -fi - -DIRNAME=`dirname ${BASH_SOURCE[0]}` -DIRNAME=`readlink --canonicalize ${DIRNAME}` - -if [[ -r /opt/rh/devtoolset-6/enable ]]; then - source /opt/rh/devtoolset-6/enable -fi - -mkdir -p cmake-build-${CONFIG} -pushd cmake-build-${CONFIG} &> /dev/null -if [[ ! -r Makefile ]]; then - cmake .. -DCMAKE_BUILD_TYPE=${CONFIG} -fi -rm -f *.rpm -make -j8 package || exit 1 -rm -f *-Unspecified.rpm -popd &> /dev/null diff --git a/libs/libmdbx/src/src/alloy.c b/libs/libmdbx/src/src/alloy.c deleted file mode 100644 index c2b512005d..0000000000 --- a/libs/libmdbx/src/src/alloy.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#define MDBX_ALLOY 1 /* alloyed build */ -#include "internals.h" /* must be included first */ - -#include "core.c" -#include "osal.c" -#include "version.c" - -#if defined(_WIN32) || defined(_WIN64) -#include "lck-windows.c" -#else -#include "lck-posix.c" -#endif diff --git a/libs/libmdbx/src/src/debug_begin.h b/libs/libmdbx/src/src/debug_begin.h deleted file mode 100644 index f3306b5dc0..0000000000 --- a/libs/libmdbx/src/src/debug_begin.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma push_macro("mdbx_trace") -#pragma push_macro("mdbx_debug") -#pragma push_macro("mdbx_verbose") -#pragma push_macro("mdbx_notice") -#pragma push_macro("mdbx_warning") -#pragma push_macro("mdbx_error") -#pragma push_macro("mdbx_assert") - -#undef mdbx_trace -#define mdbx_trace(fmt, ...) \ - mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__) - -#undef mdbx_debug -#define mdbx_debug(fmt, ...) \ - mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__) - -#undef mdbx_verbose -#define mdbx_verbose(fmt, ...) \ - mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__) - -#undef mdbx_notice -#define mdbx_notice(fmt, ...) \ - mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__) - -#undef mdbx_warning -#define mdbx_warning(fmt, ...) \ - mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__) - -#undef mdbx_error -#define mdbx_error(fmt, ...) \ - mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__) - -#undef mdbx_assert -#define mdbx_assert(env, expr) mdbx_ensure(env, expr) diff --git a/libs/libmdbx/src/src/debug_end.h b/libs/libmdbx/src/src/debug_end.h deleted file mode 100644 index e361119bc0..0000000000 --- a/libs/libmdbx/src/src/debug_end.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma pop_macro("mdbx_trace") -#pragma pop_macro("mdbx_debug") -#pragma pop_macro("mdbx_verbose") -#pragma pop_macro("mdbx_notice") -#pragma pop_macro("mdbx_warning") -#pragma pop_macro("mdbx_error") -#pragma pop_macro("mdbx_assert") diff --git a/libs/libmdbx/src/src/defs.h b/libs/libmdbx/src/src/defs.h deleted file mode 100644 index f4672d5b99..0000000000 --- a/libs/libmdbx/src/src/defs.h +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once -/* *INDENT-OFF* */ -/* clang-format off */ - -#ifndef __GNUC_PREREQ -# if defined(__GNUC__) && defined(__GNUC_MINOR__) -# define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) -# else -# define __GNUC_PREREQ(maj, min) (0) -# endif -#endif /* __GNUC_PREREQ */ - -#ifndef __CLANG_PREREQ -# ifdef __clang__ -# define __CLANG_PREREQ(maj,min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) -# else -# define __CLANG_PREREQ(maj,min) (0) -# endif -#endif /* __CLANG_PREREQ */ - -#ifndef __GLIBC_PREREQ -# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -# define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) -# else -# define __GLIBC_PREREQ(maj, min) (0) -# endif -#endif /* __GLIBC_PREREQ */ - -#ifndef __has_attribute -# define __has_attribute(x) (0) -#endif - -#ifndef __has_feature -# define __has_feature(x) (0) -#endif - -#ifndef __has_extension -# define __has_extension(x) (0) -#endif - -#ifndef __has_builtin -# define __has_builtin(x) (0) -#endif - -#ifndef __has_warning -# define __has_warning(x) (0) -#endif - -#ifndef __has_include -# define __has_include(x) (0) -#endif - -#ifndef __has_cpp_attribute -# define __has_cpp_attribute(x) (0) -#endif - -#if __has_feature(thread_sanitizer) -# define __SANITIZE_THREAD__ 1 -#endif - -#if __has_feature(address_sanitizer) -# define __SANITIZE_ADDRESS__ 1 -#endif - -/*----------------------------------------------------------------------------*/ - -#ifndef __extern_C -# ifdef __cplusplus -# define __extern_C extern "C" -# else -# define __extern_C -# endif -#endif /* __extern_C */ - -#ifndef __cplusplus -# ifndef bool -# define bool _Bool -# endif -# ifndef true -# define true (1) -# endif -# ifndef false -# define false (0) -# endif -#endif - -#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) -# define nullptr NULL -#endif - -/*----------------------------------------------------------------------------*/ - -#ifndef __always_inline -# if defined(__GNUC__) || __has_attribute(__always_inline__) -# define __always_inline __inline __attribute__((__always_inline__)) -# elif defined(_MSC_VER) -# define __always_inline __forceinline -# else -# define __always_inline -# endif -#endif /* __always_inline */ - -#ifndef __noinline -# if defined(__GNUC__) || __has_attribute(__noinline__) -# define __noinline __attribute__((__noinline__)) -# elif defined(_MSC_VER) -# define __noinline __declspec(noinline) -# else -# define __noinline -# endif -#endif /* __noinline */ - -#ifndef __must_check_result -# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) -# define __must_check_result __attribute__((__warn_unused_result__)) -# else -# define __must_check_result -# endif -#endif /* __must_check_result */ - -#ifndef __maybe_unused -# if defined(__GNUC__) || __has_attribute(__unused__) -# define __maybe_unused __attribute__((__unused__)) -# else -# define __maybe_unused -# endif -#endif /* __maybe_unused */ - -#if !defined(__noop) && !defined(_MSC_VER) -# define __noop(...) do {} while(0) -#endif /* __noop */ - -#ifndef __fallthrough -# if defined(__cplusplus) && __has_cpp_attribute(fallthrough) -# define __fallthrough [[fallthrough]] -# elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L -# define __fallthrough [[fallthrough]] -# elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) -# define __fallthrough __attribute__((__fallthrough__)) -# elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L &&\ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") -# define __fallthrough [[clang::fallthrough]] -# else -# define __fallthrough -# endif -#endif /* __fallthrough */ - -#ifndef __unreachable -# if __GNUC_PREREQ(4,5) || __has_builtin(__builtin_unreachable) -# define __unreachable() __builtin_unreachable() -# elif defined(_MSC_VER) -# define __unreachable() __assume(0) -# else -# define __unreachable() __noop() -# endif -#endif /* __unreachable */ - -#ifndef __prefetch -# if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) -# define __prefetch(ptr) __builtin_prefetch(ptr) -# else -# define __prefetch(ptr) __noop(ptr) -# endif -#endif /* __prefetch */ - -#ifndef __noreturn -# if defined(__GNUC__) || __has_attribute(__noreturn__) -# define __noreturn __attribute__((__noreturn__)) -# elif defined(_MSC_VER) -# define __noreturn __declspec(noreturn) -# else -# define __noreturn -# endif -#endif /* __noreturn */ - -#ifndef __nothrow -# if defined(__cplusplus) -# if __cplusplus < 201703L -# define __nothrow throw() -# else -# define __nothrow noexcept(true) -# endif /* __cplusplus */ -# elif defined(__GNUC__) || __has_attribute(__nothrow__) -# define __nothrow __attribute__((__nothrow__)) -# elif defined(_MSC_VER) && defined(__cplusplus) -# define __nothrow __declspec(nothrow) -# else -# define __nothrow -# endif -#endif /* __nothrow */ - -#ifndef __pure_function - /* Many functions have no effects except the return value and their - * return value depends only on the parameters and/or global variables. - * Such a function can be subject to common subexpression elimination - * and loop optimization just as an arithmetic operator would be. - * These functions should be declared with the attribute pure. */ -# if (defined(__GNUC__) || __has_attribute(__pure__)) && \ - (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \ - || !defined(__cplusplus) || !__has_feature(cxx_exceptions)) -# define __pure_function __attribute__((__pure__)) -# else -# define __pure_function -# endif -#endif /* __pure_function */ - -#ifndef __const_function - /* Many functions do not examine any values except their arguments, - * and have no effects except the return value. Basically this is just - * slightly more strict class than the PURE attribute, since function - * is not allowed to read global memory. - * - * Note that a function that has pointer arguments and examines the - * data pointed to must not be declared const. Likewise, a function - * that calls a non-const function usually must not be const. - * It does not make sense for a const function to return void. */ -# if (defined(__GNUC__) || __has_attribute(__pure__)) && \ - (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \ - || !defined(__cplusplus) || !__has_feature(cxx_exceptions)) -# define __const_function __attribute__((__const__)) -# else -# define __const_function -# endif -#endif /* __const_function */ - -#ifndef __hidden -# if defined(__GNUC__) || __has_attribute(__visibility__) -# define __hidden __attribute__((__visibility__("hidden"))) -# else -# define __hidden -# endif -#endif /* __hidden */ - -#ifndef __optimize -# if defined(__OPTIMIZE__) -# if (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__optimize__) -# define __optimize(ops) __attribute__((__optimize__(ops))) -# else -# define __optimize(ops) -# endif -# else -# define __optimize(ops) -# endif -#endif /* __optimize */ - -#ifndef __hot -# if defined(__OPTIMIZE__) -# if defined(__e2k__) -# define __hot __attribute__((__hot__)) __optimize(3) -# elif defined(__clang__) && !__has_attribute(__hot_) \ - && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) - /* just put frequently used functions in separate section */ -# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") -# elif defined(__GNUC__) || __has_attribute(__hot__) -# define __hot __attribute__((__hot__)) __optimize("O3") -# else -# define __hot __optimize("O3") -# endif -# else -# define __hot -# endif -#endif /* __hot */ - -#ifndef __cold -# if defined(__OPTIMIZE__) -# if defined(__e2k__) -# define __cold __attribute__((__cold__)) __optimize(1) -# elif defined(__clang__) && !__has_attribute(cold) \ - && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) - /* just put infrequently used functions in separate section */ -# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") -# elif defined(__GNUC__) || __has_attribute(cold) -# define __cold __attribute__((__cold__)) __optimize("Os") -# else -# define __cold __optimize("Os") -# endif -# else -# define __cold -# endif -#endif /* __cold */ - -#ifndef __flatten -# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) -# define __flatten __attribute__((__flatten__)) -# else -# define __flatten -# endif -#endif /* __flatten */ - -#ifndef likely -# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) -# define likely(cond) __builtin_expect(!!(cond), 1) -# else -# define likely(x) (x) -# endif -#endif /* likely */ - -#ifndef unlikely -# if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) -# define unlikely(cond) __builtin_expect(!!(cond), 0) -# else -# define unlikely(x) (x) -# endif -#endif /* unlikely */ - -#ifndef __printf_args -# if defined(__GNUC__) || __has_attribute(__format__) -# define __printf_args(format_index, first_arg) \ - __attribute__((__format__(__printf__, format_index, first_arg))) -# else -# define __printf_args(format_index, first_arg) -# endif -#endif /* __printf_args */ - -#ifndef __anonymous_struct_extension__ -# if defined(__GNUC__) -# define __anonymous_struct_extension__ __extension__ -# else -# define __anonymous_struct_extension__ -# endif -#endif /* __anonymous_struct_extension__ */ - -#ifndef __Wpedantic_format_voidptr - static __inline __maybe_unused const void* __pure_function - __Wpedantic_format_voidptr(const void* ptr) {return ptr;} -# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ - -/*----------------------------------------------------------------------------*/ - -#if defined(MDBX_USE_VALGRIND) -# include <valgrind/memcheck.h> -# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE - /* LY: available since Valgrind 3.10 */ -# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) -# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) -# endif -#elif !defined(RUNNING_ON_VALGRIND) -# define VALGRIND_CREATE_MEMPOOL(h,r,z) -# define VALGRIND_DESTROY_MEMPOOL(h) -# define VALGRIND_MEMPOOL_TRIM(h,a,s) -# define VALGRIND_MEMPOOL_ALLOC(h,a,s) -# define VALGRIND_MEMPOOL_FREE(h,a) -# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) -# define VALGRIND_MAKE_MEM_NOACCESS(a,s) -# define VALGRIND_MAKE_MEM_DEFINED(a,s) -# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) -# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) -# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) -# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) -# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) -# define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ - -#ifdef __SANITIZE_ADDRESS__ -# include <sanitizer/asan_interface.h> -#elif !defined(ASAN_POISON_MEMORY_REGION) -# define ASAN_POISON_MEMORY_REGION(addr, size) \ - ((void)(addr), (void)(size)) -# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - ((void)(addr), (void)(size)) -#endif /* __SANITIZE_ADDRESS__ */ - -/*----------------------------------------------------------------------------*/ - -#ifndef ARRAY_LENGTH -# ifdef __cplusplus - template <typename T, size_t N> - char (&__ArraySizeHelper(T (&array)[N]))[N]; -# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) -# else -# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) -# endif -#endif /* ARRAY_LENGTH */ - -#ifndef ARRAY_END -# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) -#endif /* ARRAY_END */ - -#ifndef STRINGIFY -# define STRINGIFY_HELPER(x) #x -# define STRINGIFY(x) STRINGIFY_HELPER(x) -#endif /* STRINGIFY */ - -#define CONCAT(a,b) a##b -#define XCONCAT(a,b) CONCAT(a,b) - -#ifndef offsetof -# define offsetof(type, member) __builtin_offsetof(type, member) -#endif /* offsetof */ - -#ifndef container_of -# define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) -#endif /* container_of */ - -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) - -#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) - -#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) - -#ifndef STATIC_ASSERT_MSG -# if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ - || __has_feature(c_static_assert) -# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) -# elif defined(static_assert) -# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) -# elif defined(_MSC_VER) -# include <crtdbg.h> -# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -# else -# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} -# endif -#endif /* STATIC_ASSERT */ - -#ifndef STATIC_ASSERT -# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) -#endif - -/* *INDENT-ON* */ -/* clang-format on */ diff --git a/libs/libmdbx/src/src/internals.h b/libs/libmdbx/src/src/internals.h deleted file mode 100644 index d9edc9d53a..0000000000 --- a/libs/libmdbx/src/src/internals.h +++ /dev/null @@ -1,1376 +0,0 @@ -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#pragma once -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif - -/* *INDENT-OFF* */ -/* clang-format off */ - -/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ -#ifndef MDBX_DEBUG -# ifdef NDEBUG -# define MDBX_DEBUG 0 -# else -# define MDBX_DEBUG 1 -# endif -#endif - -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -# undef NDEBUG -#endif - -#ifdef MDBX_ALLOY -/* Amalgamated build */ -# define MDBX_INTERNAL_FUNC static -# define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -# define MDBX_INTERNAL_FUNC -# define MDBX_INTERNAL_VAR extern -#endif /* MDBX_ALLOY */ - -#ifndef MDBX_DISABLE_GNU_SOURCE -#define MDBX_DISABLE_GNU_SOURCE 0 -#endif -#if MDBX_DISABLE_GNU_SOURCE -#undef _GNU_SOURCE -#elif defined(__linux__) || defined(__gnu_linux__) -#define _GNU_SOURCE -#endif - -/*----------------------------------------------------------------------------*/ - -/* Should be defined before any includes */ -#ifndef _FILE_OFFSET_BITS -# define _FILE_OFFSET_BITS 64 -#endif - -#ifdef __APPLE__ -#define _DARWIN_C_SOURCE -#endif - -#ifdef _MSC_VER -# if _MSC_VER < 1400 -# error "Microsoft Visual C++ 8.0 (Visual Studio 2005) or later version is required" -# endif -# ifndef _CRT_SECURE_NO_WARNINGS -# define _CRT_SECURE_NO_WARNINGS -# endif -#if _MSC_VER > 1800 -# pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif -#if _MSC_VER > 1913 -# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ -#endif -#pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ -#pragma warning(disable : 4702) /* unreachable code */ -#pragma warning(disable : 4706) /* assignment within conditional expression */ -#pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ -#pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for aligment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ -#endif /* _MSC_VER (warnings) */ - -#include "../mdbx.h" -#include "defs.h" - -#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) - /* Actualy libmdbx was not tested with compilers older than GCC from RHEL6. - * But you could remove this #error and try to continue at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -# warning "libmdbx required GCC >= 4.2" -#endif - -#if defined(__clang__) && !__CLANG_PREREQ(3,8) - /* Actualy libmdbx was not tested with CLANG older than 3.8. - * But you could remove this #error and try to continue at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -# warning "libmdbx required CLANG >= 3.8" -#endif - -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) - /* Actualy libmdbx was not tested with something older than glibc 2.12 (from RHEL6). - * But you could remove this #error and try to continue at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -# warning "libmdbx required at least GLIBC 2.12." -#endif - -#ifdef __SANITIZE_THREAD__ -# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ - -#if __has_warning("-Wconstant-logical-operand") -# if defined(__clang__) -# pragma clang diagnostic ignored "-Wconstant-logical-operand" -# elif defined(__GNUC__) -# pragma GCC diagnostic ignored "-Wconstant-logical-operand" -# else -# pragma warning disable "constant-logical-operand" -# endif -#endif /* -Wconstant-logical-operand */ - -#if defined(__LCC__) && (__LCC__ <= 121) - /* bug #2798 */ -# pragma diag_suppress alignment_reduction_ignored -#elif defined(__ICC) -# pragma warning(disable: 3453 1366) -#elif __has_warning("-Walignment-reduction-ignored") -# if defined(__clang__) -# pragma clang diagnostic ignored "-Walignment-reduction-ignored" -# elif defined(__GNUC__) -# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" -# else -# pragma warning disable "alignment-reduction-ignored" -# endif -#endif /* -Walignment-reduction-ignored */ - -/* *INDENT-ON* */ -/* clang-format on */ - -#include "osal.h" - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(MDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - -#include "options.h" - -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ - -/* The minimum number of keys required in a database page. - * Setting this to a larger value will place a smaller bound on the - * maximum size of a data item. Data items larger than this size will - * be pushed into overflow pages instead of being stored directly in - * the B-tree node. This value used to default to 4. With a page size - * of 4096 bytes that meant that any item larger than 1024 bytes would - * go into an overflow page. That also meant that on average 2-3KB of - * each overflow page was wasted space. The value cannot be lower than - * 2 because then there would no longer be a tree structure. With this - * value, items larger than 2KB will go into overflow pages, and on - * average only 1KB will be wasted. */ -#define MDBX_MINKEYS 2 - -/* A stamp that identifies a file as an MDBX file. - * There's nothing special about this value other than that it is easily - * recognizable, and it will reflect any byte order mismatches. */ -#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) - -/* The version number for a database's datafile format. */ -#define MDBX_DATA_VERSION 2 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 3 - -/* handle for the DB used to track free pages. */ -#define FREE_DBI 0 -/* handle for the default DB. */ -#define MAIN_DBI 1 -/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ -#define CORE_DBS 2 -#define MAX_DBI (INT16_MAX - CORE_DBS) -#if MAX_DBI != MDBX_MAX_DBI -#error "Opps, MAX_DBI != MDBX_MAX_DBI" -#endif - -/* Number of meta pages - also hardcoded elsewhere */ -#define NUM_METAS 3 - -/* A page number in the database. - * - * MDBX uses 32 bit for page numbers. This limits database - * size up to 2^44 bytes, in case of 4K pages. */ -typedef uint32_t pgno_t; -#define PRIaPGNO PRIu32 -#define MAX_PAGENO UINT32_C(0x7FFFffff) -#define MIN_PAGENO NUM_METAS - -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) - -/* A transaction ID. */ -typedef uint64_t txnid_t; -#define PRIaTXN PRIi64 -#define MIN_TXNID UINT64_C(1) -#define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) -#define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef MDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define MDBX_TXNID_STEP 1u -#else -#define MDBX_TXNID_STEP 2u -#endif -#endif /* MDBX_TXNID_STEP */ - -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ -typedef uint16_t indx_t; - -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 1) - -typedef union mdbx_safe64 { - volatile uint64_t inconsistent; -#if MDBX_64BIT_ATOMIC - volatile uint64_t atomic; -#endif /* MDBX_64BIT_ATOMIC */ - struct { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - volatile uint32_t low; - volatile uint32_t high; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - volatile uint32_t high; - volatile uint32_t low; -#else -#error "FIXME: Unsupported byte order" -#endif /* __BYTE_ORDER__ */ - }; -} mdbx_safe64_t; - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last commited modification */ -} MDBX_db; - -/* database size-related parameters */ -typedef struct mdbx_geo_t { - uint16_t grow; /* datafile growth step in pages */ - uint16_t shrink; /* datafile shrink threshold in pages */ - pgno_t lower; /* minimal size of datafile in pages */ - pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, - * but actually the file may be shorter. */ -} mdbx_geo_t; - -/* Meta page content. - * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { - /* Stamp identifying this as an MDBX file. - * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint64_t mm_magic_and_version; - - /* txnid that committed this page, the first of a two-phase-update pair */ - mdbx_safe64_t mm_txnid_a; - - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ - - mdbx_geo_t mm_geo; /* database size-related parameters */ - - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize -/* Any persistent environment flags, see mdbx_env */ -#define mm_flags mm_dbs[FREE_DBI].md_flags - mdbx_canary mm_canary; - -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) - volatile uint64_t mm_datasync_sign; - - /* txnid that committed this page, the second of a two-phase-update pair */ - mdbx_safe64_t mm_txnid_b; - - /* Number of non-meta pages which were put in GC after COW. May be 0 in case - * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint64_t mm_pages_retired; - - /* The analogue /proc/sys/kernel/random/boot_id or similar to determine - * whether the system was rebooted after the last use of the database files. - * If there was no reboot, but there is no need to rollback to the last - * steady sync point. Zeros mean that no relevant information is available - * from the system. */ - bin128_t mm_bootid; - -} MDBX_meta; - -/* Common header for all page types. The page type depends on mp_flags. - * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. - * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. - * - * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) - * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. - * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once - * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { - union { - struct MDBX_page *mp_next; /* for in-memory list of freed pages */ - uint64_t mp_txnid; /* txnid during which the page has been COW-ed */ - }; - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01 /* branch page */ -#define P_LEAF 0x02 /* leaf page */ -#define P_OVERFLOW 0x04 /* overflow page */ -#define P_META 0x08 /* meta page */ -#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ -#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ -#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ -#define P_KEEP 0x8000 /* leave this page alone during spill */ - uint16_t mp_flags; - union { - struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ - }; - uint32_t mp_pages; /* number of overflow pages */ - }; - pgno_t mp_pgno; /* page number */ - - /* dynamic size */ - indx_t mp_ptrs[/* C99 */]; -} MDBX_page; - -/* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) - -#pragma pack(pop) - -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void mdbx_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t mdbx_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t mdbx_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t mdbx_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV -MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - -/* Reader Lock Table - * - * Readers don't acquire any locks for their data access. Instead, they - * simply record their transaction ID in the reader table. The reader - * mutex is needed just to find an empty slot in the reader table. The - * slot's address is saved in thread-specific data so that subsequent - * read transactions started by the same thread need no further locking to - * proceed. - * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. - * - * Since the database uses multi-version concurrency control, readers don't - * actually need any locking. This table is used to keep track of which - * readers are using data from which old transactions, so that we'll know - * when a particular old transaction is no longer in use. Old transactions - * that have discarded any data pages can then have those pages reclaimed - * for use by a later write transaction. - * - * The lock table is constructed such that reader slots are aligned with the - * processor's cache line size. Any slot is only ever used by one thread. - * This alignment guarantees that there will be no contention or cache - * thrashing as threads update their own slot info, and also eliminates - * any need for locking when accessing a slot. - * - * A writer thread will scan every slot in the table to determine the oldest - * outstanding reader transaction. Any freed pages older than this will be - * reclaimed by the writer. The writer doesn't use any locks when scanning - * this table. This means that there's no guarantee that the writer will - * see the most up-to-date reader info, but that's not required for correct - * operation - all we need is to know the upper bound on the oldest reader, - * we don't care at all about the newest reader. So the only consequence of - * reading stale information here is that old pages might hang around a - * while longer before being reclaimed. That's actually good anyway, because - * the longer we delay reclaiming old pages, the more likely it is that a - * string of contiguous pages can be found after coalescing old pages from - * many old transactions together. */ - -/* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. - * Multiple readers that start at the same time will probably have the - * same ID here. Again, it's not important to exclude them from - * anything; all we need to know is which version of the DB they - * started from so we can avoid overwriting any data used in that - * particular version. */ - mdbx_safe64_t /* txnid_t */ mr_txnid; - - /* The information we store in a single slot of the reader table. - * In addition to a transaction ID, we also record the process and - * thread ID that owns a slot, so that we can detect stale information, - * e.g. threads or processes that went away without cleaning up. - * - * NOTE: We currently don't check for stale records. - * We simply re-init the table when we know that we're the only process - * opening the lock file. */ - - /* The thread ID of the thread owning this txn. */ -#if MDBX_WORDBITS >= 64 - volatile uint64_t mr_tid; -#else - volatile uint32_t mr_tid; - volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, - unused for now */ -#endif - /* The process ID of the process owning this reader txn. */ - volatile uint32_t mr_pid; - - /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - volatile pgno_t mr_snapshot_pages_used; - /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - volatile uint64_t mr_snapshot_pages_retired; -} MDBX_reader; - -/* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { - /* Stamp identifying this as an MDBX file. - * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; - - /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; - - /* Flags which environment was opened. */ - volatile uint32_t mti_envmode; - - /* Threshold of un-synced-with-disk pages for auto-sync feature, - * zero means no-threshold, i.e. auto-sync is disabled. */ - volatile pgno_t mti_autosync_threshold; - - /* Low 32-bit of txnid with which meta-pages was synced, - * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ - volatile uint32_t mti_meta_sync_txnid; - - /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. - * The time value is represented in a suitable system-dependent form, for - * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). - * Zero means timed auto-sync is disabled. */ - volatile uint64_t mti_autosync_period; - - /* Marker to distinguish uniqueness of DB/CLK.*/ - volatile uint64_t mti_bait_uniqueness; - - alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ - - /* Write transation lock. */ -#if MDBX_LOCKING > 0 - mdbx_ipclock_t mti_wlock; -#endif /* MDBX_LOCKING > 0 */ - - volatile txnid_t mti_oldest_reader; - - /* Timestamp of the last steady sync. Value is represented in a suitable - * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or - * clock_gettime(CLOCK_MONOTONIC). */ - volatile uint64_t mti_sync_timestamp; - - /* Number un-synced-with-disk pages for auto-sync feature. */ - volatile pgno_t mti_unsynced_pages; - - /* Number of page which was discarded last time by madvise(MADV_FREE). */ - volatile pgno_t mti_discarded_tail; - - /* Timestamp of the last readers check. */ - volatile uint64_t mti_reader_check_timestamp; - - alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ - - /* Readeaders registration lock. */ -#if MDBX_LOCKING > 0 - mdbx_ipclock_t mti_rlock; -#endif /* MDBX_LOCKING > 0 */ - - /* The number of slots that have been used in the reader table. - * This always records the maximum count, it is not decremented - * when readers release their slots. */ - volatile unsigned mti_numreaders; - volatile unsigned mti_readers_refresh_flag; - - alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ - MDBX_reader mti_readers[/* C99 */]; -} MDBX_lockinfo; - -/* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) - -#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) -#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) - -#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) - -#ifndef MDBX_ASSUME_MALLOC_OVERHEAD -#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) -#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ - -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) -#if defined(_WIN32) || defined(_WIN64) -#define MAX_MAPSIZE32 UINT32_C(0x38000000) -#else -#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) -#endif -#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) - -#if MDBX_WORDBITS >= 64 -#define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_READERS_LIMIT \ - ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) -#else -#define MDBX_READERS_LIMIT 1024 -#define MAX_MAPSIZE MAX_MAPSIZE32 -#endif /* MDBX_WORDBITS */ - -/*----------------------------------------------------------------------------*/ -/* Two kind lists of pages (aka PNL) */ - -/* An PNL is an Page Number List, a sorted array of IDs. The first element of - * the array is a counter for how many actual page-numbers are in the list. - * PNLs are sorted in descending order, this allow cut off a page with lowest - * pgno (at the tail) just truncating the list */ -#define MDBX_PNL_ASCENDING 0 -typedef pgno_t *MDBX_PNL; - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) -#else -#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) -#endif - -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; - -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef union MDBX_DP { - struct { - pgno_t pgno; - MDBX_page *ptr; - }; - struct { - unsigned sorted; - unsigned length; - }; -} MDBX_DP; - -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. - * The first element's length member is a count of how many actual - * elements are in the array. */ -typedef MDBX_DP *MDBX_DPL; - -/* PNL sizes */ -#define MDBX_PNL_GRANULATE 1024 -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) - -#if MDBX_HUGE_TRANSACTIONS -#define MDBX_PNL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) -#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 2) -#else -#define MDBX_PNL_MAX \ - ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) -#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) -#endif /* MDBX_HUGE_TRANSACTIONS */ - -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) - -#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) -#define MDBX_PNL_SIZE(pl) ((pl)[0]) -#define MDBX_PNL_FIRST(pl) ((pl)[1]) -#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) -#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) -#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) -#else -#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) -#endif - -#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) -#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) - -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - size_t mt_signature; - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - mdbx_geo_t mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_BEGIN_FLAGS \ - (MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC | MDBX_MAPASYNC | MDBX_RDONLY | \ - MDBX_TRYTXN) - /* Additional flag for mdbx_sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - - /* internal txn flags */ -#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ -#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ -#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ -#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ -#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ - /* most operations on the txn are currently illegal */ -#define MDBX_TXN_BLOCKED \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD) - -#if (TXN_FLAGS & MDBX_TXN_BEGIN_FLAGS) || \ - ((MDBX_TXN_BEGIN_FLAGS | TXN_FLAGS) & MDBX_SHRINK_ALLOWED) -#error "Opps, some flags overlapped or wrong" -#endif - - unsigned mt_flags; - /* The ID of this transaction. IDs are integers incrementing from 1. - * Only committed write transactions increment the ID. If a transaction - * aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - unsigned *mt_dbiseqs; - /* In write txns, array of cursors for each DB */ - MDBX_cursor **mt_cursors; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ -#define DBI_AUDITED 0x80 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - mdbx_canary mt_canary; - - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ - pgno_t loose_refund_wl /* FIXME: describe */; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - unsigned dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_DPL dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - unsigned loose_count; - /* Number of retired to parent pages (tw.retired2parent_pages) */ - unsigned retired2parent_count; - /* The list of parent's txn dirty pages that retired (became unused) - * in this transaction, linked through `mp_next`. */ - MDBX_page *retired2parent_pages; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL spill_pages; - } tw; - }; -}; - -/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. - * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to - * raise this on a 64 bit machine. */ -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 28 -#else -#define CURSOR_STACK 20 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_SIGNATURE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - unsigned mc_snum; /* number of pushed pages */ - unsigned mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ -#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ - - /* Cursor checing flags. */ -#define C_COPYING 0x100 /* skip key-value length check (copying simplify) */ -#define C_UPDATING 0x200 /* update/rebalance pending */ -#define C_RETIRING 0x400 /* refs to child pages may be invalid */ -#define C_SKIPORD 0x800 /* don't check keys ordering */ - - unsigned mc_flags; /* see mdbx_cursor */ - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; - /* The mt_dbistate for this Dup DB */ - uint8_t mx_dbistate; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - mdbx_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.dxb -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd; - mdbx_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd -#define me_lck me_lck_mmap.lck - - unsigned me_psize; /* DB page size, inited from me_os_psize */ - unsigned me_psize2log; /* log2 of DB page size */ - unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - mdbx_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - mdbx_thread_key_t me_txkey; /* thread-key for readers */ - char *me_path; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn; /* current write transaction */ - MDBX_txn *me_txn0; /* prealloc'd write transaction */ - - /* write-txn lock */ -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - -#if MDBX_LOCKING > 0 - mdbx_ipclock_t *me_wlock; -#endif /* MDBX_LOCKING > 0 */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - unsigned *me_dbiseqs; /* array of dbi sequence numbers */ - volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ - MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - /* MDBX_DP of pages written during a write txn. */ - MDBX_DPL me_dirtylist; - /* Number of freelist items that can fit in a single overflow page */ - unsigned me_maxgc_ov1page; - unsigned me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_maxkey_nd, me_maxkey_ds; - unsigned me_maxval_nd, me_maxval_ds; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - volatile uint64_t *me_sync_timestamp; - volatile uint64_t *me_autosync_period; - volatile pgno_t *me_unsynced_pages; - volatile pgno_t *me_autosync_threshold; - volatile pgno_t *me_discarded_tail; - volatile uint32_t *me_meta_sync_txnid; - MDBX_oom_func *me_oom_func; /* Callback for kicking laggard readers */ - struct { -#if MDBX_LOCKING > 0 - mdbx_ipclock_t wlock; -#endif /* MDBX_LOCKING > 0 */ - txnid_t oldest; - uint64_t sync_timestamp; - uint64_t autosync_period; - pgno_t autosync_pending; - pgno_t autosync_threshold; - pgno_t discarded_tail; - uint32_t meta_sync_txnid; - } me_lckless_stub; -#if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - MDBX_env *me_lcklist_next; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if defined(_WIN32) || defined(_WIN64) - MDBX_srwlock me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; -#else - mdbx_fastmutex_t me_remap_guard; -#endif -}; - -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -#ifdef MDBX_ALLOY -static uint8_t mdbx_runtime_flags = MDBX_RUNTIME_FLAGS_INIT; -static uint8_t mdbx_loglevel = MDBX_DEBUG; -#else -extern uint8_t mdbx_runtime_flags; -extern uint8_t mdbx_loglevel; -#endif /* MDBX_ALLOY */ -MDBX_INTERNAL_VAR MDBX_debug_func *mdbx_debug_logger; - -MDBX_INTERNAL_FUNC void mdbx_debug_log(int type, const char *function, int line, - const char *fmt, ...) - __printf_args(4, 5); - -#if MDBX_DEBUG - -#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) - -#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) - -#ifdef MDBX_LOGLEVEL_BUILD -#define mdbx_log_enabled(msg) \ - (msg <= MDBX_LOGLEVEL_BUILD && unlikely(msg <= mdbx_loglevel)) -#else -#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) -#endif /* MDBX_LOGLEVEL_BUILD */ - -#else /* MDBX_DEBUG */ - -#define mdbx_audit_enabled() (0) - -#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERTIONS) -#define mdbx_assert_enabled() (1) -#else -#define mdbx_assert_enabled() (0) -#endif /* NDEBUG */ - -#ifdef MDBX_LOGLEVEL_BUILD -#define mdbx_log_enabled(msg) (msg <= MDBX_LOGLEVEL_BUILD) -#else -#define mdbx_log_enabled(msg) (0) -#endif /* MDBX_LOGLEVEL_BUILD */ - -#endif /* MDBX_DEBUG */ - -#if defined(__ANDROID_API__) -#define mdbx_panic(fmt, ...) \ - __android_log_assert("panic", "mdbx", fmt, __VA_ARGS__) -#else -MDBX_INTERNAL_FUNC void mdbx_panic(const char *fmt, ...) __printf_args(1, 2); -#endif - -#if !MDBX_DEBUG && defined(__ANDROID_API__) -#define mdbx_assert_fail(env, msg, func, line) \ - __android_log_assert(msg, "mdbx", "%s:%u", func, line) -#else -MDBX_INTERNAL_FUNC void mdbx_assert_fail(const MDBX_env *env, const char *msg, - const char *func, int line); -#endif - -#define mdbx_debug_extra(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \ - mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define mdbx_debug_extra_print(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \ - mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define mdbx_trace(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_TRACE)) \ - mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ - __VA_ARGS__); \ - } while (0) - -#define mdbx_debug(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \ - mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ - __VA_ARGS__); \ - } while (0) - -#define mdbx_verbose(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ - mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ - __VA_ARGS__); \ - } while (0) - -#define mdbx_notice(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ - mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ - __VA_ARGS__); \ - } while (0) - -#define mdbx_warning(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_WARN)) \ - mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ - __VA_ARGS__); \ - } while (0) - -#define mdbx_error(fmt, ...) \ - do { \ - if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ - mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ - __VA_ARGS__); \ - } while (0) - -#define mdbx_fatal(fmt, ...) \ - mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#define mdbx_ensure_msg(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - mdbx_assert_fail(env, msg, __func__, __LINE__); \ - } while (0) - -#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define mdbx_assert(env, expr) \ - do { \ - if (mdbx_assert_enabled()) \ - mdbx_ensure(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) - -#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#undef assert -#define assert(expr) mdbx_assert(NULL, expr) -#endif - -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ - -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define mdbx_flush_incoherent_cpu_writeback() mdbx_memory_barrier() -#else -#define mdbx_flush_incoherent_cpu_writeback() mdbx_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ - -static __maybe_unused __inline void -mdbx_flush_incoherent_mmap(void *addr, size_t nbytes, const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - mdbx_assert(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ - -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush(addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ - -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; -#endif -} - -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int mdbx_reader_check0(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, - MDBX_reader *begin, MDBX_reader *end); -MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); - -MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); -MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); -MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); - -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) - -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_LMDB_ERRCODE + 10) - -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) - -/* Key size which fits in a DKBUF. */ -#define DKBUF_MAXKEYSIZE 511 /* FIXME */ - -#if MDBX_DEBUG -#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) -#define DVAL(x) \ - mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) -#else -#define DKBUF ((void)(0)) -#define DKEY(x) ("-") -#define DVAL(x) ("-") -#endif - -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) - -/* Test if the flags f are set in a flag word w. */ -#define F_ISSET(w, f) (((w) & (f)) == (f)) - -/* Round n up to an even number. */ -#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_mapsize(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) -/* Test if a page is dirty */ -#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) - -#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) - -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ - - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ - - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) - uint8_t mn_data[/* C99 */]; /* key and data are appended here */ -} MDBX_node; - -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) - -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_ACCEDE) - -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID - -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Opps, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Opps, some flags overlapped or wrong" -#endif - -/* max number of pages to commit in one writev() call */ -#define MDBX_COMMIT_PAGES 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ -#undef MDBX_COMMIT_PAGES -#define MDBX_COMMIT_PAGES IOV_MAX -#endif - -/* - * / - * | -1, a < b - * CMP2INT(a,b) = < 0, a == b - * | 1, a > b - * \ - */ -#if 1 -/* LY: fast enough on most systems */ -#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) -#else -#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) -#endif - -/* Do not spill pages to disk if txn is getting full, may fail instead */ -#define MDBX_NOSPILL 0x8000 - -static __maybe_unused __inline pgno_t pgno_add(pgno_t base, pgno_t augend) { - assert(base <= MAX_PAGENO); - return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; -} - -static __maybe_unused __inline pgno_t pgno_sub(pgno_t base, pgno_t subtrahend) { - assert(base >= MIN_PAGENO); - return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; -} - -static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & mdbx_runtime_flags) - mdbx_osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -static __pure_function __always_inline __maybe_unused bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; -} - -static __pure_function __always_inline __maybe_unused size_t -floor_powerof2(size_t value, size_t granularity) { - assert(is_powerof2(granularity)); - return value & ~(granularity - 1); -} - -static __pure_function __always_inline __maybe_unused size_t -ceil_powerof2(size_t value, size_t granularity) { - return floor_powerof2(value + granularity - 1, granularity); -} diff --git a/libs/libmdbx/src/src/lck-posix.c b/libs/libmdbx/src/src/lck-posix.c deleted file mode 100644 index f908cb6fc0..0000000000 --- a/libs/libmdbx/src/src/lck-posix.c +++ /dev/null @@ -1,846 +0,0 @@ -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#if !(defined(_WIN32) || defined(_WIN64)) /* !Windows LCK-implementation */ - -#include "internals.h" -#include <sys/sem.h> - -/*----------------------------------------------------------------------------*/ -/* global constructor/destructor */ - -#if defined(__linux__) || defined(__gnu_linux__) - -#include <sys/utsname.h> - -#ifndef MDBX_ALLOY -uint32_t mdbx_linux_kernel_version; -bool mdbx_RunningOnWSL; -#endif /* MDBX_ALLOY */ - -static __cold bool probe_for_WSL(const char *tag) { - /* "Official" way of detecting WSL but not WSL2 - * https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 */ - return strstr(tag, "Microsoft") || strstr(tag, "WSL"); -} - -#endif /* Linux */ - -static __cold __attribute__((__constructor__)) void -mdbx_global_constructor(void) { -#if defined(__linux__) || defined(__gnu_linux__) - struct utsname buffer; - if (uname(&buffer) == 0) { - mdbx_RunningOnWSL = probe_for_WSL(buffer.version) || - probe_for_WSL(buffer.sysname) || - probe_for_WSL(buffer.release); - int i = 0; - char *p = buffer.release; - while (*p && i < 4) { - if (*p >= '0' && *p <= '9') { - long number = strtol(p, &p, 10); - if (number > 0) { - if (number > 255) - number = 255; - mdbx_linux_kernel_version += number << (24 - i * 8); - } - ++i; - } else { - ++p; - } - } - } -#endif /* Linux */ - - mdbx_rthc_global_init(); -} - -static __cold __attribute__((__destructor__)) void -mdbx_global_destructor(void) { - mdbx_rthc_global_dtor(); -} - -/*----------------------------------------------------------------------------*/ -/* lck */ - -/* Описание реализации блокировок для POSIX & Linux: - * - * lck-файл отображается в память, в нём организуется таблица читателей и - * размещаются совместно используемые posix-мьютексы (futex). Посредством - * этих мьютексов (см struct MDBX_lockinfo) реализуются: - * - Блокировка таблицы читателей для регистрации, - * т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock(). - * - Блокировка БД для пишущих транзакций, - * т.е. функции mdbx_txn_lock() и mdbx_txn_unlock(). - * - * Остальной функционал реализуется отдельно посредством файловых блокировок: - * - Первоначальный захват БД в режиме exclusive/shared и последующий перевод - * в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade(). - * - Проверка присутствие процессов-читателей, - * т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check(). - * - * Для блокировки файлов используется fcntl(F_SETLK), так как: - * - lockf() оперирует только эксклюзивной блокировкой и требует - * открытия файла в RW-режиме. - * - flock() не гарантирует атомарности при смене блокировок - * и оперирует только всем файлом целиком. - * - Для контроля процессов-читателей используются однобайтовые - * range-блокировки lck-файла посредством fcntl(F_SETLK). При этом - * в качестве позиции используется pid процесса-читателя. - * - Для первоначального захвата и shared/exclusive выполняется блокировка - * основного файла БД и при успехе lck-файла. - * - * ---------------------------------------------------------------------------- - * УДЕРЖИВАЕМЫЕ БЛОКИРОВКИ В ЗАВИСИМОСТИ ОТ РЕЖИМА И СОСТОЯНИЯ - * - * Эксклюзивный режим без lck-файла: - * = заблокирован весь dxb-файл посредством F_RDLCK или F_WRLCK, - * в зависимости от MDBX_RDONLY. - * - * Не-операционный режим на время пере-инициализации и разрушении lck-файла: - * = F_WRLCK блокировка первого байта lck-файла, другие процессы ждут её - * снятия при получении F_RDLCK через F_SETLKW. - * - блокировки dxb-файла могут меняться до снятие эксклюзивной блокировки - * lck-файла: - * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - * + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - * - * ОПЕРАЦИОННЫЙ режим с lck-файлом: - * = F_RDLCK блокировка первого байта lck-файла, другие процессы не могут - * получить F_WRLCK и таким образом видят что БД используется. - * + F_WRLCK блокировка pid-байта в clk-файле после первой транзакции чтения. - * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - * + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - */ - -#if MDBX_USE_OFDLOCKS -static int op_setlk, op_setlkw, op_getlk; -static void __cold choice_fcntl() { - assert(!op_setlk && !op_setlkw && !op_getlk); - if ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 -#if defined(__linux__) || defined(__gnu_linux__) - && mdbx_linux_kernel_version > - 0x030f0000 /* OFD locks are available since 3.15, but engages here - only for 3.16 and later kernels (i.e. LTS) because - of reliability reasons */ -#endif /* linux */ - ) { - op_setlk = F_OFD_SETLK; - op_setlkw = F_OFD_SETLKW; - op_getlk = F_OFD_GETLK; - return; - } - op_setlk = F_SETLK; - op_setlkw = F_SETLKW; - op_getlk = F_GETLK; -} -#else -#define op_setlk F_SETLK -#define op_setlkw F_SETLKW -#define op_getlk F_GETLK -#endif /* MDBX_USE_OFDLOCKS */ - -#ifndef OFF_T_MAX -#define OFF_T_MAX \ - ((sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff) -#endif - -static int lck_op(mdbx_filehandle_t fd, int cmd, int lck, off_t offset, - off_t len) { - mdbx_jitter4testing(true); - for (;;) { - struct flock lock_op; - memset(&lock_op, 0, sizeof(lock_op)); - lock_op.l_type = lck; - lock_op.l_whence = SEEK_SET; - lock_op.l_start = offset; - lock_op.l_len = len; - int rc = fcntl(fd, cmd, &lock_op); - mdbx_jitter4testing(true); - if (rc != -1) { - if (cmd == op_getlk) { - /* Checks reader by pid. Returns: - * MDBX_RESULT_TRUE - if pid is live (reader holds a lock). - * MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */ - return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; - } - return MDBX_SUCCESS; - } - rc = errno; - if (rc != EINTR || cmd == op_setlkw) { - mdbx_assert(nullptr, MDBX_IS_ERROR(rc)); - return rc; - } - } -} - -MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - assert(env->me_pid > 0); - if (unlikely(mdbx_getpid() != env->me_pid)) - return MDBX_PANIC; - return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1); -} - -MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - assert(env->me_pid > 0); - return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1); -} - -MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - assert(pid > 0); - return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1); -} - -/*---------------------------------------------------------------------------*/ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV -MDBX_INTERNAL_FUNC int mdbx_ipclock_stub(mdbx_ipclock_t *ipc) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - return sem_init(ipc, false, 1) ? errno : 0; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - return pthread_mutex_init(ipc, nullptr); -#else -#error "FIXME" -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_ipclock_destroy(mdbx_ipclock_t *ipc) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - return sem_destroy(ipc) ? errno : 0; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - return pthread_mutex_destroy(ipc); -#else -#error "FIXME" -#endif -} -#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */ - -static int check_fstat(MDBX_env *env) { - struct stat st; - - int rc = MDBX_SUCCESS; - if (fstat(env->me_lazy_fd, &st)) { - rc = errno; - mdbx_error("fstat(%s), err %d", "DXB", rc); - return rc; - } - - if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { -#ifdef EBADFD - rc = EBADFD; -#else - rc = EPERM; -#endif - mdbx_error("%s %s, err %d", "DXB", - (st.st_nlink < 1) ? "file was removed" : "not a regular file", - rc); - return rc; - } - - if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) { - mdbx_verbose("dxb-file is too short (%u), exclusive-lock needed", - (unsigned)st.st_size); - rc = MDBX_RESULT_TRUE; - } - - //---------------------------------------------------------------------------- - - if (fstat(env->me_lfd, &st)) { - rc = errno; - mdbx_error("fstat(%s), err %d", "LCK", rc); - return rc; - } - - if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { -#ifdef EBADFD - rc = EBADFD; -#else - rc = EPERM; -#endif - mdbx_error("%s %s, err %d", "LCK", - (st.st_nlink < 1) ? "file was removed" : "not a regular file", - rc); - return rc; - } - - /* Checking file size for detect the situation when we got the shared lock - * immediately after mdbx_lck_destroy(). */ - if (st.st_size < (off_t)(sizeof(MDBX_lockinfo) + sizeof(MDBX_reader))) { - mdbx_verbose("lck-file is too short (%u), exclusive-lock needed", - (unsigned)st.st_size); - rc = MDBX_RESULT_TRUE; - } - - return rc; -} - -MDBX_INTERNAL_FUNC int __cold mdbx_lck_seize(MDBX_env *env) { - assert(env->me_lazy_fd != INVALID_HANDLE_VALUE); - if (unlikely(mdbx_getpid() != env->me_pid)) - return MDBX_PANIC; -#if MDBX_USE_OFDLOCKS - if (unlikely(op_setlk == 0)) - choice_fcntl(); -#endif /* MDBX_USE_OFDLOCKS */ - - int rc = MDBX_SUCCESS; -#if defined(__linux__) || defined(__gnu_linux__) - if (unlikely(mdbx_RunningOnWSL)) { - rc = ENOLCK /* No record locks available */; - mdbx_error("%s, err %u", - "WSL (Windows Subsystem for Linux) is mad and trouble-full, " - "injecting failure to avoid data loss", - rc); - return rc; - } -#endif /* Linux */ - - if (env->me_lfd == INVALID_HANDLE_VALUE) { - /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ - rc = - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); - if (rc != MDBX_SUCCESS) { - mdbx_error("%s, err %u", "without-lck", rc); - mdbx_assert(env, MDBX_IS_ERROR(rc)); - return rc; - } - return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; - } -#if defined(_POSIX_PRIORITY_SCHEDULING) && _POSIX_PRIORITY_SCHEDULING > 0 - sched_yield(); -#endif - -retry: - if (rc == MDBX_RESULT_TRUE) { - rc = lck_op(env->me_lfd, op_setlk, F_UNLCK, 0, 1); - if (rc != MDBX_SUCCESS) { - mdbx_error("%s, err %u", "unlock-before-retry", rc); - mdbx_assert(env, MDBX_IS_ERROR(rc)); - return rc; - } - } - - /* Firstly try to get exclusive locking. */ - rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); - if (rc == MDBX_SUCCESS) { - rc = check_fstat(env); - if (MDBX_IS_ERROR(rc)) - return rc; - - continue_dxb_exclusive: - rc = - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); - if (rc == MDBX_SUCCESS) - return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; - - int err = check_fstat(env); - if (MDBX_IS_ERROR(err)) - return err; - - /* the cause may be a collision with POSIX's file-lock recovery. */ - if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || - rc == EDEADLK)) { - mdbx_error("%s, err %u", "dxb-exclusive", rc); - mdbx_assert(env, MDBX_IS_ERROR(rc)); - return rc; - } - - /* Fallback to lck-shared */ - } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || - rc == EWOULDBLOCK || rc == EDEADLK)) { - mdbx_error("%s, err %u", "try-exclusive", rc); - mdbx_assert(env, MDBX_IS_ERROR(rc)); - return rc; - } - - /* Here could be one of two: - * - mdbx_lck_destroy() from the another process was hold the lock - * during a destruction. - * - either mdbx_lck_seize() from the another process was got the exclusive - * lock and doing initialization. - * For distinguish these cases will use size of the lck-file later. */ - - /* Wait for lck-shared now. */ - /* Here may be await during transient processes, for instance until another - * competing process doesn't call lck_downgrade(). */ - rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1); - if (rc != MDBX_SUCCESS) { - mdbx_error("%s, err %u", "try-shared", rc); - mdbx_assert(env, MDBX_IS_ERROR(rc)); - return rc; - } - - rc = check_fstat(env); - if (rc == MDBX_RESULT_TRUE) - goto retry; - if (rc != MDBX_SUCCESS) { - mdbx_error("%s, err %u", "lck_fstat", rc); - return rc; - } - - /* got shared, retry exclusive */ - rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); - if (rc == MDBX_SUCCESS) - goto continue_dxb_exclusive; - - if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || - rc == EDEADLK)) { - mdbx_error("%s, err %u", "try-exclusive", rc); - mdbx_assert(env, MDBX_IS_ERROR(rc)); - return rc; - } - - /* Lock against another process operating in without-lck or exclusive mode. */ - rc = - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1); - if (rc != MDBX_SUCCESS) { - mdbx_error("%s, err %u", "lock-against-without-lck", rc); - mdbx_assert(env, MDBX_IS_ERROR(rc)); - return rc; - } - - /* Done: return with shared locking. */ - return MDBX_RESULT_FALSE; -} - -MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - if (unlikely(mdbx_getpid() != env->me_pid)) - return MDBX_PANIC; - - int rc = MDBX_SUCCESS; - if ((env->me_flags & MDBX_EXCLUSIVE) == 0) { - rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, 0, env->me_pid); - if (rc == MDBX_SUCCESS) - rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, env->me_pid + 1, - OFF_T_MAX - env->me_pid - 1); - } - if (rc == MDBX_SUCCESS) - rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1); - if (unlikely(rc != 0)) { - mdbx_error("%s, err %u", "lck", rc); - assert(MDBX_IS_ERROR(rc)); - } - return rc; -} - -MDBX_INTERNAL_FUNC int __cold mdbx_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor) { - if (unlikely(mdbx_getpid() != env->me_pid)) - return MDBX_PANIC; - - int rc = MDBX_SUCCESS; - struct stat lck_info; - if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor && - env->me_lck && - /* try get exclusive access */ - lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 && - /* if LCK was not removed */ - fstat(env->me_lfd, &lck_info) == 0 && lck_info.st_nlink > 0 && - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, - OFF_T_MAX) == 0) { - - mdbx_verbose("%p got exclusive, drown locks", (void *)env); -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - if (env->me_sysv_ipc.semid != -1) - rc = semctl(env->me_sysv_ipc.semid, 2, IPC_RMID) ? errno : 0; -#else - rc = mdbx_ipclock_destroy(&env->me_lck->mti_rlock); - if (rc == 0) - rc = mdbx_ipclock_destroy(&env->me_lck->mti_wlock); -#endif /* MDBX_LOCKING */ - - mdbx_assert(env, rc == 0); - if (rc == 0) { - const bool synced = env->me_lck_mmap.lck->mti_unsynced_pages == 0; - mdbx_munmap(&env->me_lck_mmap); - if (synced) - rc = ftruncate(env->me_lfd, 0) ? errno : 0; - } - - mdbx_jitter4testing(false); - } - - /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored - * after file was closed. - * - * 2) File locks would be released (by kernel) while the file-descriptors will - * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel, - * locks should be released here explicitly with properly order. */ - - /* close dxb and restore lock */ - if (env->me_dsync_fd != INVALID_HANDLE_VALUE) { - if (unlikely(close(env->me_dsync_fd) != 0) && rc == MDBX_SUCCESS) - rc = errno; - env->me_dsync_fd = INVALID_HANDLE_VALUE; - } - if (env->me_lazy_fd != INVALID_HANDLE_VALUE) { - if (unlikely(close(env->me_lazy_fd) != 0) && rc == MDBX_SUCCESS) - rc = errno; - env->me_lazy_fd = INVALID_HANDLE_VALUE; - if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { - /* restore file-lock */ - rc = lck_op( - inprocess_neighbor->me_lazy_fd, F_SETLKW, - (inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, - (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) - ? 0 - : inprocess_neighbor->me_pid, - (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1); - } - } - - /* close clk and restore locks */ - if (env->me_lfd != INVALID_HANDLE_VALUE) { - if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS) - rc = errno; - env->me_lfd = INVALID_HANDLE_VALUE; - if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { - /* restore file-locks */ - rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1); - if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader) - rc = mdbx_rpid_set(inprocess_neighbor); - } - } - - if (inprocess_neighbor && rc != MDBX_SUCCESS) - inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR; - return rc; -} - -/*---------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC int __cold mdbx_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag) { -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - int semid = -1; - /* don't initialize semaphores twice */ - (void)inprocess_neighbor; - if (global_uniqueness_flag == MDBX_RESULT_TRUE) { - struct stat st; - if (fstat(env->me_lazy_fd, &st)) - return errno; - sysv_retry_create: - semid = semget(env->me_sysv_ipc.key, 2, - IPC_CREAT | IPC_EXCL | - (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))); - if (unlikely(semid == -1)) { - int err = errno; - if (err != EEXIST) - return err; - - /* remove and re-create semaphore set */ - semid = semget(env->me_sysv_ipc.key, 2, 0); - if (semid == -1) { - err = errno; - if (err != ENOENT) - return err; - goto sysv_retry_create; - } - if (semctl(semid, 2, IPC_RMID)) { - err = errno; - if (err != EIDRM) - return err; - } - goto sysv_retry_create; - } - - unsigned short val_array[2] = {1, 1}; - if (semctl(semid, 2, SETALL, val_array)) - return errno; - } else { - semid = semget(env->me_sysv_ipc.key, 2, 0); - if (semid == -1) - return errno; - - /* check read & write access */ - struct semid_ds data[2]; - if (semctl(semid, 2, IPC_STAT, data) || semctl(semid, 2, IPC_SET, data)) - return errno; - } - - env->me_sysv_ipc.semid = semid; - return MDBX_SUCCESS; - -#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX - (void)inprocess_neighbor; - if (global_uniqueness_flag != MDBX_RESULT_TRUE) - return MDBX_SUCCESS; -#error "FIXME: Not implemented" -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - - /* don't initialize semaphores twice */ - (void)inprocess_neighbor; - if (global_uniqueness_flag == MDBX_RESULT_TRUE) { - if (sem_init(&env->me_lck->mti_rlock, true, 1)) - return errno; - if (sem_init(&env->me_lck->mti_wlock, true, 1)) - return errno; - } - return MDBX_SUCCESS; - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - if (inprocess_neighbor) - return MDBX_SUCCESS /* don't need any initialization for mutexes - if LCK already opened/used inside current process */ - ; - - /* FIXME: Unfortunately, there is no other reliable way but to long testing - * on each platform. On the other hand, behavior like FreeBSD is incorrect - * and we can expect it to be rare. Moreover, even on FreeBSD without - * additional in-process initialization, the probability of an problem - * occurring is vanishingly small, and the symptom is a return of EINVAL - * while locking a mutex. In other words, in the worst case, the problem - * results in an EINVAL error at the start of the transaction, but NOT data - * loss, nor database corruption, nor other fatal troubles. Thus, the code - * below I am inclined to think the workaround for erroneous platforms (like - * FreeBSD), rather than a defect of libmdbx. */ -#if defined(__FreeBSD__) - /* seems that shared mutexes on FreeBSD required in-process initialization */ - (void)global_uniqueness_flag; -#else - /* shared mutexes on many other platforms (including Darwin and Linux's - * futexes) doesn't need any addition in-process initialization */ - if (global_uniqueness_flag != MDBX_RESULT_TRUE) - return MDBX_SUCCESS; -#endif - - pthread_mutexattr_t ma; - int rc = pthread_mutexattr_init(&ma); - if (rc) - return rc; - - rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED); - if (rc) - goto bailout; - -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust) - rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); -#elif defined(PTHREAD_MUTEX_ROBUST_NP) || \ - defined(pthread_mutexattr_setrobust_np) - rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); -#elif _POSIX_THREAD_PROCESS_SHARED < 200809L - rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); -#else - rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); -#endif - if (rc) - goto bailout; -#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ - -#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && \ - !defined(MDBX_SAFE4QEMU) - rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT); - if (rc == ENOTSUP) - rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE); - if (rc && rc != ENOTSUP) - goto bailout; -#endif /* PTHREAD_PRIO_INHERIT */ - - rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK); - if (rc && rc != ENOTSUP) - goto bailout; - - rc = pthread_mutex_init(&env->me_lck->mti_rlock, &ma); - if (rc) - goto bailout; - rc = pthread_mutex_init(&env->me_lck->mti_wlock, &ma); - -bailout: - pthread_mutexattr_destroy(&ma); - return rc; -#else -#error "FIXME" -#endif /* MDBX_LOCKING > 0 */ -} - -static int __cold mdbx_ipclock_failed(MDBX_env *env, mdbx_ipclock_t *ipc, - const int err) { - int rc = err; -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV - if (err == EOWNERDEAD) { - /* We own the mutex. Clean up after dead previous owner. */ - - const bool rlocked = (env->me_lck && ipc == &env->me_lck->mti_rlock); - rc = MDBX_SUCCESS; - if (!rlocked) { - if (unlikely(env->me_txn)) { - /* env is hosed if the dead thread was ours */ - env->me_flags |= MDBX_FATAL_ERROR; - env->me_txn = NULL; - rc = MDBX_PANIC; - } - } - mdbx_notice("%clock owner died, %s", (rlocked ? 'r' : 'w'), - (rc ? "this process' env is hosed" : "recovering")); - - int check_rc = mdbx_reader_check0(env, rlocked, NULL); - check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - rc = (rc == MDBX_SUCCESS) ? check_rc : rc; -#else -#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutex_consistent) - int mreco_rc = pthread_mutex_consistent(ipc); -#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutex_consistent_np) - int mreco_rc = pthread_mutex_consistent_np(ipc); -#elif _POSIX_THREAD_PROCESS_SHARED < 200809L - int mreco_rc = pthread_mutex_consistent_np(ipc); -#else - int mreco_rc = pthread_mutex_consistent(ipc); -#endif - check_rc = (mreco_rc == 0) ? check_rc : mreco_rc; - - if (unlikely(mreco_rc)) - mdbx_error("lock recovery failed, %s", mdbx_strerror(mreco_rc)); - - rc = (rc == MDBX_SUCCESS) ? check_rc : rc; - if (MDBX_IS_ERROR(rc)) - pthread_mutex_unlock(ipc); -#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ - return rc; - } -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 - (void)ipc; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - (void)ipc; -#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX -#warning "TODO" - (void)ipc; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - - mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err)); - if (rc != EDEADLK) - env->me_flags |= MDBX_FATAL_ERROR; - return rc; -} - -static int mdbx_ipclock_lock(MDBX_env *env, mdbx_ipclock_t *ipc, - const bool dont_wait) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - int rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc); - rc = (rc == EBUSY && dont_wait) ? MDBX_BUSY : rc; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - int rc = MDBX_SUCCESS; - if (dont_wait) { - if (sem_trywait(ipc)) { - rc = errno; - if (rc == EAGAIN) - rc = MDBX_BUSY; - } - } else if (sem_wait(ipc)) - rc = errno; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - struct sembuf op = {.sem_num = (ipc != env->me_wlock), - .sem_op = -1, - .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO}; - int rc; - if (semop(env->me_sysv_ipc.semid, &op, 1)) { - rc = errno; - if (dont_wait && rc == EAGAIN) - rc = MDBX_BUSY; - } else { - rc = *ipc ? EOWNERDEAD : MDBX_SUCCESS; - *ipc = env->me_pid; - } -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - - if (unlikely(rc != MDBX_SUCCESS && rc != MDBX_BUSY)) - rc = mdbx_ipclock_failed(env, ipc, rc); - return rc; -} - -static int mdbx_ipclock_unlock(MDBX_env *env, mdbx_ipclock_t *ipc) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - int rc = pthread_mutex_unlock(ipc); - (void)env; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - int rc = sem_post(ipc) ? errno : MDBX_SUCCESS; - (void)env; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - if (unlikely(*ipc != (pid_t)env->me_pid)) - return EPERM; - *ipc = 0; - struct sembuf op = { - .sem_num = (ipc != env->me_wlock), .sem_op = 1, .sem_flg = SEM_UNDO}; - int rc = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - return rc; -} - -MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) { - mdbx_trace("%s", ">>"); - mdbx_jitter4testing(true); - int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_rlock, false); - mdbx_trace("<< rc %d", rc); - return rc; -} - -MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) { - mdbx_trace("%s", ">>"); - int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_rlock); - mdbx_trace("<< rc %d", rc); - if (unlikely(rc != MDBX_SUCCESS)) - mdbx_panic("%s() failed: err %d\n", __func__, rc); - mdbx_jitter4testing(true); -} - -int mdbx_txn_lock(MDBX_env *env, bool dont_wait) { - mdbx_trace("%swait %s", dont_wait ? "dont-" : "", ">>"); - mdbx_jitter4testing(true); - int rc = mdbx_ipclock_lock(env, env->me_wlock, dont_wait); - mdbx_trace("<< rc %d", rc); - return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS; -} - -void mdbx_txn_unlock(MDBX_env *env) { - mdbx_trace("%s", ">>"); - int rc = mdbx_ipclock_unlock(env, env->me_wlock); - mdbx_trace("<< rc %d", rc); - if (unlikely(rc != MDBX_SUCCESS)) - mdbx_panic("%s() failed: err %d\n", __func__, rc); - mdbx_jitter4testing(true); -} - -#else -#ifdef _MSC_VER -#pragma warning(disable : 4206) /* nonstandard extension used: translation \ - unit is empty */ -#endif /* _MSC_VER (warnings) */ -#endif /* !Windows LCK-implementation */ diff --git a/libs/libmdbx/src/src/lck-windows.c b/libs/libmdbx/src/src/lck-windows.c deleted file mode 100644 index 8ec4659a28..0000000000 --- a/libs/libmdbx/src/src/lck-windows.c +++ /dev/null @@ -1,819 +0,0 @@ -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#if defined(_WIN32) || defined(_WIN64) /* Windows LCK-implementation */ - -/* PREAMBLE FOR WINDOWS: - * - * We are not concerned for performance here. - * If you are running Windows a performance could NOT be the goal. - * Otherwise please use Linux. */ - -#include "internals.h" - -static void mdbx_winnt_import(void); - -#if MDBX_BUILD_SHARED_LIBRARY -#if MDBX_AVOID_CRT && defined(NDEBUG) -/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks. - * - * Define dll's entry point only for Release build when NDEBUG is defined and - * MDBX_AVOID_CRT=ON. if the entry point isn't defined then MSVC's will - * automatically use DllMainCRTStartup() from CRT library, which also - * automatically call DllMain() from our mdbx.dll */ -#pragma comment(linker, "/ENTRY:DllMain") -#endif /* MDBX_AVOID_CRT */ - -BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved) -#else -#if !MDBX_CONFIG_MANUAL_TLS_CALLBACK -static -#endif /* !MDBX_CONFIG_MANUAL_TLS_CALLBACK */ - void NTAPI - mdbx_dll_handler(PVOID module, DWORD reason, PVOID reserved) -#endif /* MDBX_BUILD_SHARED_LIBRARY */ -{ - (void)reserved; - switch (reason) { - case DLL_PROCESS_ATTACH: - mdbx_winnt_import(); - mdbx_rthc_global_init(); - break; - case DLL_PROCESS_DETACH: - mdbx_rthc_global_dtor(); - break; - - case DLL_THREAD_ATTACH: - break; - case DLL_THREAD_DETACH: - mdbx_rthc_thread_dtor(module); - break; - } -#if MDBX_BUILD_SHARED_LIBRARY - return TRUE; -#endif -} - -#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK -/* *INDENT-OFF* */ -/* clang-format off */ -#if defined(_MSC_VER) -# pragma const_seg(push) -# pragma data_seg(push) - -# ifdef _WIN64 - /* kick a linker to create the TLS directory if not already done */ -# pragma comment(linker, "/INCLUDE:_tls_used") - /* Force some symbol references. */ -# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor") - /* specific const-segment for WIN64 */ -# pragma const_seg(".CRT$XLB") - const -# else - /* kick a linker to create the TLS directory if not already done */ -# pragma comment(linker, "/INCLUDE:__tls_used") - /* Force some symbol references. */ -# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor") - /* specific data-segment for WIN32 */ -# pragma data_seg(".CRT$XLB") -# endif - - __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_dll_handler; -# pragma data_seg(pop) -# pragma const_seg(pop) - -#elif defined(__GNUC__) -# ifdef _WIN64 - const -# endif - PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_dll_handler; -#else -# error FIXME -#endif -/* *INDENT-ON* */ -/* clang-format on */ -#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK */ - -/*----------------------------------------------------------------------------*/ - -#define LCK_SHARED 0 -#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK -#define LCK_WAITFOR 0 -#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY - -static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset, - size_t bytes) { - OVERLAPPED ov; - ov.hEvent = 0; - ov.Offset = (DWORD)offset; - ov.OffsetHigh = HIGH_DWORD(offset); - return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov); -} - -static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset, - size_t bytes) { - return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, - HIGH_DWORD(bytes)); -} - -/*----------------------------------------------------------------------------*/ -/* global `write` lock for write-txt processing, - * exclusive locking both meta-pages) */ - -#define LCK_MAXLEN (1u + (size_t)(MAXSSIZE_T)) -#define LCK_META_OFFSET 0 -#define LCK_META_LEN (MAX_PAGESIZE * NUM_METAS) -#define LCK_BODY_OFFSET LCK_META_LEN -#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET) -#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN -#define LCK_WHOLE 0, LCK_MAXLEN - -int mdbx_txn_lock(MDBX_env *env, bool dontwait) { - if (dontwait) { - if (!TryEnterCriticalSection(&env->me_windowsbug_lock)) - return MDBX_BUSY; - } else { - EnterCriticalSection(&env->me_windowsbug_lock); - } - - if ((env->me_flags & MDBX_EXCLUSIVE) || - flock(env->me_lazy_fd, - dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) - : (LCK_EXCLUSIVE | LCK_WAITFOR), - LCK_BODY)) - return MDBX_SUCCESS; - int rc = GetLastError(); - LeaveCriticalSection(&env->me_windowsbug_lock); - return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY; -} - -void mdbx_txn_unlock(MDBX_env *env) { - int rc = (env->me_flags & MDBX_EXCLUSIVE) - ? TRUE - : funlock(env->me_lazy_fd, LCK_BODY); - LeaveCriticalSection(&env->me_windowsbug_lock); - if (!rc) - mdbx_panic("%s failed: err %u", __func__, GetLastError()); -} - -/*----------------------------------------------------------------------------*/ -/* global `read` lock for readers registration, - * exclusive locking `mti_numreaders` (second) cacheline */ - -#define LCK_LO_OFFSET 0 -#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders) -#define LCK_UP_OFFSET LCK_LO_LEN -#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET) -#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN -#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN - -MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) { - mdbx_srwlock_AcquireShared(&env->me_remap_guard); - if (env->me_lfd == INVALID_HANDLE_VALUE) - return MDBX_SUCCESS; /* readonly database in readonly filesystem */ - - /* transite from S-? (used) to S-E (locked), e.g. exclusive lock upper-part */ - if ((env->me_flags & MDBX_EXCLUSIVE) || - flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) - return MDBX_SUCCESS; - - int rc = GetLastError(); - mdbx_srwlock_ReleaseShared(&env->me_remap_guard); - return rc; -} - -MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) { - if (env->me_lfd != INVALID_HANDLE_VALUE) { - /* transite from S-E (locked) to S-? (used), e.g. unlock upper-part */ - if ((env->me_flags & MDBX_EXCLUSIVE) == 0 && - !funlock(env->me_lfd, LCK_UPPER)) - mdbx_panic("%s failed: err %u", __func__, GetLastError()); - } - mdbx_srwlock_ReleaseShared(&env->me_remap_guard); -} - -static int suspend_and_append(mdbx_handle_array_t **array, - const DWORD ThreadId) { - const unsigned limit = (*array)->limit; - if ((*array)->count == limit) { - void *ptr = mdbx_realloc( - (limit > ARRAY_LENGTH((*array)->handles)) - ? *array - : /* don't free initial array on the stack */ NULL, - sizeof(mdbx_handle_array_t) + - sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles))); - if (!ptr) - return MDBX_ENOMEM; - if (limit == ARRAY_LENGTH((*array)->handles)) - memcpy(ptr, *array, sizeof(mdbx_handle_array_t)); - *array = (mdbx_handle_array_t *)ptr; - (*array)->limit = limit * 2; - } - - HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, - FALSE, ThreadId); - if (hThread == NULL) - return GetLastError(); - - if (SuspendThread(hThread) == -1) { - int err = GetLastError(); - DWORD ExitCode; - if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || - !GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE) - err = MDBX_SUCCESS; - CloseHandle(hThread); - return err; - } - - (*array)->handles[(*array)->count++] = hThread; - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int -mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { - const uintptr_t CurrentTid = GetCurrentThreadId(); - int rc; - if (env->me_lck) { - /* Scan LCK for threads of the current process */ - const MDBX_reader *const begin = env->me_lck->mti_readers; - const MDBX_reader *const end = begin + env->me_lck->mti_numreaders; - const uintptr_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0; - for (const MDBX_reader *reader = begin; reader < end; ++reader) { - if (reader->mr_pid != env->me_pid || !reader->mr_tid) { - skip_lck: - continue; - } - if (reader->mr_tid == CurrentTid || reader->mr_tid == WriteTxnOwner) - goto skip_lck; - if (env->me_flags & MDBX_NOTLS) { - /* Skip duplicates in no-tls mode */ - for (const MDBX_reader *scan = reader; --scan >= begin;) - if (scan->mr_tid == reader->mr_tid) - goto skip_lck; - } - - rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid); - if (rc != MDBX_SUCCESS) { - bailout_lck: - (void)mdbx_resume_threads_after_remap(*array); - return rc; - } - } - if (WriteTxnOwner && WriteTxnOwner != CurrentTid) { - rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner); - if (rc != MDBX_SUCCESS) - goto bailout_lck; - } - } else { - /* Without LCK (i.e. read-only mode). - * Walk thougth a snapshot of all running threads */ - mdbx_assert(env, - env->me_txn0 == NULL || (env->me_flags & MDBX_EXCLUSIVE) != 0); - const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); - if (hSnapshot == INVALID_HANDLE_VALUE) - return GetLastError(); - - THREADENTRY32 entry; - entry.dwSize = sizeof(THREADENTRY32); - - if (!Thread32First(hSnapshot, &entry)) { - rc = GetLastError(); - bailout_toolhelp: - CloseHandle(hSnapshot); - (void)mdbx_resume_threads_after_remap(*array); - return rc; - } - - do { - if (entry.th32OwnerProcessID != env->me_pid || - entry.th32ThreadID == CurrentTid) - continue; - - rc = suspend_and_append(array, entry.th32ThreadID); - if (rc != MDBX_SUCCESS) - goto bailout_toolhelp; - - } while (Thread32Next(hSnapshot, &entry)); - - rc = GetLastError(); - if (rc != ERROR_NO_MORE_FILES) - goto bailout_toolhelp; - CloseHandle(hSnapshot); - } - - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int -mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) { - int rc = MDBX_SUCCESS; - for (unsigned i = 0; i < array->count; ++i) { - const HANDLE hThread = array->handles[i]; - if (ResumeThread(hThread) == -1) { - const int err = GetLastError(); - DWORD ExitCode; - if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && - GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE) - rc = err; - } - CloseHandle(hThread); - } - return rc; -} - -/*----------------------------------------------------------------------------*/ -/* global `initial` lock for lockfile initialization, - * exclusive/shared locking first cacheline */ - -/* Briefly descritpion of locking schema/algorithm: - * - Windows does not support upgrading or downgrading for file locking. - * - Therefore upgrading/downgrading is emulated by shared and exclusive - * locking of upper and lower halves. - * - In other words, we have FSM with possible 9 states, - * i.e. free/shared/exclusive x free/shared/exclusive == 9. - * Only 6 states of FSM are used, which 2 of ones are transitive. - * - * States: - * ?-? = free, i.e. unlocked - * S-? = used, i.e. shared lock - * E-? = exclusive-read, i.e. operational exclusive - * ?-S - * ?-E = middle (transitive state) - * S-S - * S-E = locked (transitive state) - * E-S - * E-E = exclusive-write, i.e. exclusive due (re)initialization - * - * The mdbx_lck_seize() moves the locking-FSM from the initial free/unlocked - * state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible, - * or to the "used" (and returns MDBX_RESULT_FALSE). - * - * The mdbx_lck_downgrade() moves the locking-FSM from "exclusive write" - * state to the "used" (i.e. shared) state. - * - * The mdbx_lck_upgrade() moves the locking-FSM from "used" (i.e. shared) - * state to the "exclusive write" state. - */ - -static void lck_unlock(MDBX_env *env) { - int err; - - if (env->me_lfd != INVALID_HANDLE_VALUE) { - /* double `unlock` for robustly remove overlapped shared/exclusive locks */ - while (funlock(env->me_lfd, LCK_LOWER)) - ; - err = GetLastError(); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - (void)err; - SetLastError(ERROR_SUCCESS); - - while (funlock(env->me_lfd, LCK_UPPER)) - ; - err = GetLastError(); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - (void)err; - SetLastError(ERROR_SUCCESS); - } - - if (env->me_lazy_fd != INVALID_HANDLE_VALUE) { - /* explicitly unlock to avoid latency for other processes (windows kernel - * releases such locks via deferred queues) */ - while (funlock(env->me_lazy_fd, LCK_BODY)) - ; - err = GetLastError(); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - (void)err; - SetLastError(ERROR_SUCCESS); - - while (funlock(env->me_lazy_fd, LCK_WHOLE)) - ; - err = GetLastError(); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - (void)err; - SetLastError(ERROR_SUCCESS); - } -} - -/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE) - * or as 'used' (S-? and returns MDBX_RESULT_FALSE). - * Oherwise returns an error. */ -static int internal_seize_lck(HANDLE lfd) { - int rc; - assert(lfd != INVALID_HANDLE_VALUE); - - /* 1) now on ?-? (free), get ?-E (middle) */ - mdbx_jitter4testing(false); - if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) { - rc = GetLastError() /* 2) something went wrong, give up */; - mdbx_error("%s, err %u", "?-?(free) >> ?-E(middle)", rc); - return rc; - } - - /* 3) now on ?-E (middle), try E-E (exclusive-write) */ - mdbx_jitter4testing(false); - if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER)) - return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */; - - /* 5) still on ?-E (middle) */ - rc = GetLastError(); - mdbx_jitter4testing(false); - if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) { - /* 6) something went wrong, give up */ - if (!funlock(lfd, LCK_UPPER)) - mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)", - GetLastError()); - return rc; - } - - /* 7) still on ?-E (middle), try S-E (locked) */ - mdbx_jitter4testing(false); - rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE - : GetLastError(); - - mdbx_jitter4testing(false); - if (rc != MDBX_RESULT_FALSE) - mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc); - - /* 8) now on S-E (locked) or still on ?-E (middle), - * transite to S-? (used) or ?-? (free) */ - if (!funlock(lfd, LCK_UPPER)) - mdbx_panic("%s(%s) failed: err %u", __func__, - "X-E(locked/middle) >> X-?(used/free)", GetLastError()); - - /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */ - return rc; -} - -MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) { - int rc; - - assert(env->me_lazy_fd != INVALID_HANDLE_VALUE); - if (env->me_flags & MDBX_EXCLUSIVE) - return MDBX_RESULT_TRUE /* nope since files were must be opened - non-shareable */ - ; - - if (env->me_lfd == INVALID_HANDLE_VALUE) { - /* LY: without-lck mode (e.g. on read-only filesystem) */ - mdbx_jitter4testing(false); - if (!flock(env->me_lazy_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) { - rc = GetLastError(); - mdbx_error("%s, err %u", "without-lck", rc); - return rc; - } - return MDBX_RESULT_FALSE; - } - - rc = internal_seize_lck(env->me_lfd); - mdbx_jitter4testing(false); - if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) { - /* Check that another process don't operates in without-lck mode. - * Doing such check by exclusive locking the body-part of db. Should be - * noted: - * - we need an exclusive lock for do so; - * - we can't lock meta-pages, otherwise other process could get an error - * while opening db in valid (non-conflict) mode. */ - if (!flock(env->me_lazy_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) { - rc = GetLastError(); - mdbx_error("%s, err %u", "lock-against-without-lck", rc); - mdbx_jitter4testing(false); - lck_unlock(env); - } else { - mdbx_jitter4testing(false); - if (!funlock(env->me_lazy_fd, LCK_BODY)) - mdbx_panic("%s(%s) failed: err %u", __func__, - "unlock-against-without-lck", GetLastError()); - } - } - - return rc; -} - -MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) { - /* Transite from exclusive-write state (E-E) to used (S-?) */ - assert(env->me_lazy_fd != INVALID_HANDLE_VALUE); - assert(env->me_lfd != INVALID_HANDLE_VALUE); - - if (env->me_flags & MDBX_EXCLUSIVE) - return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ - ; - /* 1) now at E-E (exclusive-write), transite to ?_E (middle) */ - if (!funlock(env->me_lfd, LCK_LOWER)) - mdbx_panic("%s(%s) failed: err %u", __func__, - "E-E(exclusive-write) >> ?-E(middle)", GetLastError()); - - /* 2) now at ?-E (middle), transite to S-E (locked) */ - if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) { - int rc = GetLastError() /* 3) something went wrong, give up */; - mdbx_error("%s, err %u", "?-E(middle) >> S-E(locked)", rc); - return rc; - } - - /* 4) got S-E (locked), continue transition to S-? (used) */ - if (!funlock(env->me_lfd, LCK_UPPER)) - mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)", - GetLastError()); - - return MDBX_SUCCESS /* 5) now at S-? (used), done */; -} - -MDBX_INTERNAL_FUNC int mdbx_lck_upgrade(MDBX_env *env) { - /* Transite from used state (S-?) to exclusive-write (E-E) */ - assert(env->me_lfd != INVALID_HANDLE_VALUE); - - if (env->me_flags & MDBX_EXCLUSIVE) - return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ - ; - - int rc; - /* 1) now on S-? (used), try S-E (locked) */ - mdbx_jitter4testing(false); - if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_UPPER)) { - rc = GetLastError() /* 2) something went wrong, give up */; - mdbx_verbose("%s, err %u", "S-?(used) >> S-E(locked)", rc); - return rc; - } - - /* 3) now on S-E (locked), transite to ?-E (middle) */ - if (!funlock(env->me_lfd, LCK_LOWER)) - mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)", - GetLastError()); - - /* 4) now on ?-E (middle), try E-E (exclusive-write) */ - mdbx_jitter4testing(false); - if (!flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER)) { - rc = GetLastError() /* 5) something went wrong, give up */; - mdbx_verbose("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc); - return rc; - } - - return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */; -} - -MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag) { - (void)env; - (void)inprocess_neighbor; - (void)global_uniqueness_flag; - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor) { - /* LY: should unmap before releasing the locks to avoid race condition and - * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */ - if (env->me_map) - mdbx_munmap(&env->me_dxb_mmap); - if (env->me_lck) { - const bool synced = env->me_lck_mmap.lck->mti_unsynced_pages == 0; - mdbx_munmap(&env->me_lck_mmap); - if (synced && !inprocess_neighbor && env->me_lfd != INVALID_HANDLE_VALUE && - mdbx_lck_upgrade(env) == MDBX_SUCCESS) - /* this will fail if LCK is used/mmapped by other process(es) */ - mdbx_ftruncate(env->me_lfd, 0); - } - lck_unlock(env); - return MDBX_SUCCESS; -} - -/*----------------------------------------------------------------------------*/ -/* reader checking (by pid) */ - -MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) { - (void)env; - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) { - (void)env; - return MDBX_SUCCESS; -} - -/* Checks reader by pid. - * - * Returns: - * MDBX_RESULT_TRUE, if pid is live (unable to acquire lock) - * MDBX_RESULT_FALSE, if pid is dead (lock acquired) - * or otherwise the errcode. */ -MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) { - (void)env; - HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid); - int rc; - if (likely(hProcess)) { - rc = WaitForSingleObject(hProcess, 0); - if (unlikely(rc == WAIT_FAILED)) - rc = GetLastError(); - CloseHandle(hProcess); - } else { - rc = GetLastError(); - } - - switch (rc) { - case ERROR_INVALID_PARAMETER: - /* pid seems invalid */ - return MDBX_RESULT_FALSE; - case WAIT_OBJECT_0: - /* process just exited */ - return MDBX_RESULT_FALSE; - case ERROR_ACCESS_DENIED: - /* The ERROR_ACCESS_DENIED would be returned for CSRSS-processes, etc. - * assume pid exists */ - return MDBX_RESULT_TRUE; - case WAIT_TIMEOUT: - /* pid running */ - return MDBX_RESULT_TRUE; - default: - /* failure */ - return rc; - } -} - -//---------------------------------------------------------------------------- -// Stub for slim read-write lock -// Copyright (C) 1995-2002 Brad Wilson - -static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) { - srwl->readerCount = srwl->writerCount = 0; -} - -static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) { - while (true) { - assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); - - // If there's a writer already, spin without unnecessarily - // interlocking the CPUs - if (srwl->writerCount != 0) { - YieldProcessor(); - continue; - } - - // Add to the readers list - _InterlockedIncrement(&srwl->readerCount); - - // Check for writers again (we may have been pre-empted). If - // there are no writers writing or waiting, then we're done. - if (srwl->writerCount == 0) - break; - - // Remove from the readers list, spin, try again - _InterlockedDecrement(&srwl->readerCount); - YieldProcessor(); - } -} - -static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) { - assert(srwl->readerCount > 0); - _InterlockedDecrement(&srwl->readerCount); -} - -static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) { - while (true) { - assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); - - // If there's a writer already, spin without unnecessarily - // interlocking the CPUs - if (srwl->writerCount != 0) { - YieldProcessor(); - continue; - } - - // See if we can become the writer (expensive, because it inter- - // locks the CPUs, so writing should be an infrequent process) - if (_InterlockedExchange(&srwl->writerCount, 1) == 0) - break; - } - - // Now we're the writer, but there may be outstanding readers. - // Spin until there aren't any more; new readers will wait now - // that we're the writer. - while (srwl->readerCount != 0) { - assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); - YieldProcessor(); - } -} - -static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) { - assert(srwl->writerCount == 1 && srwl->readerCount >= 0); - srwl->writerCount = 0; -} - -MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared, - mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive, - mdbx_srwlock_ReleaseExclusive; - -/*----------------------------------------------------------------------------*/ - -#if 0 /* LY: unused for now */ -static DWORD WINAPI stub_DiscardVirtualMemory(PVOID VirtualAddress, - SIZE_T Size) { - return VirtualAlloc(VirtualAddress, Size, MEM_RESET, PAGE_NOACCESS) - ? ERROR_SUCCESS - : GetLastError(); -} -#endif /* unused for now */ - -static uint64_t WINAPI stub_GetTickCount64(void) { - LARGE_INTEGER Counter, Frequency; - return (QueryPerformanceFrequency(&Frequency) && - QueryPerformanceCounter(&Counter)) - ? Counter.QuadPart * 1000ul / Frequency.QuadPart - : 0; -} - -/*----------------------------------------------------------------------------*/ -#ifndef MDBX_ALLOY -MDBX_NtExtendSection mdbx_NtExtendSection; -MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx; -MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW; -MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; -MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle; -MDBX_NtFsControlFile mdbx_NtFsControlFile; -MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; -MDBX_GetTickCount64 mdbx_GetTickCount64; -#if 0 /* LY: unused for now */ -MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; -MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; -MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; -#endif /* unused for now */ -#endif /* MDBX_ALLOY */ - -static void mdbx_winnt_import(void) { - const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll"); - -#define GET_PROC_ADDR(dll, ENTRY) \ - mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(dll, #ENTRY) - - if (GetProcAddress(hNtdll, "wine_get_version")) { - assert(mdbx_RunningUnderWine()); - } else { - GET_PROC_ADDR(hNtdll, NtFsControlFile); - GET_PROC_ADDR(hNtdll, NtExtendSection); - assert(!mdbx_RunningUnderWine()); - } - - const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll"); - GET_PROC_ADDR(hKernel32dll, GetFileInformationByHandleEx); - GET_PROC_ADDR(hKernel32dll, GetTickCount64); - if (!mdbx_GetTickCount64) - mdbx_GetTickCount64 = stub_GetTickCount64; - if (!mdbx_RunningUnderWine()) { - GET_PROC_ADDR(hKernel32dll, SetFileInformationByHandle); - GET_PROC_ADDR(hKernel32dll, GetVolumeInformationByHandleW); - GET_PROC_ADDR(hKernel32dll, GetFinalPathNameByHandleW); - GET_PROC_ADDR(hKernel32dll, PrefetchVirtualMemory); - } - -#if 0 /* LY: unused for now */ - if (!mdbx_RunningUnderWine()) { - GET_PROC_ADDR(hKernel32dll, DiscardVirtualMemory); - GET_PROC_ADDR(hKernel32dll, OfferVirtualMemory); - GET_PROC_ADDR(hKernel32dll, ReclaimVirtualMemory); - } - if (!mdbx_DiscardVirtualMemory) - mdbx_DiscardVirtualMemory = stub_DiscardVirtualMemory; - if (!mdbx_OfferVirtualMemory) - mdbx_OfferVirtualMemory = stub_OfferVirtualMemory; - if (!mdbx_ReclaimVirtualMemory) - mdbx_ReclaimVirtualMemory = stub_ReclaimVirtualMemory; -#endif /* unused for now */ - -#undef GET_PROC_ADDR - - const MDBX_srwlock_function init = - (MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock"); - if (init != NULL) { - mdbx_srwlock_Init = init; - mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress( - hKernel32dll, "AcquireSRWLockShared"); - mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress( - hKernel32dll, "ReleaseSRWLockShared"); - mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress( - hKernel32dll, "AcquireSRWLockExclusive"); - mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress( - hKernel32dll, "ReleaseSRWLockExclusive"); - } else { - mdbx_srwlock_Init = stub_srwlock_Init; - mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared; - mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared; - mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive; - mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive; - } -} - -#endif /* Windows LCK-implementation */ diff --git a/libs/libmdbx/src/src/mdbx_chk.c b/libs/libmdbx/src/src/mdbx_chk.c deleted file mode 100644 index 657a28a730..0000000000 --- a/libs/libmdbx/src/src/mdbx_chk.c +++ /dev/null @@ -1,1485 +0,0 @@ -/* mdbx_chk.c - memory-mapped database check tool */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#ifdef _MSC_VER -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif -#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ -#endif /* _MSC_VER (warnings) */ - -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "internals.h" - -typedef struct flagbit { - int bit; - const char *name; -} flagbit; - -const flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"}, - {MDBX_INTEGERKEY, "integerkey"}, - {MDBX_REVERSEKEY, "reversekey"}, - {MDBX_DUPFIXED, "dupfixed"}, - {MDBX_REVERSEDUP, "reversedup"}, - {MDBX_INTEGERDUP, "integerdup"}, - {0, nullptr}}; - -#if defined(_WIN32) || defined(_WIN64) -#include "wingetopt.h" - -static volatile BOOL user_break; -static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { - (void)dwCtrlType; - user_break = 1; - return true; -} - -#else /* WINDOWS */ - -static volatile sig_atomic_t user_break; -static void signal_handler(int sig) { - (void)sig; - user_break = 1; -} - -#endif /* !WINDOWS */ - -#define EXIT_INTERRUPTED (EXIT_FAILURE + 4) -#define EXIT_FAILURE_SYS (EXIT_FAILURE + 3) -#define EXIT_FAILURE_MDB (EXIT_FAILURE + 2) -#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1) -#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE - -typedef struct { - const char *name; - struct { - uint64_t branch, large_count, large_volume, leaf; - uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed; - uint64_t total, empty, other; - } pages; - uint64_t payload_bytes; - uint64_t lost_bytes; -} walk_dbi_t; - -struct { - short *pagemap; - uint64_t total_payload_bytes; - uint64_t pgcount; - walk_dbi_t dbi[MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1]; -} walk; - -#define dbi_free walk.dbi[FREE_DBI] -#define dbi_main walk.dbi[MAIN_DBI] -#define dbi_meta walk.dbi[CORE_DBS] - -int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE; -MDBX_env *env; -MDBX_txn *txn; -MDBX_envinfo envinfo; -MDBX_stat envstat; -size_t userdb_count, skipped_subdb; -uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages, - unused_pages, backed_pages; -unsigned verbose; -bool ignore_wrong_order, quiet, dont_traversal; -const char *only_subdb; - -struct problem { - struct problem *pr_next; - size_t count; - const char *caption; -}; - -struct problem *problems_list; -uint64_t total_problems; - -static void __printf_args(1, 2) print(const char *msg, ...) { - if (!quiet) { - va_list args; - - fflush(stderr); - va_start(args, msg); - vfprintf(stdout, msg, args); - va_end(args); - } -} - -static void __printf_args(1, 2) error(const char *msg, ...) { - total_problems++; - - if (!quiet) { - va_list args; - - fflush(nullptr); - va_start(args, msg); - fputs(" ! ", stderr); - vfprintf(stderr, msg, args); - va_end(args); - fflush(nullptr); - } -} - -static int check_user_break(void) { - switch (user_break) { - case 0: - return MDBX_SUCCESS; - case 1: - print(" - interrupted by signal\n"); - fflush(nullptr); - user_break = 2; - } - return MDBX_EINTR; -} - -static void pagemap_cleanup(void) { - for (size_t i = CORE_DBS + /* account pseudo-entry for meta */ 1; - i < ARRAY_LENGTH(walk.dbi); ++i) { - if (walk.dbi[i].name) { - mdbx_free((void *)walk.dbi[i].name); - walk.dbi[i].name = nullptr; - } - } - - mdbx_free(walk.pagemap); - walk.pagemap = nullptr; -} - -static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) { - static walk_dbi_t *last; - - if (dbi_name == MDBX_PGWALK_MAIN) - return &dbi_main; - if (dbi_name == MDBX_PGWALK_GC) - return &dbi_free; - if (dbi_name == MDBX_PGWALK_META) - return &dbi_meta; - - if (last && strcmp(last->name, dbi_name) == 0) - return last; - - walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1; - for (; dbi < ARRAY_END(walk.dbi) && dbi->name; ++dbi) { - if (strcmp(dbi->name, dbi_name) == 0) - return last = dbi; - } - - if (verbose > 0 && !silent) { - print(" - found '%s' area\n", dbi_name); - fflush(nullptr); - } - - if (dbi == ARRAY_END(walk.dbi)) - return nullptr; - - dbi->name = mdbx_strdup(dbi_name); - return last = dbi; -} - -static void __printf_args(4, 5) - - problem_add(const char *object, uint64_t entry_number, const char *msg, - const char *extra, ...) { - total_problems++; - - if (!quiet) { - int need_fflush = 0; - struct problem *p; - - for (p = problems_list; p; p = p->pr_next) - if (p->caption == msg) - break; - - if (!p) { - p = mdbx_calloc(1, sizeof(*p)); - p->caption = msg; - p->pr_next = problems_list; - problems_list = p; - need_fflush = 1; - } - - p->count++; - if (verbose > 1) { - print(" %s #%" PRIu64 ": %s", object, entry_number, msg); - if (extra) { - va_list args; - printf(" ("); - va_start(args, extra); - vfprintf(stdout, extra, args); - va_end(args); - printf(")"); - } - printf("\n"); - if (need_fflush) - fflush(nullptr); - } - } -} - -static struct problem *problems_push(void) { - struct problem *p = problems_list; - problems_list = nullptr; - return p; -} - -static size_t problems_pop(struct problem *list) { - size_t count = 0; - - if (problems_list) { - int i; - - print(" - problems: "); - for (i = 0; problems_list; ++i) { - struct problem *p = problems_list->pr_next; - count += problems_list->count; - print("%s%s (%" PRIuPTR ")", i ? ", " : "", problems_list->caption, - problems_list->count); - mdbx_free(problems_list); - problems_list = p; - } - print("\n"); - fflush(nullptr); - } - - problems_list = list; - return count; -} - -static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, - void *const ctx, const int deep, - const char *const dbi_name_or_tag, const size_t page_size, - const MDBX_page_type_t pagetype, const size_t nentries, - const size_t payload_bytes, const size_t header_bytes, - const size_t unused_bytes) { - (void)ctx; - if (deep > 42) { - problem_add("deep", deep, "too large", nullptr); - return MDBX_CORRUPTED /* avoid infinite loop/recursion */; - } - - if (pagetype == MDBX_page_void) - return MDBX_SUCCESS; - - walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name_or_tag, false); - if (!dbi) - return MDBX_ENOMEM; - - const size_t page_bytes = payload_bytes + header_bytes + unused_bytes; - walk.pgcount += pgnumber; - - const char *pagetype_caption; - bool branch = false; - switch (pagetype) { - default: - problem_add("page", pgno, "unknown page-type", "type %u, deep %i", - (unsigned)pagetype, deep); - pagetype_caption = "unknown"; - dbi->pages.other += pgnumber; - break; - case MDBX_page_meta: - pagetype_caption = "meta"; - dbi->pages.other += pgnumber; - break; - case MDBX_page_large: - pagetype_caption = "large"; - dbi->pages.large_volume += pgnumber; - dbi->pages.large_count += 1; - break; - case MDBX_page_branch: - pagetype_caption = "branch"; - dbi->pages.branch += pgnumber; - branch = true; - break; - case MDBX_page_leaf: - pagetype_caption = "leaf"; - dbi->pages.leaf += pgnumber; - break; - case MDBX_page_dupfixed_leaf: - pagetype_caption = "leaf-dupfixed"; - dbi->pages.leaf_dupfixed += pgnumber; - break; - case MDBX_subpage_leaf: - pagetype_caption = "subleaf-dupsort"; - dbi->pages.subleaf_dupsort += 1; - break; - case MDBX_subpage_dupfixed_leaf: - pagetype_caption = "subleaf-dupfixed"; - dbi->pages.subleaf_dupfixed += 1; - break; - } - - if (pgnumber) { - if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi->name) == 0)) { - if (pgnumber == 1) - print(" %s-page %" PRIu64, pagetype_caption, pgno); - else - print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber); - print(" of %s: header %" PRIiPTR ", payload %" PRIiPTR - ", unused %" PRIiPTR ", deep %i\n", - dbi->name, header_bytes, payload_bytes, unused_bytes, deep); - } - - bool already_used = false; - for (unsigned n = 0; n < pgnumber; ++n) { - uint64_t spanpgno = pgno + n; - if (spanpgno >= alloc_pages) - problem_add("page", spanpgno, "wrong page-no", - "%s-page: %" PRIu64 " > %" PRIu64 ", deep %i", - pagetype_caption, spanpgno, alloc_pages, deep); - else if (walk.pagemap[spanpgno]) { - walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1]; - problem_add("page", spanpgno, - (branch && coll_dbi == dbi) ? "loop" : "already used", - "%s-page: by %s, deep %i", pagetype_caption, coll_dbi->name, - deep); - already_used = true; - } else { - walk.pagemap[spanpgno] = (short)(dbi - walk.dbi + 1); - dbi->pages.total += 1; - } - } - - if (already_used) - return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */ - : MDBX_SUCCESS; - } - - if (unused_bytes > page_size) - problem_add("page", pgno, "illegal unused-bytes", - "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0, - unused_bytes, envstat.ms_psize); - - if (header_bytes < (int)sizeof(long) || - (size_t)header_bytes >= envstat.ms_psize - sizeof(long)) - problem_add("page", pgno, "illegal header-length", - "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR, - pagetype_caption, sizeof(long), header_bytes, - envstat.ms_psize - sizeof(long)); - if (payload_bytes < 1) { - if (nentries > 1) { - problem_add("page", pgno, "zero size-of-entry", - "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries", - pagetype_caption, payload_bytes, nentries); - /* if ((size_t)header_bytes + unused_bytes < page_size) { - // LY: hush a misuse error - page_bytes = page_size; - } */ - } else { - problem_add("page", pgno, "empty", - "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR - " entries, deep %i", - pagetype_caption, payload_bytes, nentries, deep); - dbi->pages.empty += 1; - } - } - - if (pgnumber) { - if (page_bytes != page_size) { - problem_add("page", pgno, "misused", - "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR - "h + %" PRIuPTR "p + %" PRIuPTR "u), deep %i", - pagetype_caption, page_size, page_bytes, header_bytes, - payload_bytes, unused_bytes, deep); - if (page_size > page_bytes) - dbi->lost_bytes += page_size - page_bytes; - } else { - dbi->payload_bytes += payload_bytes + header_bytes; - walk.total_payload_bytes += payload_bytes + header_bytes; - } - } - - return check_user_break(); -} - -typedef int(visitor)(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data); -static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler, - bool silent); - -static int handle_userdb(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data) { - (void)record_number; - (void)key; - (void)data; - return check_user_break(); -} - -static int handle_freedb(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data) { - char *bad = ""; - pgno_t *iptr = data->iov_base; - - if (key->iov_len != sizeof(txnid_t)) - problem_add("entry", record_number, "wrong txn-id size", - "key-size %" PRIiPTR, key->iov_len); - else { - txnid_t txnid; - memcpy(&txnid, key->iov_base, sizeof(txnid)); - if (txnid < 1 || txnid > envinfo.mi_recent_txnid) - problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid); - else { - if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t)) - problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR, - data->iov_len); - size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0; - if (number < 1 || number > MDBX_PNL_MAX) - problem_add("entry", txnid, "wrong idl length", "%" PRIuPTR, number); - else if ((number + 1) * sizeof(pgno_t) > data->iov_len) { - problem_add("entry", txnid, "trimmed idl", - "%" PRIuSIZE " > %" PRIuSIZE " (corruption)", - (number + 1) * sizeof(pgno_t), data->iov_len); - number = data->iov_len / sizeof(pgno_t) - 1; - } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >= - /* LY: allow gap upto one page. it is ok - * and better than shink-and-retry inside mdbx_update_gc() */ - envstat.ms_psize) - problem_add("entry", txnid, "extra idl space", - "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", - (number + 1) * sizeof(pgno_t), data->iov_len); - - gc_pages += number; - if (envinfo.mi_latter_reader_txnid > txnid) - reclaimable_pages += number; - - pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : txn->mt_next_pgno; - pgno_t span = 1; - for (unsigned i = 0; i < number; ++i) { - if (check_user_break()) - return MDBX_EINTR; - const pgno_t pgno = iptr[i]; - if (pgno < NUM_METAS) - problem_add("entry", txnid, "wrong idl entry", - "pgno %" PRIaPGNO " < meta-pages %u", pgno, NUM_METAS); - else if (pgno >= backed_pages) - problem_add("entry", txnid, "wrong idl entry", - "pgno %" PRIaPGNO " > backed-pages %" PRIu64, pgno, - backed_pages); - else if (pgno >= alloc_pages) - problem_add("entry", txnid, "wrong idl entry", - "pgno %" PRIaPGNO " > alloc-pages %" PRIu64, pgno, - alloc_pages - 1); - else { - if (MDBX_PNL_DISORDERED(prev, pgno)) { - bad = " [bad sequence]"; - problem_add("entry", txnid, "bad sequence", - "%" PRIaPGNO " %c [%u].%" PRIaPGNO, prev, - (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), - i, pgno); - } - if (walk.pagemap) { - int idx = walk.pagemap[pgno]; - if (idx == 0) - walk.pagemap[pgno] = -1; - else if (idx > 0) - problem_add("page", pgno, "already used", "by %s", - walk.dbi[idx - 1].name); - else - problem_add("page", pgno, "already listed in GC", nullptr); - } - } - prev = pgno; - while (i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) - : pgno_sub(pgno, span))) - ++span; - } - if (verbose > 3 && !only_subdb) { - print(" transaction %" PRIaTXN ", %" PRIuPTR - " pages, maxspan %" PRIaPGNO "%s\n", - txnid, number, span, bad); - if (verbose > 4) { - for (unsigned i = 0; i < number; i += span) { - const pgno_t pgno = iptr[i]; - for (span = 1; - i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) - : pgno_sub(pgno, span)); - ++span) - ; - if (span > 1) { - print(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pgno, span); - } else - print(" %9" PRIaPGNO "\n", pgno); - } - } - } - } - } - - return check_user_break(); -} - -static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; -} - -static int handle_maindb(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data) { - char *name; - int rc; - size_t i; - - name = key->iov_base; - for (i = 0; i < key->iov_len; ++i) { - if (name[i] < ' ') - return handle_userdb(record_number, key, data); - } - - name = mdbx_malloc(key->iov_len + 1); - memcpy(name, key->iov_base, key->iov_len); - name[key->iov_len] = '\0'; - userdb_count++; - - rc = process_db(~0u, name, handle_userdb, false); - mdbx_free(name); - if (rc != MDBX_INCOMPATIBLE) - return rc; - - return handle_userdb(record_number, key, data); -} - -static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler, - bool silent) { - MDBX_cursor *mc; - MDBX_stat ms; - MDBX_val key, data; - MDBX_val prev_key, prev_data; - unsigned flags; - int rc, i; - struct problem *saved_list; - uint64_t problems_count; - - uint64_t record_count = 0, dups = 0; - uint64_t key_bytes = 0, data_bytes = 0; - - if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & mdbx_txn_flags(txn)) { - print(" ! abort processing '%s' due to a previous error\n", - dbi_name ? dbi_name : "@MAIN"); - return MDBX_BAD_TXN; - } - - if (dbi_handle == ~0u) { - rc = mdbx_dbi_open_ex( - txn, dbi_name, MDBX_ACCEDE, &dbi_handle, - (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr, - (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr); - if (rc) { - if (!dbi_name || - rc != - MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ { - error("mdbx_open '%s' failed, error %d %s\n", - dbi_name ? dbi_name : "main", rc, mdbx_strerror(rc)); - } - return rc; - } - } - - if (dbi_handle >= CORE_DBS && dbi_name && only_subdb && - strcmp(only_subdb, dbi_name) != 0) { - if (verbose) { - print("Skip processing '%s'...\n", dbi_name); - fflush(nullptr); - } - skipped_subdb++; - return MDBX_SUCCESS; - } - - if (!silent && verbose) { - print("Processing '%s'...\n", dbi_name ? dbi_name : "@MAIN"); - fflush(nullptr); - } - - rc = mdbx_dbi_flags(txn, dbi_handle, &flags); - if (rc) { - error("mdbx_dbi_flags failed, error %d %s\n", rc, mdbx_strerror(rc)); - return rc; - } - - rc = mdbx_dbi_stat(txn, dbi_handle, &ms, sizeof(ms)); - if (rc) { - error("mdbx_dbi_stat failed, error %d %s\n", rc, mdbx_strerror(rc)); - return rc; - } - - if (!silent && verbose) { - print(" - dbi-id %d, flags:", dbi_handle); - if (!flags) - print(" none"); - else { - for (i = 0; dbflags[i].bit; i++) - if (flags & dbflags[i].bit) - print(" %s", dbflags[i].name); - } - print(" (0x%02X)\n", flags); - if (verbose > 1) { - print(" - page size %u, entries %" PRIu64 "\n", ms.ms_psize, - ms.ms_entries); - print(" - b-tree depth %u, pages: branch %" PRIu64 ", leaf %" PRIu64 - ", overflow %" PRIu64 "\n", - ms.ms_depth, ms.ms_branch_pages, ms.ms_leaf_pages, - ms.ms_overflow_pages); - } - } - - walk_dbi_t *dbi = (dbi_handle < CORE_DBS) - ? &walk.dbi[dbi_handle] - : pagemap_lookup_dbi(dbi_name, true); - if (!dbi) { - error("too many DBIs or out of memory\n"); - return MDBX_ENOMEM; - } - if (!dont_traversal) { - const uint64_t subtotal_pages = - ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages; - if (subtotal_pages != dbi->pages.total) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", - "subtotal", subtotal_pages, dbi->pages.total); - if (ms.ms_branch_pages != dbi->pages.branch) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch", - ms.ms_branch_pages, dbi->pages.branch); - const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed; - if (ms.ms_leaf_pages != allleaf_pages) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", - "all-leaf", ms.ms_leaf_pages, allleaf_pages); - if (ms.ms_overflow_pages != dbi->pages.large_volume) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", - "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume); - } - rc = mdbx_cursor_open(txn, dbi_handle, &mc); - if (rc) { - error("mdbx_cursor_open failed, error %d %s\n", rc, mdbx_strerror(rc)); - return rc; - } - /* if (ignore_wrong_order) { - mc->mc_flags |= C_SKIPORD; - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; - } */ - - const size_t maxkeysize = mdbx_env_get_maxkeysize_ex(env, flags); - - saved_list = problems_push(); - prev_key.iov_base = nullptr; - prev_key.iov_len = 0; - prev_data.iov_base = nullptr; - prev_data.iov_len = 0; - rc = mdbx_cursor_get(mc, &key, &data, MDBX_FIRST); - while (rc == MDBX_SUCCESS) { - rc = check_user_break(); - if (rc) - goto bailout; - - bool bad_key = false; - if (key.iov_len > maxkeysize) { - problem_add("entry", record_count, "key length exceeds max-key-size", - "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize); - bad_key = true; - } else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) && - key.iov_len != sizeof(uint32_t)) { - problem_add("entry", record_count, "wrong key length", - "%" PRIuPTR " != 4or8", key.iov_len); - bad_key = true; - } - - bool bad_data = false; - if ((flags & MDBX_INTEGERDUP) && data.iov_len != sizeof(uint64_t) && - data.iov_len != sizeof(uint32_t)) { - problem_add("entry", record_count, "wrong data length", - "%" PRIuPTR " != 4or8", data.iov_len); - bad_data = true; - } - - if (prev_key.iov_base && !bad_data) { - if ((flags & MDBX_DUPFIXED) && prev_data.iov_len != data.iov_len) { - problem_add("entry", record_count, "different data length", - "%" PRIuPTR " != %" PRIuPTR, prev_data.iov_len, - data.iov_len); - bad_data = true; - } - - if (!bad_key) { - int cmp = mdbx_cmp(txn, dbi_handle, &key, &prev_key); - if (cmp == 0) { - ++dups; - if ((flags & MDBX_DUPSORT) == 0) { - problem_add("entry", record_count, "duplicated entries", nullptr); - if (data.iov_len == prev_data.iov_len && - memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0) { - problem_add("entry", record_count, "complete duplicate", nullptr); - } - } else if (!bad_data) { - cmp = mdbx_dcmp(txn, dbi_handle, &data, &prev_data); - if (cmp == 0) { - problem_add("entry", record_count, "complete duplicate", nullptr); - } else if (cmp < 0 && !ignore_wrong_order) { - problem_add("entry", record_count, "wrong order of multi-values", - nullptr); - } - } - } else if (cmp < 0 && !ignore_wrong_order) { - problem_add("entry", record_count, "wrong order of entries", nullptr); - } - } - } else if (verbose) { - if (flags & MDBX_INTEGERKEY) - print(" - fixed key-size %" PRIuPTR "\n", key.iov_len); - if (flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) - print(" - fixed data-size %" PRIuPTR "\n", data.iov_len); - } - - if (handler) { - rc = handler(record_count, &key, &data); - if (MDBX_IS_ERROR(rc)) - goto bailout; - } - - record_count++; - key_bytes += key.iov_len; - data_bytes += data.iov_len; - - if (!bad_key) - prev_key = key; - if (!bad_data) - prev_data = data; - rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT); - } - if (rc != MDBX_NOTFOUND) - error("mdbx_cursor_get failed, error %d %s\n", rc, mdbx_strerror(rc)); - else - rc = 0; - - if (record_count != ms.ms_entries) - problem_add("entry", record_count, "differentent number of entries", - "%" PRIu64 " != %" PRIu64, record_count, ms.ms_entries); -bailout: - problems_count = problems_pop(saved_list); - if (!silent && verbose) { - print(" - summary: %" PRIu64 " records, %" PRIu64 " dups, %" PRIu64 - " key's bytes, %" PRIu64 " data's " - "bytes, %" PRIu64 " problems\n", - record_count, dups, key_bytes, data_bytes, problems_count); - fflush(nullptr); - } - - mdbx_cursor_close(mc); - return (rc || problems_count) ? MDBX_RESULT_TRUE : MDBX_SUCCESS; -} - -static void usage(char *prog) { - fprintf(stderr, - "usage: %s [-V] [-v] [-q] [-c] [-w] [-d] [-i] [-s subdb] dbpath\n" - " -V\t\tprint version and exit\n" - " -v\t\tmore verbose, could be used multiple times\n" - " -q\t\tbe quiet\n" - " -c\t\tforce cooperative mode (don't try exclusive)\n" - " -w\t\tlock DB for writing while checking\n" - " -d\t\tdisable page-by-page traversal of B-tree\n" - " -i\t\tignore wrong order errors (for custom comparators case)\n" - " -s subdb\tprocess a specific subdatabase only\n", - prog); - exit(EXIT_INTERRUPTED); -} - -const char *meta_synctype(uint64_t sign) { - switch (sign) { - case MDBX_DATASIGN_NONE: - return "no-sync/legacy"; - case MDBX_DATASIGN_WEAK: - return "weak"; - default: - return "steady"; - } -} - -static __inline bool meta_ot(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, - uint64_t sign_b, const bool roolback2steady) { - if (txn_a == txn_b) - return SIGN_IS_STEADY(sign_b); - - if (roolback2steady && SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) - return SIGN_IS_STEADY(sign_b); - - return txn_a < txn_b; -} - -static __inline bool meta_eq(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, - uint64_t sign_b) { - if (txn_a != txn_b) - return false; - - if (SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) - return false; - - return true; -} - -static __inline int meta_recent(const bool roolback2steady) { - - if (meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, roolback2steady)) - return meta_ot(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, - roolback2steady) - ? 1 - : 2; - - return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, roolback2steady) - ? 2 - : 0; -} - -static __inline int meta_tail(int head) { - - if (head == 0) - return meta_ot(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) - ? 1 - : 2; - if (head == 1) - return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) - ? 0 - : 2; - if (head == 2) - return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, true) - ? 0 - : 1; - assert(false); - return -1; -} - -static int meta_steady(void) { return meta_recent(true); } - -static int meta_head(void) { return meta_recent(false); } - -void verbose_meta(int num, txnid_t txnid, uint64_t sign, uint64_t bootid_h, - uint64_t bootid_l) { - print(" - meta-%d: %s %" PRIu64, num, meta_synctype(sign), txnid); - bool stay = true; - const bool bootid_match = bootid_h == envinfo.mi_bootid.current.h && - bootid_l == envinfo.mi_bootid.current.l && - (bootid_h | bootid_l) != 0; - - const int steady = meta_steady(); - const int head = meta_head(); - if (num == steady && num == head) { - print(", head"); - stay = false; - } else if (num == steady) { - print(", head-steady"); - stay = false; - } else if (num == head) { - print(", head-weak%s", bootid_match ? "-intact (same boot-id)" : ""); - stay = false; - } - if (num == meta_tail(head)) { - print(", tail"); - stay = false; - } - if (stay) - print(", stay"); - - if (txnid > envinfo.mi_recent_txnid && - (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE) - print(", rolled-back %" PRIu64 " (%" PRIu64 " >>> %" PRIu64 ")", - txnid - envinfo.mi_recent_txnid, txnid, envinfo.mi_recent_txnid); - print("\n"); -} - -static uint64_t get_meta_txnid(const unsigned meta_id) { - switch (meta_id) { - default: - assert(false); - error("unexpected meta_id %u\n", meta_id); - return 0; - case 0: - return envinfo.mi_meta0_txnid; - case 1: - return envinfo.mi_meta1_txnid; - case 2: - return envinfo.mi_meta2_txnid; - } -} - -static void print_size(const char *prefix, const uint64_t value, - const char *suffix) { - const char sf[] = - "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */ - double k = 1024.0; - size_t i; - for (i = 0; sf[i + 1] && value / k > 1000.0; ++i) - k *= 1024; - print("%s%" PRIu64 " (%.2f %cb)%s", prefix, value, value / k, sf[i], suffix); -} - -int main(int argc, char *argv[]) { - int rc; - char *prog = argv[0]; - char *envname; - int problems_maindb = 0, problems_freedb = 0, problems_meta = 0; - bool locked = false; - - double elapsed; -#if defined(_WIN32) || defined(_WIN64) - uint64_t timestamp_start, timestamp_finish; - timestamp_start = GetTickCount64(); -#else - struct timespec timestamp_start, timestamp_finish; - if (clock_gettime(CLOCK_MONOTONIC, ×tamp_start)) { - rc = errno; - error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc)); - return EXIT_FAILURE_SYS; - } -#endif - - dbi_meta.name = "@META"; - dbi_free.name = "@GC"; - dbi_main.name = "@MAIN"; - atexit(pagemap_cleanup); - - if (argc < 2) - usage(prog); - - for (int i; (i = getopt(argc, argv, "Vvqnwcdis:")) != EOF;) { - switch (i) { - case 'V': - printf("mdbx_chk version %d.%d.%d.%d\n" - " - source: %s %s, commit %s, tree %s\n" - " - anchor: %s\n" - " - build: %s for %s by %s\n" - " - flags: %s\n" - " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); - return EXIT_SUCCESS; - case 'v': - verbose++; - break; - case 'q': - quiet = true; - break; - case 'n': - envflags |= MDBX_NOSUBDIR; - break; - case 'w': - envflags &= ~MDBX_RDONLY; - break; - case 'c': - envflags = (envflags & ~MDBX_EXCLUSIVE) | MDBX_ACCEDE; - break; - case 'd': - dont_traversal = true; - break; - case 's': - if (only_subdb && strcmp(only_subdb, optarg)) - usage(prog); - only_subdb = optarg; - break; - case 'i': - ignore_wrong_order = true; - break; - default: - usage(prog); - } - } - - if (optind != argc - 1) - usage(prog); - -#if defined(_WIN32) || defined(_WIN64) - SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); -#else -#ifdef SIGPIPE - signal(SIGPIPE, signal_handler); -#endif -#ifdef SIGHUP - signal(SIGHUP, signal_handler); -#endif - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); -#endif /* !WINDOWS */ - - envname = argv[optind]; - print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname, - (envflags & MDBX_RDONLY) ? "only" : "write"); - fflush(nullptr); - - rc = mdbx_env_create(&env); - if (rc) { - error("mdbx_env_create failed, error %d %s\n", rc, mdbx_strerror(rc)); - return rc < 0 ? EXIT_FAILURE_MDB : EXIT_FAILURE_SYS; - } - - rc = mdbx_env_set_maxdbs(env, MAX_DBI); - if (rc) { - error("mdbx_env_set_maxdbs failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - rc = mdbx_env_open(env, envname, envflags, 0); - if ((envflags & MDBX_EXCLUSIVE) && - (rc == MDBX_BUSY || -#if defined(_WIN32) || defined(_WIN64) - rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION -#else - rc == EBUSY || rc == EAGAIN -#endif - )) { - envflags &= ~MDBX_EXCLUSIVE; - rc = mdbx_env_open(env, envname, envflags | MDBX_ACCEDE, 0); - } - - if (rc) { - error("mdbx_env_open failed, error %d %s\n", rc, mdbx_strerror(rc)); - if (rc == MDBX_WANNA_RECOVERY && (envflags & MDBX_RDONLY)) - print("Please run %s in the read-write mode (with '-w' option).\n", prog); - goto bailout; - } - if (verbose) - print(" - %s mode\n", - (envflags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative"); - - if ((envflags & MDBX_RDONLY) == 0) { - rc = mdbx_txn_lock(env, false); - if (rc != MDBX_SUCCESS) { - error("mdbx_txn_lock failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - locked = true; - } - - rc = mdbx_txn_begin(env, nullptr, MDBX_RDONLY, &txn); - if (rc) { - error("mdbx_txn_begin() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - rc = mdbx_env_info_ex(env, txn, &envinfo, sizeof(envinfo)); - if (rc) { - error("mdbx_env_info failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - rc = mdbx_env_stat_ex(env, txn, &envstat, sizeof(envstat)); - if (rc) { - error("mdbx_env_stat failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - mdbx_filehandle_t dxb_fd; - rc = mdbx_env_get_fd(env, &dxb_fd); - if (rc) { - error("mdbx_env_get_fd failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - uint64_t dxb_filesize = 0; -#if defined(_WIN32) || defined(_WIN64) - { - BY_HANDLE_FILE_INFORMATION info; - if (!GetFileInformationByHandle(dxb_fd, &info)) - rc = GetLastError(); - else - dxb_filesize = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; - } -#else - { - struct stat st; - STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - if (fstat(dxb_fd, &st)) - rc = errno; - else - dxb_filesize = st.st_size; - } -#endif - if (rc) { - error("mdbx_filesize failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - errno = 0; - const uint64_t dxbfile_pages = dxb_filesize / envinfo.mi_dxb_pagesize; - alloc_pages = txn->mt_next_pgno; - backed_pages = envinfo.mi_geo.current / envinfo.mi_dxb_pagesize; - if (backed_pages > dxbfile_pages) { - print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", - backed_pages, dxbfile_pages); - ++problems_meta; - } - if (dxbfile_pages < NUM_METAS) - print(" ! file-pages %" PRIu64 " < %u\n", dxbfile_pages, NUM_METAS); - if (backed_pages < NUM_METAS) - print(" ! backed-pages %" PRIu64 " < %u\n", backed_pages, NUM_METAS); - if (backed_pages < NUM_METAS || dxbfile_pages < NUM_METAS) - goto bailout; - if (backed_pages > MAX_PAGENO) { - print(" ! backed-pages %" PRIu64 " > max-pages %" PRIaPGNO "\n", - backed_pages, MAX_PAGENO); - ++problems_meta; - backed_pages = MAX_PAGENO; - } - - if ((envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { - if (backed_pages > dxbfile_pages) { - print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", - backed_pages, dxbfile_pages); - ++problems_meta; - backed_pages = dxbfile_pages; - } - if (alloc_pages > backed_pages) { - print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", - alloc_pages, backed_pages); - ++problems_meta; - alloc_pages = backed_pages; - } - } else { - /* LY: DB may be shrinked by writer downto the allocated pages. */ - if (alloc_pages > backed_pages) { - print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", - alloc_pages, backed_pages); - ++problems_meta; - alloc_pages = backed_pages; - } - if (alloc_pages > dxbfile_pages) { - print(" ! alloc-pages %" PRIu64 " > file-pages %" PRIu64 "\n", - alloc_pages, dxbfile_pages); - ++problems_meta; - alloc_pages = dxbfile_pages; - } - if (backed_pages > dxbfile_pages) - backed_pages = dxbfile_pages; - } - - if (verbose) { - print(" - pagesize %u (%u system), max keysize %d..%d" - ", max readers %u\n", - envinfo.mi_dxb_pagesize, envinfo.mi_sys_pagesize, - mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT), - mdbx_env_get_maxkeysize_ex(env, 0), envinfo.mi_maxreaders); - print_size(" - mapsize ", envinfo.mi_mapsize, "\n"); - if (envinfo.mi_geo.lower == envinfo.mi_geo.upper) - print_size(" - fixed datafile: ", envinfo.mi_geo.current, ""); - else { - print_size(" - dynamic datafile: ", envinfo.mi_geo.lower, ""); - print_size(" .. ", envinfo.mi_geo.upper, ", "); - print_size("+", envinfo.mi_geo.grow, ", "); - print_size("-", envinfo.mi_geo.shrink, "\n"); - print_size(" - current datafile: ", envinfo.mi_geo.current, ""); - } - printf(", %" PRIu64 " pages\n", - envinfo.mi_geo.current / envinfo.mi_dxb_pagesize); -#if defined(_WIN32) || defined(_WIN64) - if (envinfo.mi_geo.shrink && envinfo.mi_geo.current != envinfo.mi_geo.upper) - print(" WARNING: Due Windows system limitations a " - "file couldn't\n be truncated while database " - "is opened. So, the size of\n database file " - "may by large than the database itself,\n " - "until it will be closed or reopened in read-write mode.\n"); -#endif - print(" - transactions: recent %" PRIu64 ", latter reader %" PRIu64 - ", lag %" PRIi64 "\n", - envinfo.mi_recent_txnid, envinfo.mi_latter_reader_txnid, - envinfo.mi_recent_txnid - envinfo.mi_latter_reader_txnid); - - verbose_meta(0, envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_bootid.meta0.h, envinfo.mi_bootid.meta0.l); - verbose_meta(1, envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, - envinfo.mi_bootid.meta1.h, envinfo.mi_bootid.meta1.l); - verbose_meta(2, envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, - envinfo.mi_bootid.meta2.h, envinfo.mi_bootid.meta2.l); - } - - if (verbose > 1) - print(" - performs check for meta-pages clashes\n"); - if (meta_eq(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign)) { - print(" ! meta-%d and meta-%d are clashed\n", 0, 1); - ++problems_meta; - } - if (meta_eq(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign)) { - print(" ! meta-%d and meta-%d are clashed\n", 1, 2); - ++problems_meta; - } - if (meta_eq(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, - envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign)) { - print(" ! meta-%d and meta-%d are clashed\n", 2, 0); - ++problems_meta; - } - - const unsigned steady_meta_id = meta_recent(true); - const uint64_t steady_meta_txnid = get_meta_txnid(steady_meta_id); - const unsigned weak_meta_id = meta_recent(false); - const uint64_t weak_meta_txnid = get_meta_txnid(weak_meta_id); - if (envflags & MDBX_EXCLUSIVE) { - if (verbose > 1) - print(" - performs full check recent-txn-id with meta-pages\n"); - if (steady_meta_txnid != envinfo.mi_recent_txnid) { - print(" ! steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64 - " != %" PRIi64 ")\n", - steady_meta_id, steady_meta_txnid, envinfo.mi_recent_txnid); - ++problems_meta; - } - } else if (locked) { - if (verbose > 1) - print(" - performs lite check recent-txn-id with meta-pages (not a " - "monopolistic mode)\n"); - if (weak_meta_txnid != envinfo.mi_recent_txnid) { - print(" ! weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64 - " != %" PRIi64 ")\n", - weak_meta_id, weak_meta_txnid, envinfo.mi_recent_txnid); - ++problems_meta; - } - } else if (verbose) { - print(" - skip check recent-txn-id with meta-pages (monopolistic or " - "read-write mode only)\n"); - } - total_problems += problems_meta; - - if (!dont_traversal) { - struct problem *saved_list; - size_t traversal_problems; - uint64_t empty_pages, lost_bytes; - - print("Traversal b-tree by txn#%" PRIaTXN "...\n", txn->mt_txnid); - fflush(nullptr); - walk.pagemap = mdbx_calloc((size_t)backed_pages, sizeof(*walk.pagemap)); - if (!walk.pagemap) { - rc = errno ? errno : MDBX_ENOMEM; - error("calloc failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - saved_list = problems_push(); - rc = mdbx_env_pgwalk(txn, pgvisitor, nullptr, ignore_wrong_order); - traversal_problems = problems_pop(saved_list); - - if (rc) { - if (rc != MDBX_EINTR || !check_user_break()) - error("mdbx_env_pgwalk failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - for (uint64_t n = 0; n < alloc_pages; ++n) - if (!walk.pagemap[n]) - unused_pages += 1; - - empty_pages = lost_bytes = 0; - for (walk_dbi_t *dbi = &dbi_main; dbi < ARRAY_END(walk.dbi) && dbi->name; - ++dbi) { - empty_pages += dbi->pages.empty; - lost_bytes += dbi->lost_bytes; - } - - if (verbose) { - uint64_t total_page_bytes = walk.pgcount * envstat.ms_psize; - print(" - pages: walked %" PRIu64 ", left/unused %" PRIu64 "\n", - walk.pgcount, unused_pages); - if (verbose > 1) { - for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; - ++dbi) { - print(" %s: subtotal %" PRIu64, dbi->name, dbi->pages.total); - if (dbi->pages.other && dbi->pages.other != dbi->pages.total) - print(", other %" PRIu64, dbi->pages.other); - if (dbi->pages.branch) - print(", branch %" PRIu64, dbi->pages.branch); - if (dbi->pages.large_count) - print(", large %" PRIu64, dbi->pages.large_count); - uint64_t all_leaf = dbi->pages.leaf + dbi->pages.leaf_dupfixed; - if (all_leaf) { - print(", leaf %" PRIu64, all_leaf); - if (verbose > 2 && - (dbi->pages.leaf_dupfixed | dbi->pages.subleaf_dupsort | - dbi->pages.subleaf_dupsort)) - print(" (usual %" PRIu64 ", sub-dupsort %" PRIu64 - ", dupfixed %" PRIu64 ", sub-dupfixed %" PRIu64 ")", - dbi->pages.leaf, dbi->pages.subleaf_dupsort, - dbi->pages.leaf_dupfixed, dbi->pages.subleaf_dupfixed); - } - print("\n"); - } - } - - if (verbose > 1) - print(" - usage: total %" PRIu64 " bytes, payload %" PRIu64 - " (%.1f%%), unused " - "%" PRIu64 " (%.1f%%)\n", - total_page_bytes, walk.total_payload_bytes, - walk.total_payload_bytes * 100.0 / total_page_bytes, - total_page_bytes - walk.total_payload_bytes, - (total_page_bytes - walk.total_payload_bytes) * 100.0 / - total_page_bytes); - if (verbose > 2) { - for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; - ++dbi) - if (dbi->pages.total) { - uint64_t dbi_bytes = dbi->pages.total * envstat.ms_psize; - print(" %s: subtotal %" PRIu64 " bytes (%.1f%%)," - " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)", - dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes, - dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes, - dbi_bytes - dbi->payload_bytes, - (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes); - if (dbi->pages.empty) - print(", %" PRIu64 " empty pages", dbi->pages.empty); - if (dbi->lost_bytes) - print(", %" PRIu64 " bytes lost", dbi->lost_bytes); - print("\n"); - } else - print(" %s: empty\n", dbi->name); - } - print(" - summary: average fill %.1f%%", - walk.total_payload_bytes * 100.0 / total_page_bytes); - if (empty_pages) - print(", %" PRIu64 " empty pages", empty_pages); - if (lost_bytes) - print(", %" PRIu64 " bytes lost", lost_bytes); - print(", %" PRIuPTR " problems\n", traversal_problems); - } - } else if (verbose) { - print("Skipping b-tree walk...\n"); - fflush(nullptr); - } - - if (!verbose) - print("Iterating DBIs...\n"); - problems_maindb = process_db(~0u, /* MAIN_DBI */ nullptr, nullptr, false); - problems_freedb = process_db(FREE_DBI, "@GC", handle_freedb, false); - - if (verbose) { - uint64_t value = envinfo.mi_mapsize / envstat.ms_psize; - double percent = value / 100.0; - print(" - space: %" PRIu64 " total pages", value); - print(", backed %" PRIu64 " (%.1f%%)", backed_pages, - backed_pages / percent); - print(", allocated %" PRIu64 " (%.1f%%)", alloc_pages, - alloc_pages / percent); - - if (verbose > 1) { - value = envinfo.mi_mapsize / envstat.ms_psize - alloc_pages; - print(", remained %" PRIu64 " (%.1f%%)", value, value / percent); - - value = alloc_pages - gc_pages; - print(", used %" PRIu64 " (%.1f%%)", value, value / percent); - - print(", gc %" PRIu64 " (%.1f%%)", gc_pages, gc_pages / percent); - - value = gc_pages - reclaimable_pages; - print(", detained %" PRIu64 " (%.1f%%)", value, value / percent); - - print(", reclaimable %" PRIu64 " (%.1f%%)", reclaimable_pages, - reclaimable_pages / percent); - } - - value = - envinfo.mi_mapsize / envstat.ms_psize - alloc_pages + reclaimable_pages; - print(", available %" PRIu64 " (%.1f%%)\n", value, value / percent); - } - - if (problems_maindb == 0 && problems_freedb == 0) { - if (!dont_traversal && - (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { - if (walk.pgcount != alloc_pages - gc_pages) { - error("used pages mismatch (%" PRIu64 "(walked) != %" PRIu64 - "(allocated - GC))\n", - walk.pgcount, alloc_pages - gc_pages); - } - if (unused_pages != gc_pages) { - error("gc pages mismatch (%" PRIu64 "(walked) != %" PRIu64 "(GC))\n", - unused_pages, gc_pages); - } - } else if (verbose) { - print(" - skip check used and gc pages (btree-traversal with " - "monopolistic or read-write mode only)\n"); - } - - if (!process_db(MAIN_DBI, nullptr, handle_maindb, true)) { - if (!userdb_count && verbose) - print(" - does not contain multiple databases\n"); - } - } - - if (rc == 0 && total_problems == 1 && problems_meta == 1 && !dont_traversal && - (envflags & MDBX_RDONLY) == 0 && !only_subdb && - steady_meta_txnid < envinfo.mi_recent_txnid) { - print("Perform sync-to-disk for make steady checkpoint at txn-id #%" PRIi64 - "\n", - envinfo.mi_recent_txnid); - fflush(nullptr); - if (locked) { - mdbx_txn_unlock(env); - locked = false; - } - rc = mdbx_env_sync_ex(env, true, false); - if (rc != MDBX_SUCCESS) - error("mdbx_env_pgwalk failed, error %d %s\n", rc, mdbx_strerror(rc)); - else { - total_problems -= 1; - problems_meta -= 1; - } - } - -bailout: - if (txn) - mdbx_txn_abort(txn); - if (locked) { - mdbx_txn_unlock(env); - locked = false; - } - if (env) { - const bool dont_sync = rc != 0 || total_problems; - mdbx_env_close_ex(env, dont_sync); - } - fflush(nullptr); - if (rc) { - if (rc < 0) - return user_break ? EXIT_INTERRUPTED : EXIT_FAILURE_SYS; - return EXIT_FAILURE_MDB; - } - -#if defined(_WIN32) || defined(_WIN64) - timestamp_finish = GetTickCount64(); - elapsed = (timestamp_finish - timestamp_start) * 1e-3; -#else - if (clock_gettime(CLOCK_MONOTONIC, ×tamp_finish)) { - rc = errno; - error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc)); - return EXIT_FAILURE_SYS; - } - elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec + - (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9; -#endif /* !WINDOWS */ - - if (total_problems) { - print("Total %" PRIu64 " error%s detected, elapsed %.3f seconds.\n", - total_problems, (total_problems > 1) ? "s are" : " is", elapsed); - if (problems_meta || problems_maindb || problems_freedb) - return EXIT_FAILURE_CHECK_MAJOR; - return EXIT_FAILURE_CHECK_MINOR; - } - print("No error is detected, elapsed %.3f seconds\n", elapsed); - return EXIT_SUCCESS; -} diff --git a/libs/libmdbx/src/src/mdbx_copy.c b/libs/libmdbx/src/src/mdbx_copy.c deleted file mode 100644 index cde9573a15..0000000000 --- a/libs/libmdbx/src/src/mdbx_copy.c +++ /dev/null @@ -1,142 +0,0 @@ -/* mdbx_copy.c - memory-mapped database backup tool */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#ifdef _MSC_VER -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif -#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ -#endif /* _MSC_VER (warnings) */ - -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "internals.h" - -#if defined(_WIN32) || defined(_WIN64) -#include "wingetopt.h" - -static volatile BOOL user_break; -static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { - (void)dwCtrlType; - user_break = true; - return true; -} - -#else /* WINDOWS */ - -static volatile sig_atomic_t user_break; -static void signal_handler(int sig) { - (void)sig; - user_break = 1; -} - -#endif /* !WINDOWS */ - -static void usage(const char *prog) { - fprintf(stderr, - "usage: %s [-V] [-q] [-c] src_path [dest_path]\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -c\t\tenable compactification (skip unused pages)\n" - " src_path\tsource database\n" - " dest_path\tdestination (stdout if not specified)\n", - prog); - exit(EXIT_FAILURE); -} - -int main(int argc, char *argv[]) { - int rc; - MDBX_env *env = NULL; - const char *progname = argv[0], *act; - unsigned flags = MDBX_RDONLY; - unsigned cpflags = 0; - bool quiet = false; - - for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { - if (argv[1][1] == 'n' && argv[1][2] == '\0') - flags |= MDBX_NOSUBDIR; - else if (argv[1][1] == 'c' && argv[1][2] == '\0') - cpflags |= MDBX_CP_COMPACT; - else if (argv[1][1] == 'q' && argv[1][2] == '\0') - quiet = true; - else if ((argv[1][1] == 'h' && argv[1][2] == '\0') || - strcmp(argv[1], "--help") == 0) - usage(progname); - else if (argv[1][1] == 'V' && argv[1][2] == '\0') { - printf("mdbx_copy version %d.%d.%d.%d\n" - " - source: %s %s, commit %s, tree %s\n" - " - anchor: %s\n" - " - build: %s for %s by %s\n" - " - flags: %s\n" - " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); - return EXIT_SUCCESS; - } else - argc = 0; - } - - if (argc < 2 || argc > 3) - usage(progname); - -#if defined(_WIN32) || defined(_WIN64) - SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); -#else -#ifdef SIGPIPE - signal(SIGPIPE, signal_handler); -#endif -#ifdef SIGHUP - signal(SIGHUP, signal_handler); -#endif - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); -#endif /* !WINDOWS */ - - if (!quiet) { - fprintf((argc == 2) ? stderr : stdout, - "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]); - fflush(NULL); - } - - act = "opening environment"; - rc = mdbx_env_create(&env); - if (rc == MDBX_SUCCESS) { - rc = mdbx_env_open(env, argv[1], flags, 0640); - } - if (rc == MDBX_SUCCESS) { - act = "copying"; - if (argc == 2) { - mdbx_filehandle_t fd; -#if defined(_WIN32) || defined(_WIN64) - fd = GetStdHandle(STD_OUTPUT_HANDLE); -#else - fd = fileno(stdout); -#endif - rc = mdbx_env_copy2fd(env, fd, cpflags); - } else - rc = mdbx_env_copy(env, argv[2], cpflags); - } - if (rc) - fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, - mdbx_strerror(rc)); - mdbx_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/libs/libmdbx/src/src/mdbx_dump.c b/libs/libmdbx/src/src/mdbx_dump.c deleted file mode 100644 index b921a8139b..0000000000 --- a/libs/libmdbx/src/src/mdbx_dump.c +++ /dev/null @@ -1,481 +0,0 @@ -/* mdbx_dump.c - memory-mapped database dump tool */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#ifdef _MSC_VER -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif -#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ -#endif /* _MSC_VER (warnings) */ - -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "internals.h" - -#include <ctype.h> - -#define PRINT 1 -#define GLOBAL 2 -static int mode = GLOBAL; - -typedef struct flagbit { - int bit; - char *name; -} flagbit; - -flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"}, - {MDBX_DUPSORT, "dupsort"}, - {MDBX_INTEGERKEY, "integerkey"}, - {MDBX_DUPFIXED, "dupfixed"}, - {MDBX_INTEGERDUP, "integerdup"}, - {MDBX_REVERSEDUP, "reversedup"}, - {0, nullptr}}; - -#if defined(_WIN32) || defined(_WIN64) -#include "wingetopt.h" - -static volatile BOOL user_break; -static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { - (void)dwCtrlType; - user_break = true; - return true; -} - -#else /* WINDOWS */ - -static volatile sig_atomic_t user_break; -static void signal_handler(int sig) { - (void)sig; - user_break = 1; -} - -#endif /* !WINDOWS */ - -static const char hexc[] = "0123456789abcdef"; - -static void dumpbyte(unsigned char c) { - putchar(hexc[c >> 4]); - putchar(hexc[c & 0xf]); -} - -static void text(MDBX_val *v) { - unsigned char *c, *end; - - putchar(' '); - c = v->iov_base; - end = c + v->iov_len; - while (c < end) { - if (isprint(*c) && *c != '\\') { - putchar(*c); - } else { - putchar('\\'); - dumpbyte(*c); - } - c++; - } - putchar('\n'); -} - -static void dumpval(MDBX_val *v) { - unsigned char *c, *end; - - putchar(' '); - c = v->iov_base; - end = c + v->iov_len; - while (c < end) - dumpbyte(*c++); - putchar('\n'); -} - -bool quiet = false, rescue = false; -const char *prog; -static void error(const char *func, int rc) { - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); -} - -/* Dump in BDB-compatible format */ -static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { - unsigned int flags; - int rc = mdbx_dbi_flags(txn, dbi, &flags); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_flags", rc); - return rc; - } - - MDBX_stat ms; - rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms)); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_stat", rc); - return rc; - } - - MDBX_envinfo info; - rc = mdbx_env_info_ex(mdbx_txn_env(txn), txn, &info, sizeof(info)); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_info_ex", rc); - return rc; - } - - printf("VERSION=3\n"); - if (mode & GLOBAL) { - mode -= GLOBAL; - if (info.mi_geo.upper != info.mi_geo.lower) - printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 - ",g%" PRIu64 "\n", - info.mi_geo.lower, info.mi_geo.current, info.mi_geo.upper, - info.mi_geo.shrink, info.mi_geo.grow); - printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper); - printf("maxreaders=%u\n", info.mi_maxreaders); - - mdbx_canary canary; - rc = mdbx_canary_get(txn, &canary); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_canary_get", rc); - return rc; - } - if (canary.v) - printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n", - canary.v, canary.x, canary.y, canary.z); - } - printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); - if (name) - printf("database=%s\n", name); - printf("type=btree\n"); - printf("db_pagesize=%u\n", ms.ms_psize); - /* if (ms.ms_mod_txnid) - printf("txnid=%" PRIaTXN "\n", ms.ms_mod_txnid); - else if (!name) - printf("txnid=%" PRIaTXN "\n", mdbx_txn_id(txn)); */ - - printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | - MDBX_INTEGERDUP | MDBX_REVERSEDUP)) - ? 1 - : 0); - for (int i = 0; dbflags[i].bit; i++) - if (flags & dbflags[i].bit) - printf("%s=1\n", dbflags[i].name); - - uint64_t sequence; - rc = mdbx_dbi_sequence(txn, dbi, &sequence, 0); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_sequence", rc); - return rc; - } - if (sequence) - printf("sequence=%" PRIu64 "\n", sequence); - - printf("HEADER=END\n"); /*-------------------------------------------------*/ - - MDBX_cursor *cursor; - MDBX_val key, data; - rc = mdbx_cursor_open(txn, dbi, &cursor); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); - return rc; - } - if (MDBX_DEBUG > 0 && rescue) { - cursor->mc_flags |= C_SKIPORD; - if (cursor->mc_xcursor) - cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; - } - - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == - MDBX_SUCCESS) { - if (user_break) { - rc = MDBX_EINTR; - break; - } - if (mode & PRINT) { - text(&key); - text(&data); - } else { - dumpval(&key); - dumpval(&data); - } - } - printf("DATA=END\n"); - if (rc == MDBX_NOTFOUND) - rc = MDBX_SUCCESS; - if (unlikely(rc != MDBX_SUCCESS)) - error("mdbx_cursor_get", rc); - return rc; -} - -static void usage(void) { - fprintf(stderr, - "usage: %s [-V] [-q] [-f file] [-l] [-p] [-a|-s subdb] [-r] " - "dbpath\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -f\t\twrite to file instead of stdout\n" - " -l\t\tlist subDBs and exit\n" - " -p\t\tuse printable characters\n" - " -a\t\tdump main DB and all subDBs,\n" - " \t\tby default dump only the main DB\n" - " -s\t\tdump only the named subDB\n" - " -r\t\trescure mode (ignore errors to dump corrupted DB)\n", - prog); - exit(EXIT_FAILURE); -} - -static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; -} - -int main(int argc, char *argv[]) { - int i, rc; - MDBX_env *env; - MDBX_txn *txn; - MDBX_dbi dbi; - prog = argv[0]; - char *envname; - char *subname = nullptr; - unsigned envflags = 0; - bool alldbs = false, list = false; - - if (argc < 2) - usage(); - - while ((i = getopt(argc, argv, "af:lnps:Vrq")) != EOF) { - switch (i) { - case 'V': - printf("mdbx_dump version %d.%d.%d.%d\n" - " - source: %s %s, commit %s, tree %s\n" - " - anchor: %s\n" - " - build: %s for %s by %s\n" - " - flags: %s\n" - " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); - return EXIT_SUCCESS; - case 'l': - list = true; - /*FALLTHROUGH*/; - __fallthrough; - case 'a': - if (subname) - usage(); - alldbs = true; - break; - case 'f': - if (freopen(optarg, "w", stdout) == nullptr) { - fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, - mdbx_strerror(errno)); - exit(EXIT_FAILURE); - } - break; - case 'n': - envflags |= MDBX_NOSUBDIR; - break; - case 'p': - mode |= PRINT; - break; - case 's': - if (alldbs) - usage(); - subname = optarg; - break; - case 'q': - quiet = true; - break; - case 'r': - rescue = true; - break; - default: - usage(); - } - } - - if (optind != argc - 1) - usage(); - -#if defined(_WIN32) || defined(_WIN64) - SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); -#else -#ifdef SIGPIPE - signal(SIGPIPE, signal_handler); -#endif -#ifdef SIGHUP - signal(SIGHUP, signal_handler); -#endif - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); -#endif /* !WINDOWS */ - - envname = argv[optind]; - if (!quiet) { - fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname); - fflush(nullptr); - } - - rc = mdbx_env_create(&env); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_create", rc); - return EXIT_FAILURE; - } - - if (alldbs || subname) { - rc = mdbx_env_set_maxdbs(env, 2); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_set_maxdbs", rc); - goto env_close; - } - } - - rc = mdbx_env_open( - env, envname, - envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE : MDBX_RDONLY), 0); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_open", rc); - goto env_close; - } - - rc = mdbx_txn_begin(env, nullptr, MDBX_RDONLY, &txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_begin", rc); - goto env_close; - } - - rc = mdbx_dbi_open(txn, subname, MDBX_ACCEDE, &dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_open", rc); - goto txn_abort; - } - - if (alldbs) { - assert(dbi == MAIN_DBI); - - MDBX_cursor *cursor; - rc = mdbx_cursor_open(txn, MAIN_DBI, &cursor); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); - goto txn_abort; - } - if (MDBX_DEBUG > 0 && rescue) { - cursor->mc_flags |= C_SKIPORD; - if (cursor->mc_xcursor) - cursor->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; - } - - bool have_raw = false; - int count = 0; - MDBX_val key; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { - if (user_break) { - rc = MDBX_EINTR; - break; - } - - if (memchr(key.iov_base, '\0', key.iov_len)) - continue; - subname = mdbx_malloc(key.iov_len + 1); - memcpy(subname, key.iov_base, key.iov_len); - subname[key.iov_len] = '\0'; - - MDBX_dbi sub_dbi; - rc = mdbx_dbi_open_ex(txn, subname, MDBX_ACCEDE, &sub_dbi, - rescue ? equal_or_greater : nullptr, - rescue ? equal_or_greater : nullptr); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == MDBX_INCOMPATIBLE) { - have_raw = true; - continue; - } - error("mdbx_dbi_open", rc); - if (!rescue) - break; - } else { - count++; - if (list) { - printf("%s\n", subname); - } else { - rc = dump_sdb(txn, sub_dbi, subname); - if (unlikely(rc != MDBX_SUCCESS)) { - if (!rescue) - break; - fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, - envname, mdbx_strerror(rc), subname); - /* Here is a hack for rescue mode, don't do that: - * - we should restart transaction in case error due - * database corruption; - * - but we won't close cursor, reopen and re-positioning it - * for new a transaction; - * - this is possible since DB is opened in read-only exclusive - * mode and transaction is the same, i.e. has the same address - * and so on. */ - rc = mdbx_txn_reset(txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_reset", rc); - goto env_close; - } - rc = mdbx_txn_renew(txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_renew", rc); - goto env_close; - } - } - } - rc = mdbx_dbi_close(env, sub_dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_close", rc); - break; - } - } - mdbx_free(subname); - } - mdbx_cursor_close(cursor); - cursor = nullptr; - - if (have_raw && (!count /* || rescue */)) - rc = dump_sdb(txn, MAIN_DBI, nullptr); - else if (!count) { - fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, - envname); - rc = MDBX_NOTFOUND; - } - } else { - rc = dump_sdb(txn, dbi, subname); - } - - switch (rc) { - case MDBX_NOTFOUND: - rc = MDBX_SUCCESS; - case MDBX_SUCCESS: - break; - case MDBX_EINTR: - fprintf(stderr, "Interrupted by signal/user\n"); - break; - default: - if (unlikely(rc != MDBX_SUCCESS)) - error("mdbx_cursor_get", rc); - } - - mdbx_dbi_close(env, dbi); -txn_abort: - mdbx_txn_abort(txn); -env_close: - mdbx_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/libs/libmdbx/src/src/mdbx_load.c b/libs/libmdbx/src/src/mdbx_load.c deleted file mode 100644 index 463f020706..0000000000 --- a/libs/libmdbx/src/src/mdbx_load.c +++ /dev/null @@ -1,820 +0,0 @@ -/* mdbx_load.c - memory-mapped database load tool */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#ifdef _MSC_VER -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif -#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ -#endif /* _MSC_VER (warnings) */ - -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "internals.h" - -#include <ctype.h> - -#if defined(_WIN32) || defined(_WIN64) -#include "wingetopt.h" - -static volatile BOOL user_break; -static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { - (void)dwCtrlType; - user_break = true; - return true; -} - -#else /* WINDOWS */ - -static volatile sig_atomic_t user_break; -static void signal_handler(int sig) { - (void)sig; - user_break = 1; -} - -#endif /* !WINDOWS */ - -static char *prog; -static size_t lineno; -static void error(const char *func, int rc) { - if (lineno) - fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n", - prog, lineno, func, rc, mdbx_strerror(rc)); - else - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, - mdbx_strerror(rc)); -} - -static char *valstr(char *line, const char *item) { - const size_t len = strlen(item); - if (strncmp(line, item, len) != 0) - return nullptr; - if (line[len] != '=') { - if (line[len] > ' ') - return nullptr; - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", - prog, lineno, item); - exit(EXIT_FAILURE); - } - char *ptr = strchr(line, '\n'); - if (ptr) - *ptr = '\0'; - return line + len + 1; -} - -static bool valnum(char *line, const char *item, uint64_t *value) { - char *str = valstr(line, item); - if (!str) - return false; - - char *end = nullptr; - *value = strtoull(str, &end, 0); - if (end && *end) { - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n", prog, - lineno, item); - exit(EXIT_FAILURE); - } - return true; -} - -static bool valbool(char *line, const char *item, bool *value) { - uint64_t u64; - if (!valnum(line, item, &u64)) - return false; - - if (u64 > 1) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n", prog, - lineno, item); - exit(EXIT_FAILURE); - } - *value = u64 != 0; - return true; -} - -/*----------------------------------------------------------------------------*/ - -static char *subname = nullptr; -static int dbi_flags; -static txnid_t txnid; -static uint64_t sequence; -static mdbx_canary canary; -static MDBX_envinfo envinfo; - -#define PRINT 1 -#define NOHDR 2 -#define GLOBAL 4 -static int mode = GLOBAL; - -static MDBX_val kbuf, dbuf; -static MDBX_val k0buf; - -#define STRLENOF(s) (sizeof(s) - 1) - -typedef struct flagbit { - unsigned bit; - char *name; - unsigned len; -} flagbit; - -#define S(s) s, STRLENOF(s) - -flagbit dbflags[] = { - {MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")}, - {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")}, - {MDBX_DUPFIXED, S("dupfixed")}, {MDBX_INTEGERDUP, S("integerdup")}, - {MDBX_REVERSEDUP, S("reversedup")}, {0, nullptr, 0}}; - -static int readhdr(void) { - /* reset parameters */ - if (subname) { - free(subname); - subname = nullptr; - } - dbi_flags = 0; - txnid = 0; - sequence = 0; - - while (true) { - errno = 0; - if (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) == nullptr) - return errno ? errno : EOF; - if (user_break) - return MDBX_EINTR; - - lineno++; - uint64_t u64; - - if (valnum(dbuf.iov_base, "VERSION", &u64)) { - if (u64 != 3) { - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value %" PRIu64 - " for %s\n", - prog, lineno, u64, "VERSION"); - exit(EXIT_FAILURE); - } - continue; - } - - if (valnum(dbuf.iov_base, "db_pagesize", &u64)) { - if (!(mode & GLOBAL) && envinfo.mi_dxb_pagesize != u64) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "db_pagesize"); - else if (u64 < MDBX_MIN_PAGESIZE || u64 > MDBX_MAX_PAGESIZE) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64 - " for %s\n", - prog, lineno, u64, "db_pagesize"); - else - envinfo.mi_dxb_pagesize = (uint32_t)u64; - continue; - } - - char *str = valstr(dbuf.iov_base, "format"); - if (str) { - if (strcmp(str, "print") == 0) { - mode |= PRINT; - continue; - } - if (strcmp(str, "bytevalue") == 0) { - mode &= ~PRINT; - continue; - } - fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", - prog, lineno, str, "format"); - exit(EXIT_FAILURE); - } - - str = valstr(dbuf.iov_base, "database"); - if (str) { - if (*str) { - subname = mdbx_strdup(str); - if (!subname) { - perror("strdup()"); - exit(EXIT_FAILURE); - } - } - continue; - } - - str = valstr(dbuf.iov_base, "type"); - if (str) { - if (strcmp(str, "btree") != 0) { - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, - lineno, str, "type"); - exit(EXIT_FAILURE); - } - continue; - } - - if (valnum(dbuf.iov_base, "mapaddr", &u64)) { - if (u64) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "mapaddr"); - continue; - } - - if (valnum(dbuf.iov_base, "mapsize", &u64)) { - if (!(mode & GLOBAL)) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "mapsize"); - else if (u64 < MIN_MAPSIZE || u64 > MAX_MAPSIZE64) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "mapsize"); - else - envinfo.mi_mapsize = (size_t)u64; - continue; - } - - if (valnum(dbuf.iov_base, "maxreaders", &u64)) { - if (!(mode & GLOBAL)) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "maxreaders"); - else if (u64 < 1 || u64 > MDBX_READERS_LIMIT) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "maxreaders"); - else - envinfo.mi_maxreaders = (int)u64; - continue; - } - - if (valnum(dbuf.iov_base, "txnid", &u64)) { - if (u64 < MIN_TXNID || u64 > MAX_TXNID) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "txnid"); - txnid = u64; - continue; - } - - if (valnum(dbuf.iov_base, "sequence", &u64)) { - sequence = u64; - continue; - } - - str = valstr(dbuf.iov_base, "geometry"); - if (str) { - if (!(mode & GLOBAL)) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore values %s" - " for '%s' in non-global context\n", - prog, lineno, str, "geometry"); - else if (sscanf(str, - "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 - ",g%" PRIu64, - &envinfo.mi_geo.lower, &envinfo.mi_geo.current, - &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink, - &envinfo.mi_geo.grow) != 5) { - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", - prog, lineno, "geometry"); - exit(EXIT_FAILURE); - } - continue; - } - - str = valstr(dbuf.iov_base, "canary"); - if (str) { - if (!(mode & GLOBAL)) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore values %s" - " for '%s' in non-global context\n", - prog, lineno, str, "canary"); - else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64, - &canary.v, &canary.x, &canary.y, &canary.z) != 4) { - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", - prog, lineno, "canary"); - exit(EXIT_FAILURE); - } - continue; - } - - for (int i = 0; dbflags[i].bit; i++) { - bool value; - if (valbool(dbuf.iov_base, dbflags[i].name, &value)) { - if (value) - dbi_flags |= dbflags[i].bit; - else - dbi_flags &= ~dbflags[i].bit; - goto next; - } - } - - str = valstr(dbuf.iov_base, "HEADER"); - if (str) { - if (strcmp(str, "END") == 0) - return MDBX_SUCCESS; - } - - fprintf(stderr, - "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", prog, - lineno, (char *)dbuf.iov_base); - next:; - } - return EOF; -} - -static int badend(void) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, - lineno); - return errno ? errno : MDBX_ENODATA; -} - -static int unhex(unsigned char *c2) { - int x, c; - x = *c2++ & 0x4f; - if (x & 0x40) - x -= 55; - c = x << 4; - x = *c2 & 0x4f; - if (x & 0x40) - x -= 55; - c |= x; - return c; -} - -static int readline(MDBX_val *out, MDBX_val *buf) { - unsigned char *c1, *c2, *end; - size_t len, l2; - int c; - - if (user_break) - return MDBX_EINTR; - - errno = 0; - if (!(mode & NOHDR)) { - c = fgetc(stdin); - if (c == EOF) - return errno ? errno : EOF; - if (c != ' ') { - lineno++; - errno = 0; - if (fgets(buf->iov_base, (int)buf->iov_len, stdin)) { - if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END"))) - return EOF; - } - return badend(); - } - } - if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == nullptr) - return errno ? errno : EOF; - lineno++; - - c1 = buf->iov_base; - len = strlen((char *)c1); - l2 = len; - - /* Is buffer too short? */ - while (c1[len - 1] != '\n') { - buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2); - if (!buf->iov_base) { - fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", - prog, lineno); - return MDBX_ENOMEM; - } - c1 = buf->iov_base; - c1 += l2; - errno = 0; - if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == nullptr) - return errno ? errno : EOF; - buf->iov_len *= 2; - len = strlen((char *)c1); - l2 += len; - } - c1 = c2 = buf->iov_base; - len = l2; - c1[--len] = '\0'; - end = c1 + len; - - if (mode & PRINT) { - while (c2 < end) { - if (unlikely(*c2 == '\\')) { - if (c2[1] == '\\') { - *c1++ = '\\'; - } else { - if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) - return badend(); - *c1++ = (char)unhex(++c2); - } - c2 += 2; - } else { - /* copies are redundant when no escapes were used */ - *c1++ = *c2++; - } - } - } else { - /* odd length not allowed */ - if (len & 1) - return badend(); - while (c2 < end) { - if (!isxdigit(*c2) || !isxdigit(c2[1])) - return badend(); - *c1++ = (char)unhex(c2); - c2 += 2; - } - } - c2 = out->iov_base = buf->iov_base; - out->iov_len = c1 - c2; - - return MDBX_SUCCESS; -} - -static void usage(void) { - fprintf(stderr, - "usage: %s [-V] [-q] [-a] [-f file] [-s name] [-N] [-T] [-r] " - "dbpath\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -a\t\tappend records in input order (required for custom " - "comparators)\n" - " -f file\tread from file instead of stdin\n" - " -s name\tload into named subDB\n" - " -N\t\tuse NOOVERWRITE on puts\n" - " -T\t\tread plaintext\n" - " -r\t\trescure mode (ignore errors to load corrupted DB dump)\n", - prog); - exit(EXIT_FAILURE); -} - -static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; -} - -int main(int argc, char *argv[]) { - int i, rc; - MDBX_env *env = nullptr; - MDBX_txn *txn = nullptr; - MDBX_cursor *mc = nullptr; - MDBX_dbi dbi; - char *envname = nullptr; - int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0; - bool append = false; - bool quiet = false; - bool rescue = false; - MDBX_val prevk; - - prog = argv[0]; - if (argc < 2) - usage(); - - while ((i = getopt(argc, argv, "af:ns:NTVrq")) != EOF) { - switch (i) { - case 'V': - printf("mdbx_load version %d.%d.%d.%d\n" - " - source: %s %s, commit %s, tree %s\n" - " - anchor: %s\n" - " - build: %s for %s by %s\n" - " - flags: %s\n" - " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); - return EXIT_SUCCESS; - case 'a': - append = true; - break; - case 'f': - if (freopen(optarg, "r", stdin) == nullptr) { - fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, - mdbx_strerror(errno)); - exit(EXIT_FAILURE); - } - break; - case 'n': - envflags |= MDBX_NOSUBDIR; - break; - case 's': - subname = mdbx_strdup(optarg); - break; - case 'N': - putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA; - break; - case 'T': - mode |= NOHDR | PRINT; - break; - case 'q': - quiet = true; - break; - case 'r': - rescue = true; - break; - default: - usage(); - } - } - - if (optind != argc - 1) - usage(); - -#if defined(_WIN32) || defined(_WIN64) - SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); -#else -#ifdef SIGPIPE - signal(SIGPIPE, signal_handler); -#endif -#ifdef SIGHUP - signal(SIGHUP, signal_handler); -#endif - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); -#endif /* !WINDOWS */ - - envname = argv[optind]; - if (!quiet) - printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname); - fflush(nullptr); - - dbuf.iov_len = 4096; - dbuf.iov_base = mdbx_malloc(dbuf.iov_len); - - /* read first header for mapsize= */ - if (!(mode & NOHDR)) { - rc = readhdr(); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == EOF) - rc = MDBX_ENODATA; - error("readheader", rc); - goto env_close; - } - } - - rc = mdbx_env_create(&env); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_create", rc); - return EXIT_FAILURE; - } - - mdbx_env_set_maxdbs(env, 2); - if (envinfo.mi_maxreaders) { - rc = mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_set_maxreaders", rc); - goto env_close; - } - } - - if (envinfo.mi_mapsize) { - if (envinfo.mi_geo.current) { - rc = mdbx_env_set_geometry( - env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current, - (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink, - (intptr_t)envinfo.mi_geo.grow, - envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); - } else { - if (envinfo.mi_mapsize > MAX_MAPSIZE) { - fprintf( - stderr, - "Database size is too large for current system (mapsize=%" PRIu64 - " is great than system-limit %zu)\n", - envinfo.mi_mapsize, (size_t)MAX_MAPSIZE); - goto env_close; - } - rc = mdbx_env_set_geometry( - env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize, - (intptr_t)envinfo.mi_mapsize, 0, 0, - envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); - } - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_set_geometry", rc); - goto env_close; - } - } - - rc = mdbx_env_open(env, envname, envflags, 0664); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_open", rc); - goto env_close; - } - - kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, MDBX_DUPSORT); - if (kbuf.iov_len >= INTPTR_MAX / 4) { - fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n", - kbuf.iov_len); - goto env_close; - } - kbuf.iov_len = (kbuf.iov_len + 1) * 2; - kbuf.iov_base = malloc(kbuf.iov_len * 2); - k0buf.iov_len = kbuf.iov_len; - k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len; - prevk.iov_base = k0buf.iov_base; - - while (rc == MDBX_SUCCESS) { - if (user_break) { - rc = MDBX_EINTR; - break; - } - - rc = mdbx_txn_begin(env, nullptr, 0, &txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_begin", rc); - goto env_close; - } - - if (mode & GLOBAL) { - mode -= GLOBAL; - if (canary.v | canary.x | canary.y | canary.z) { - rc = mdbx_canary_put(txn, &canary); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_canary_put", rc); - goto txn_abort; - } - } - } - - const char *const dbi_name = subname ? subname : "@MAIN"; - rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, - append ? equal_or_greater : nullptr, - append ? equal_or_greater : nullptr); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_open_ex", rc); - goto txn_abort; - } - - uint64_t present_sequence; - rc = mdbx_dbi_sequence(txn, dbi, &present_sequence, 0); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_sequence", rc); - goto txn_abort; - } - if (present_sequence > sequence) { - fprintf(stderr, - "present sequence for '%s' value (%" PRIu64 - ") is greated than loaded (%" PRIu64 ")\n", - dbi_name, present_sequence, sequence); - rc = MDBX_RESULT_TRUE; - goto txn_abort; - } - if (present_sequence < sequence) { - rc = mdbx_dbi_sequence(txn, dbi, nullptr, sequence - present_sequence); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_sequence", rc); - goto txn_abort; - } - } - - rc = mdbx_cursor_open(txn, dbi, &mc); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); - goto txn_abort; - } - /* if (append) { - mc->mc_flags |= C_SKIPORD; - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; - } */ - - int batch = 0; - prevk.iov_len = 0; - while (rc == MDBX_SUCCESS) { - MDBX_val key; - rc = readline(&key, &kbuf); - if (rc != MDBX_SUCCESS) /* rc == EOF */ - break; - - if (user_break) { - rc = MDBX_EINTR; - break; - } - - MDBX_val data; - rc = readline(&data, &dbuf); - if (rc) { - fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", - prog, lineno); - goto txn_abort; - } - - int appflag = 0; - if (append) { - appflag = MDBX_APPEND; - if (dbi_flags & MDBX_DUPSORT) { - if (prevk.iov_len == key.iov_len && - memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0) - appflag = MDBX_APPEND | MDBX_APPENDDUP; - else - memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len); - } - } - rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag); - if (rc == MDBX_KEYEXIST && putflags) - continue; - if (rc == MDBX_BAD_VALSIZE && rescue) { - fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, - mdbx_strerror(rc)); - continue; - } - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_put", rc); - goto txn_abort; - } - batch++; - - MDBX_txn_info txn_info; - rc = mdbx_txn_info(txn, &txn_info, false); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_info", rc); - goto txn_abort; - } - - if (batch == 10000 || txn_info.txn_space_dirty > MEGABYTE * 16) { - mdbx_cursor_close(mc); - mc = nullptr; - rc = mdbx_txn_commit(txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_commit", rc); - goto env_close; - } - batch = 0; - - rc = mdbx_txn_begin(env, nullptr, 0, &txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_begin", rc); - goto env_close; - } - rc = mdbx_cursor_open(txn, dbi, &mc); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); - goto txn_abort; - } - /* if (append) { - mc->mc_flags |= C_SKIPORD; - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags |= C_SKIPORD; - } */ - } - } - - mdbx_cursor_close(mc); - mc = nullptr; - rc = mdbx_txn_commit(txn); - txn = nullptr; - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_commit", rc); - goto env_close; - } - rc = mdbx_dbi_close(env, dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_close", rc); - goto env_close; - } - - /* try read next header */ - if (!(mode & NOHDR)) - rc = readhdr(); - } - - switch (rc) { - case EOF: - rc = MDBX_SUCCESS; - case MDBX_SUCCESS: - break; - case MDBX_EINTR: - fprintf(stderr, "Interrupted by signal/user\n"); - break; - default: - if (unlikely(rc != MDBX_SUCCESS)) - error("readline", rc); - } - -txn_abort: - mdbx_cursor_close(mc); - mdbx_txn_abort(txn); -env_close: - mdbx_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/libs/libmdbx/src/src/mdbx_stat.c b/libs/libmdbx/src/src/mdbx_stat.c deleted file mode 100644 index df12717aad..0000000000 --- a/libs/libmdbx/src/src/mdbx_stat.c +++ /dev/null @@ -1,476 +0,0 @@ -/* mdbx_stat.c - memory-mapped database status tool */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#ifdef _MSC_VER -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif -#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ -#endif /* _MSC_VER (warnings) */ - -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "internals.h" - -#if defined(_WIN32) || defined(_WIN64) -#include "wingetopt.h" - -static volatile BOOL user_break; -static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { - (void)dwCtrlType; - user_break = true; - return true; -} - -#else /* WINDOWS */ - -static volatile sig_atomic_t user_break; -static void signal_handler(int sig) { - (void)sig; - user_break = 1; -} - -#endif /* !WINDOWS */ - -static void print_stat(MDBX_stat *ms) { - printf(" Pagesize: %u\n", ms->ms_psize); - printf(" Tree depth: %u\n", ms->ms_depth); - printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages); - printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages); - printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages); - printf(" Entries: %" PRIu64 "\n", ms->ms_entries); -} - -static void usage(const char *prog) { - fprintf(stderr, - "usage: %s [-V] [-e] [-f[f[f]]] [-r[r]] [-a|-s name] dbpath\n" - " -V\t\tprint version and exit\n" - " -e\t\tshow whole DB info\n" - " -f\t\tshow GC info\n" - " -r\t\tshow readers\n" - " -a\t\tprint stat of main DB and all subDBs\n" - " \t\t(default) print stat of only the main DB\n" - " -s name\tprint stat of only the named subDB\n", - prog); - exit(EXIT_FAILURE); -} - -static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, uint64_t lag, - size_t bytes_used, size_t bytes_retained) { - (void)ctx; - if (num == 1) - printf("Reader Table\n" - " #\tslot\t%6s %*s %20s %10s %13s %13s\n", - "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", - "retained"); - - printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid, - (int)sizeof(size_t) * 2, (uintptr_t)thread); - if (txnid) - printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, - bytes_used / 1048576.0, bytes_retained / 1048576.0); - else - printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0"); - - return user_break ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; -} - -const char *prog; -static void error(const char *func, int rc) { - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); -} - -int main(int argc, char *argv[]) { - int o, rc; - MDBX_env *env; - MDBX_txn *txn; - MDBX_dbi dbi; - MDBX_stat mst; - MDBX_envinfo mei; - prog = argv[0]; - char *envname; - char *subname = nullptr; - int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; - - if (argc < 2) - usage(prog); - - while ((o = getopt(argc, argv, "Vaefnrs:")) != EOF) { - switch (o) { - case 'V': - printf("mdbx_stat version %d.%d.%d.%d\n" - " - source: %s %s, commit %s, tree %s\n" - " - anchor: %s\n" - " - build: %s for %s by %s\n" - " - flags: %s\n" - " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); - return EXIT_SUCCESS; - case 'a': - if (subname) - usage(prog); - alldbs++; - break; - case 'e': - envinfo++; - break; - case 'f': - freinfo++; - break; - case 'n': - envflags |= MDBX_NOSUBDIR; - break; - case 'r': - rdrinfo++; - break; - case 's': - if (alldbs) - usage(prog); - subname = optarg; - break; - default: - usage(prog); - } - } - - if (optind != argc - 1) - usage(prog); - -#if defined(_WIN32) || defined(_WIN64) - SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); -#else -#ifdef SIGPIPE - signal(SIGPIPE, signal_handler); -#endif -#ifdef SIGHUP - signal(SIGHUP, signal_handler); -#endif - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); -#endif /* !WINDOWS */ - - envname = argv[optind]; - envname = argv[optind]; - printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname); - fflush(nullptr); - - rc = mdbx_env_create(&env); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_create", rc); - return EXIT_FAILURE; - } - - if (alldbs || subname) { - rc = mdbx_env_set_maxdbs(env, 2); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_set_maxdbs", rc); - goto env_close; - } - } - - rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_open", rc); - goto env_close; - } - - rc = mdbx_txn_begin(env, nullptr, MDBX_RDONLY, &txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_begin", rc); - goto txn_abort; - } - - if (envinfo || freinfo) { - rc = mdbx_env_info_ex(env, txn, &mei, sizeof(mei)); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_info_ex", rc); - goto txn_abort; - } - } else { - /* LY: zap warnings from gcc */ - memset(&mei, 0, sizeof(mei)); - } - - if (envinfo) { - rc = mdbx_env_stat_ex(env, txn, &mst, sizeof(mst)); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_stat_ex", rc); - goto txn_abort; - } - printf("Environment Info\n"); - printf(" Pagesize: %u\n", mst.ms_psize); - if (mei.mi_geo.lower != mei.mi_geo.upper) { - printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 - "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 - "/-%" PRIu64 ")\n", - mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, - mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize, - mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize, - mei.mi_geo.shrink / mst.ms_psize); - printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", - mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize); - printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); -#if defined(_WIN32) || defined(_WIN64) - if (mei.mi_geo.shrink && mei.mi_geo.current != mei.mi_geo.upper) - printf(" WARNING: Due Windows system limitations a " - "file couldn't\n be truncated while database " - "is opened. So, the size of\n database file " - "may by large than the database itself,\n " - "until it will be closed or reopened in read-write mode.\n"); -#endif - } else { - printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); - } - printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); - printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n", - mei.mi_latter_reader_txnid, - mei.mi_latter_reader_txnid - mei.mi_recent_txnid); - printf(" Max readers: %u\n", mei.mi_maxreaders); - printf(" Number of reader slots uses: %u\n", mei.mi_numreaders); - } else { - /* LY: zap warnings from gcc */ - memset(&mst, 0, sizeof(mst)); - } - - if (rdrinfo) { - rc = mdbx_reader_list(env, reader_list_func, nullptr); - if (MDBX_IS_ERROR(rc)) { - error("mdbx_reader_list", rc); - goto txn_abort; - } - if (rc == MDBX_RESULT_TRUE) - printf("Reader Table is empty\n"); - else if (rc == MDBX_SUCCESS && rdrinfo > 1) { - int dead; - rc = mdbx_reader_check(env, &dead); - if (MDBX_IS_ERROR(rc)) { - error("mdbx_reader_check", rc); - goto txn_abort; - } - if (rc == MDBX_RESULT_TRUE) { - printf(" %d stale readers cleared.\n", dead); - rc = mdbx_reader_list(env, reader_list_func, nullptr); - if (rc == MDBX_RESULT_TRUE) - printf(" Now Reader Table is empty\n"); - } else - printf(" No stale readers.\n"); - } - if (!(subname || alldbs || freinfo)) - goto txn_abort; - } - - if (freinfo) { - printf("Garbage Collection\n"); - dbi = 0; - MDBX_cursor *cursor; - rc = mdbx_cursor_open(txn, dbi, &cursor); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); - goto txn_abort; - } - rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst)); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_stat", rc); - goto txn_abort; - } - print_stat(&mst); - - pgno_t pages = 0, *iptr; - pgno_t reclaimable = 0; - MDBX_val key, data; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) { - if (user_break) { - rc = MDBX_EINTR; - break; - } - iptr = data.iov_base; - const pgno_t number = *iptr++; - - pages += number; - if (envinfo && mei.mi_latter_reader_txnid > *(txnid_t *)key.iov_base) - reclaimable += number; - - if (freinfo > 1) { - char *bad = ""; - pgno_t prev = - MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1; - pgno_t span = 1; - for (unsigned i = 0; i < number; ++i) { - pgno_t pg = iptr[i]; - if (MDBX_PNL_DISORDERED(prev, pg)) - bad = " [bad sequence]"; - prev = pg; - while (i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) - : pgno_sub(pg, span))) - ++span; - } - printf(" Transaction %" PRIaTXN ", %" PRIaPGNO - " pages, maxspan %" PRIaPGNO "%s\n", - *(txnid_t *)key.iov_base, number, span, bad); - if (freinfo > 2) { - for (unsigned i = 0; i < number; i += span) { - const pgno_t pg = iptr[i]; - for (span = 1; - i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) - : pgno_sub(pg, span)); - ++span) - ; - if (span > 1) - printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span); - else - printf(" %9" PRIaPGNO "\n", pg); - } - } - } - } - mdbx_cursor_close(cursor); - cursor = nullptr; - - switch (rc) { - case MDBX_SUCCESS: - case MDBX_NOTFOUND: - break; - case MDBX_EINTR: - fprintf(stderr, "Interrupted by signal/user\n"); - goto txn_abort; - default: - error("mdbx_cursor_get", rc); - goto txn_abort; - } - - if (envinfo) { - uint64_t value = mei.mi_mapsize / mst.ms_psize; - double percent = value / 100.0; - printf("Page Usage\n"); - printf(" Total: %" PRIu64 " 100%%\n", value); - - value = mei.mi_geo.current / mst.ms_psize; - printf(" Backed: %" PRIu64 " %.1f%%\n", value, value / percent); - - value = mei.mi_last_pgno + 1; - printf(" Allocated: %" PRIu64 " %.1f%%\n", value, value / percent); - - value = mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1); - printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent); - - value = mei.mi_last_pgno + 1 - pages; - printf(" Used: %" PRIu64 " %.1f%%\n", value, value / percent); - - value = pages; - printf(" GC: %" PRIu64 " %.1f%%\n", value, value / percent); - - value = pages - reclaimable; - printf(" Detained: %" PRIu64 " %.1f%%\n", value, value / percent); - - value = reclaimable; - printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent); - - value = - mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1) + reclaimable; - printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent); - } else - printf(" GC: %" PRIaPGNO " pages\n", pages); - } - - rc = mdbx_dbi_open(txn, subname, MDBX_ACCEDE, &dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_open", rc); - goto txn_abort; - } - rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst)); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_stat", rc); - goto txn_abort; - } - printf("Status of %s\n", subname ? subname : "Main DB"); - print_stat(&mst); - - if (alldbs) { - MDBX_cursor *cursor; - rc = mdbx_cursor_open(txn, dbi, &cursor); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); - goto txn_abort; - } - - MDBX_val key; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { - MDBX_dbi subdbi; - if (memchr(key.iov_base, '\0', key.iov_len)) - continue; - subname = mdbx_malloc(key.iov_len + 1); - memcpy(subname, key.iov_base, key.iov_len); - subname[key.iov_len] = '\0'; - rc = mdbx_dbi_open(txn, subname, MDBX_ACCEDE, &subdbi); - if (rc == MDBX_SUCCESS) - printf("Status of %s\n", subname); - mdbx_free(subname); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == MDBX_INCOMPATIBLE) - continue; - error("mdbx_dbi_open", rc); - goto txn_abort; - } - - rc = mdbx_dbi_stat(txn, subdbi, &mst, sizeof(mst)); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_stat", rc); - goto txn_abort; - } - print_stat(&mst); - - rc = mdbx_dbi_close(env, subdbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_close", rc); - goto txn_abort; - } - } - mdbx_cursor_close(cursor); - cursor = nullptr; - } - - switch (rc) { - case MDBX_SUCCESS: - case MDBX_NOTFOUND: - break; - case MDBX_EINTR: - fprintf(stderr, "Interrupted by signal/user\n"); - break; - default: - if (unlikely(rc != MDBX_SUCCESS)) - error("mdbx_cursor_get", rc); - } - - mdbx_dbi_close(env, dbi); -txn_abort: - mdbx_txn_abort(txn); -env_close: - mdbx_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/libs/libmdbx/src/src/miranda.c b/libs/libmdbx/src/src/miranda.c deleted file mode 100644 index df8c2dd620..0000000000 --- a/libs/libmdbx/src/src/miranda.c +++ /dev/null @@ -1,11 +0,0 @@ -#define WIN32_LEAN_AND_MEAN -#include <windows.h> - -#include "../config.h" -#include "../mdbx.h" - -BOOL APIENTRY DllMain(HANDLE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) -{ - mdbx_dll_handler(hModule, ul_reason_for_call, lpReserved); - return TRUE; -} diff --git a/libs/libmdbx/src/src/options.h b/libs/libmdbx/src/src/options.h deleted file mode 100644 index f3805b1eeb..0000000000 --- a/libs/libmdbx/src/src/options.h +++ /dev/null @@ -1,218 +0,0 @@ -/******************************************************************************* - ******************************************************************************* - ******************************************************************************* - * - * - * #### ##### ##### # #### # # #### - * # # # # # # # # ## # # - * # # # # # # # # # # # #### - * # # ##### # # # # # # # # - * # # # # # # # # ## # # - * #### # # # #### # # #### - * - * - */ - -/* Support for huge write-transactions */ -#ifndef MDBX_HUGE_TRANSACTIONS -#define MDBX_HUGE_TRANSACTIONS 0 -#endif /* MDBX_HUGE_TRANSACTIONS */ - -/* using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ -#define MDBX_OSX_WANNA_DURABILITY 0 -/* using fsync() with chance of data lost on power failure */ -#define MDBX_OSX_WANNA_SPEED 1 - -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/* Controls checking PID against reuse DB environment after the fork() */ -#ifndef MDBX_TXN_CHECKPID -#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) -/* PID check could be ommited: - * - on Linux when madvise(MADV_DONTFORK) is available. i.e. after the fork() - * mapped pages will not be available for child process. - * - in Windows where fork() not available. */ -#define MDBX_TXN_CHECKPID 0 -#else -#define MDBX_TXN_CHECKPID 1 -#endif -#define MDBX_TXN_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKPID) -#else -#define MDBX_TXN_CHECKPID_CONFIG STRINGIFY(MDBX_TXN_CHECKPID) -#endif /* MDBX_TXN_CHECKPID */ - -/* Controls checking transaction owner thread against misuse transactions from - * other threads. */ -#ifndef MDBX_TXN_CHECKOWNER -#define MDBX_TXN_CHECKOWNER 1 -#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) -#else -#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) -#endif /* MDBX_TXN_CHECKOWNER */ - -/* Does a system have battery-backed Real-Time Clock or just a fake. */ -#ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) -#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ -#else -#define MDBX_TRUST_RTC 1 -#endif -#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) -#else -#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) -#endif /* MDBX_TRUST_RTC */ - -//------------------------------------------------------------------------------ - -#define MDBX_LOCKING_WIN32FILES -1 /* Win32 File Locking API */ -#define MDBX_LOCKING_SYSV 5 /* SystemV IPC semaphores */ -#define MDBX_LOCKING_POSIX1988 1988 /* POSIX-1 Shared anonymous semaphores */ -#define MDBX_LOCKING_POSIX2001 2001 /* POSIX-2001 Shared Mutexes */ -#define MDBX_LOCKING_POSIX2008 2008 /* POSIX-2008 Robust Mutexes */ -#define MDBX_LOCKING_BENAPHORE 1995 /* BeOS Benaphores, aka Futexes */ - -#if defined(_WIN32) || defined(_WIN64) -#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES -#else -#ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) - -/* Some platforms define the EOWNERDEAD error code even though they - * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) -#define MDBX_LOCKING MDBX_LOCKING_POSIX2008 -#else -#define MDBX_LOCKING MDBX_LOCKING_POSIX2001 -#endif -#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) -#define MDBX_LOCKING MDBX_LOCKING_POSIX1988 -#else -#define MDBX_LOCKING MDBX_LOCKING_SYSV -#endif -#define MDBX_LOCKING_CONFIG "AUTO=" STRINGIFY(MDBX_LOCKING) -#else -#define MDBX_LOCKING_CONFIG STRINGIFY(MDBX_LOCKING) -#endif /* MDBX_LOCKING */ -#endif /* !Windows */ - -#ifndef MDBX_USE_OFDLOCKS -#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ -#define MDBX_USE_OFDLOCKS 1 -#else -#define MDBX_USE_OFDLOCKS 0 -#endif -#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) -#else -#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) -#endif /* MDBX_USE_OFDLOCKS */ - -//------------------------------------------------------------------------------ - -#ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) -#define MDBX_CPU_WRITEBACK_INCOHERENT 0 -#else -#define MDBX_CPU_WRITEBACK_INCOHERENT 1 -#endif -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ - -#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE -#ifdef __OpenBSD__ -#define MDBX_MMAP_INCOHERENT_FILE_WRITE 1 -#else -#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 -#endif -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ - -#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) -/* MIPS has cache coherency issues. */ -#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 -#else -/* LY: assume no relevant mmap/dcache issues. */ -#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 -#endif -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ - -#ifndef MDBX_64BIT_ATOMIC -#if MDBX_WORDBITS >= 64 -#define MDBX_64BIT_ATOMIC 1 -#else -#define MDBX_64BIT_ATOMIC 0 -#endif -#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) -#else -#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) -#endif /* MDBX_64BIT_ATOMIC */ - -#ifndef MDBX_64BIT_CAS -#if defined(ATOMIC_LLONG_LOCK_FREE) -#if ATOMIC_LLONG_LOCK_FREE > 1 -#define MDBX_64BIT_CAS 1 -#else -#define MDBX_64BIT_CAS 0 -#endif -#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) -#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 -#define MDBX_64BIT_CAS 1 -#else -#define MDBX_64BIT_CAS 0 -#endif -#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) -#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 -#define MDBX_64BIT_CAS 1 -#else -#define MDBX_64BIT_CAS 0 -#endif -#elif defined(_MSC_VER) || defined(__APPLE__) -#define MDBX_64BIT_CAS 1 -#else -#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC -#endif -#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) -#else -#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) -#endif /* MDBX_64BIT_CAS */ - -#if !defined(MDBX_UNALIGNED_OK) -#if defined(_MSC_VER) -#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ -#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) -#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ -#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ - !defined(__ALIGNED__) -#define MDBX_UNALIGNED_OK 1 -#else -#define MDBX_UNALIGNED_OK 0 -#endif -#endif /* MDBX_UNALIGNED_OK */ - -#ifndef MDBX_CACHELINE_SIZE -#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) -#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE -#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) -#define MDBX_CACHELINE_SIZE 128 -#else -#define MDBX_CACHELINE_SIZE 64 -#endif -#endif /* MDBX_CACHELINE_SIZE */ - -/******************************************************************************* - ******************************************************************************* - ******************************************************************************/ diff --git a/libs/libmdbx/src/src/osal.c b/libs/libmdbx/src/src/osal.c deleted file mode 100644 index e7c00dd59c..0000000000 --- a/libs/libmdbx/src/src/osal.c +++ /dev/null @@ -1,2247 +0,0 @@ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "internals.h" - -#if defined(_WIN32) || defined(_WIN64) - -#include <winioctl.h> - -static int waitstatus2errcode(DWORD result) { - switch (result) { - case WAIT_OBJECT_0: - return MDBX_SUCCESS; - case WAIT_FAILED: - return GetLastError(); - case WAIT_ABANDONED: - return ERROR_ABANDONED_WAIT_0; - case WAIT_IO_COMPLETION: - return ERROR_USER_APC; - case WAIT_TIMEOUT: - return ERROR_TIMEOUT; - default: - return ERROR_UNHANDLED_ERROR; - } -} - -/* Map a result from an NTAPI call to WIN32 error code. */ -static int ntstatus2errcode(NTSTATUS status) { - DWORD dummy; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - ov.Internal = status; - return GetOverlappedResult(NULL, &ov, &dummy, FALSE) ? MDBX_SUCCESS - : GetLastError(); -} - -/* We use native NT APIs to setup the memory map, so that we can - * let the DB file grow incrementally instead of always preallocating - * the full size. These APIs are defined in <wdm.h> and <ntifs.h> - * but those headers are meant for driver-level development and - * conflict with the regular user-level headers, so we explicitly - * declare them here. Using these APIs also means we must link to - * ntdll.dll, which is not linked by default in user code. */ - -extern NTSTATUS NTAPI NtCreateSection( - OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess, - IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, - IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection, - IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle); - -typedef struct _SECTION_BASIC_INFORMATION { - ULONG Unknown; - ULONG SectionAttributes; - LARGE_INTEGER SectionSize; -} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION; - -extern NTSTATUS NTAPI NtMapViewOfSection( - IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, - IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize, - IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize, - IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType, - IN ULONG Win32Protect); - -extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, - IN OPTIONAL PVOID BaseAddress); - -extern NTSTATUS NTAPI NtClose(HANDLE Handle); - -extern NTSTATUS NTAPI NtAllocateVirtualMemory( - IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits, - IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect); - -extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, - IN PVOID *BaseAddress, - IN OUT PSIZE_T RegionSize, - IN ULONG FreeType); - -#ifndef WOF_CURRENT_VERSION -typedef struct _WOF_EXTERNAL_INFO { - DWORD Version; - DWORD Provider; -} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO; -#endif /* WOF_CURRENT_VERSION */ - -#ifndef WIM_PROVIDER_CURRENT_VERSION -#define WIM_PROVIDER_HASH_SIZE 20 - -typedef struct _WIM_PROVIDER_EXTERNAL_INFO { - DWORD Version; - DWORD Flags; - LARGE_INTEGER DataSourceId; - BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE]; -} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO; -#endif /* WIM_PROVIDER_CURRENT_VERSION */ - -#ifndef FILE_PROVIDER_CURRENT_VERSION -typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 { - ULONG Version; - ULONG Algorithm; - ULONG Flags; -} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1; -#endif /* FILE_PROVIDER_CURRENT_VERSION */ - -#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED -#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL) -#endif -#ifndef STATUS_INVALID_DEVICE_REQUEST -#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L) -#endif - -#ifndef FILE_DEVICE_FILE_SYSTEM -#define FILE_DEVICE_FILE_SYSTEM 0x00000009 -#endif - -#ifndef FSCTL_GET_EXTERNAL_BACKING -#define FSCTL_GET_EXTERNAL_BACKING \ - CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) -#endif - -#endif /* _WIN32 || _WIN64 */ - -/*----------------------------------------------------------------------------*/ - -#if _POSIX_C_SOURCE > 200212 && \ - /* workaround for avoid musl libc wrong prototype */ ( \ - defined(__GLIBC__) || defined(__GNU_LIBRARY__)) -/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */ -__extern_C void __assert_fail(const char *assertion, const char *file, - unsigned line, const char *function) -#ifdef __THROW - __THROW -#else - __nothrow -#endif /* __THROW */ - __noreturn; - -#elif defined(__APPLE__) || defined(__MACH__) -__extern_C void __assert_rtn(const char *function, const char *file, int line, - const char *assertion) /* __nothrow */ -#ifdef __dead2 - __dead2 -#else - __noreturn -#endif /* __dead2 */ -#ifdef __disable_tail_calls - __disable_tail_calls -#endif /* __disable_tail_calls */ - ; - -#define __assert_fail(assertion, file, line, function) \ - __assert_rtn(function, file, line, assertion) -#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) -__extern_C void __assert_c99(const char *assection, const char *file, int line, - const char *function) __noreturn; -#define __assert_fail(assertion, file, line, function) \ - __assert_c99(assertion, file, line, function) -#elif defined(__OpenBSD__) -__extern_C __dead void __assert2(const char *file, int line, - const char *function, - const char *assertion) /* __nothrow */; -#define __assert_fail(assertion, file, line, function) \ - __assert2(file, line, function, assertion) -#elif defined(__NetBSD__) -__extern_C __dead void __assert13(const char *file, int line, - const char *function, - const char *assertion) /* __nothrow */; -#define __assert_fail(assertion, file, line, function) \ - __assert13(file, line, function, assertion) -#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || \ - defined(__DragonFly__) -__extern_C void __assert(const char *function, const char *file, int line, - const char *assertion) /* __nothrow */ -#ifdef __dead2 - __dead2 -#else - __noreturn -#endif /* __dead2 */ -#ifdef __disable_tail_calls - __disable_tail_calls -#endif /* __disable_tail_calls */ - ; -#define __assert_fail(assertion, file, line, function) \ - __assert(function, file, line, assertion) - -#endif /* __assert_fail */ - -#if !defined(__ANDROID_API__) || MDBX_DEBUG - -MDBX_INTERNAL_FUNC void __cold mdbx_assert_fail(const MDBX_env *env, - const char *msg, - const char *func, int line) { -#if MDBX_DEBUG - if (env && env->me_assert_func) { - env->me_assert_func(env, msg, func, line); - return; - } -#else - (void)env; -#endif /* MDBX_DEBUG */ - - if (mdbx_debug_logger) - mdbx_debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg); - else { -#if defined(_WIN32) || defined(_WIN64) - char *message = nullptr; - const int num = mdbx_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", - msg, func ? func : "unknown", line); - if (num < 1 || !message) - message = "<troubles with assertion-message preparation>"; - OutputDebugStringA(message); - if (IsDebuggerPresent()) - DebugBreak(); -#elif defined(__ANDROID_API__) - __android_log_assert(msg, "mdbx", "%s:%u", func, line); -#else - __assert_fail(msg, "mdbx", line, func); -#endif - } - -#if defined(_WIN32) || defined(_WIN64) - FatalExit(ERROR_UNHANDLED_ERROR); -#else - abort(); -#endif -} - -#endif /* __ANDROID_API__ || MDBX_DEBUG */ - -#if !defined(__ANDROID_API__) - -MDBX_INTERNAL_FUNC __cold void mdbx_panic(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - - char *message = nullptr; - const int num = mdbx_vasprintf(&message, fmt, ap); - va_end(ap); - const char *const const_message = - (num < 1 || !message) ? "<troubles with panic-message preparation>" - : message; - -#if defined(_WIN32) || defined(_WIN64) - OutputDebugStringA("\r\nMDBX-PANIC: "); - OutputDebugStringA(const_message); - if (IsDebuggerPresent()) - DebugBreak(); - FatalExit(ERROR_UNHANDLED_ERROR); -#else - __assert_fail(const_message, "mdbx", 0, "panic"); - abort(); -#endif -} - -#endif /* ! __ANDROID_API__ */ - -/*----------------------------------------------------------------------------*/ - -#ifndef mdbx_vasprintf -MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, - va_list ap) { - va_list ones; - va_copy(ones, ap); - int needed = vsnprintf(nullptr, 0, fmt, ap); - - if (unlikely(needed < 0 || needed >= INT_MAX)) { - *strp = nullptr; - va_end(ones); - return needed; - } - - *strp = mdbx_malloc(needed + 1); - if (unlikely(*strp == nullptr)) { - va_end(ones); -#if defined(_WIN32) || defined(_WIN64) - SetLastError(MDBX_ENOMEM); -#else - errno = MDBX_ENOMEM; -#endif - return -1; - } - - int actual = vsnprintf(*strp, needed + 1, fmt, ones); - va_end(ones); - - assert(actual == needed); - if (unlikely(actual < 0)) { - mdbx_free(*strp); - *strp = nullptr; - } - return actual; -} -#endif /* mdbx_vasprintf */ - -#ifndef mdbx_asprintf -MDBX_INTERNAL_FUNC int mdbx_asprintf(char **strp, const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - int rc = mdbx_vasprintf(strp, fmt, ap); - va_end(ap); - return rc; -} -#endif /* mdbx_asprintf */ - -#ifndef mdbx_memalign_alloc -MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, - void **result) { - assert(is_powerof2(alignment) && alignment >= sizeof(void *)); -#if defined(_WIN32) || defined(_WIN64) - (void)alignment; - *result = VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); - return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */; -#elif defined(_ISOC11_SOURCE) - *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment)); - return *result ? MDBX_SUCCESS : errno; -#elif _POSIX_VERSION >= 200112L && \ - (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17) - *result = nullptr; - return posix_memalign(result, alignment, bytes); -#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L - *result = memalign(alignment, bytes); - return *result ? MDBX_SUCCESS : errno; -#else -#error FIXME -#endif -} -#endif /* mdbx_memalign_alloc */ - -#ifndef mdbx_memalign_free -MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr) { -#if defined(_WIN32) || defined(_WIN64) - VirtualFree(ptr, 0, MEM_RELEASE); -#else - mdbx_free(ptr); -#endif -} -#endif /* mdbx_memalign_free */ - -#ifndef mdbx_strdup -char *mdbx_strdup(const char *str) { - if (!str) - return NULL; - size_t bytes = strlen(str) + 1; - char *dup = mdbx_malloc(bytes); - if (dup) - memcpy(dup, str, bytes); - return dup; -} -#endif /* mdbx_strdup */ - -/*----------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair) { - int rc; - memset(condpair, 0, sizeof(mdbx_condpair_t)); -#if defined(_WIN32) || defined(_WIN64) - if ((condpair->mutex = CreateMutexW(NULL, FALSE, NULL)) == NULL) { - rc = GetLastError(); - goto bailout_mutex; - } - if ((condpair->event[0] = CreateEventW(NULL, FALSE, FALSE, NULL)) == NULL) { - rc = GetLastError(); - goto bailout_event; - } - if ((condpair->event[1] = CreateEventW(NULL, FALSE, FALSE, NULL)) != NULL) - return MDBX_SUCCESS; - - rc = GetLastError(); - (void)CloseHandle(condpair->event[0]); -bailout_event: - (void)CloseHandle(condpair->mutex); -#else - rc = pthread_mutex_init(&condpair->mutex, NULL); - if (unlikely(rc != 0)) - goto bailout_mutex; - rc = pthread_cond_init(&condpair->cond[0], NULL); - if (unlikely(rc != 0)) - goto bailout_cond; - rc = pthread_cond_init(&condpair->cond[1], NULL); - if (likely(rc == 0)) - return MDBX_SUCCESS; - - (void)pthread_cond_destroy(&condpair->cond[0]); -bailout_cond: - (void)pthread_mutex_destroy(&condpair->mutex); -#endif -bailout_mutex: - memset(condpair, 0, sizeof(mdbx_condpair_t)); - return rc; -} - -MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair) { -#if defined(_WIN32) || defined(_WIN64) - int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : GetLastError(); - rc = CloseHandle(condpair->event[0]) ? rc : GetLastError(); - rc = CloseHandle(condpair->event[1]) ? rc : GetLastError(); -#else - int err, rc = pthread_mutex_destroy(&condpair->mutex); - rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc; - rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc; -#endif - memset(condpair, 0, sizeof(mdbx_condpair_t)); - return rc; -} - -MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair) { -#if defined(_WIN32) || defined(_WIN64) - DWORD code = WaitForSingleObject(condpair->mutex, INFINITE); - return waitstatus2errcode(code); -#else - return pthread_mutex_lock(&condpair->mutex); -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair) { -#if defined(_WIN32) || defined(_WIN64) - return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : GetLastError(); -#else - return pthread_mutex_unlock(&condpair->mutex); -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, - bool part) { -#if defined(_WIN32) || defined(_WIN64) - return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : GetLastError(); -#else - return pthread_cond_signal(&condpair->cond[part]); -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, - bool part) { -#if defined(_WIN32) || defined(_WIN64) - DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], - INFINITE, FALSE); - if (code == WAIT_OBJECT_0) { - code = WaitForSingleObject(condpair->mutex, INFINITE); - if (code == WAIT_OBJECT_0) - return MDBX_SUCCESS; - } - return waitstatus2errcode(code); -#else - return pthread_cond_wait(&condpair->cond[part], &condpair->mutex); -#endif -} - -/*----------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex) { -#if defined(_WIN32) || defined(_WIN64) - InitializeCriticalSection(fastmutex); - return MDBX_SUCCESS; -#else - return pthread_mutex_init(fastmutex, NULL); -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex) { -#if defined(_WIN32) || defined(_WIN64) - DeleteCriticalSection(fastmutex); - return MDBX_SUCCESS; -#else - return pthread_mutex_destroy(fastmutex); -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex) { -#if defined(_WIN32) || defined(_WIN64) - EnterCriticalSection(fastmutex); - return MDBX_SUCCESS; -#else - return pthread_mutex_lock(fastmutex); -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex) { -#if defined(_WIN32) || defined(_WIN64) - LeaveCriticalSection(fastmutex); - return MDBX_SUCCESS; -#else - return pthread_mutex_unlock(fastmutex); -#endif -} - -/*----------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname) { -#if defined(_WIN32) || defined(_WIN64) - const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX); - if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX) - return ERROR_INVALID_NAME; - wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t)); - if (wlen != mbstowcs(pathnameW, pathname, wlen + 1)) - return ERROR_INVALID_NAME; - return DeleteFileW(pathnameW) ? MDBX_SUCCESS : GetLastError(); -#else - return unlink(pathname) ? errno : MDBX_SUCCESS; -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, - const MDBX_env *env, const char *pathname, - mdbx_filehandle_t *fd, - mode_t unix_mode_bits) { - *fd = INVALID_HANDLE_VALUE; - -#if defined(_WIN32) || defined(_WIN64) - const size_t wlen = mbstowcs(nullptr, pathname, INT_MAX); - if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX) - return ERROR_INVALID_NAME; - wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t)); - if (wlen != mbstowcs(pathnameW, pathname, wlen + 1)) - return ERROR_INVALID_NAME; - - DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING; - DWORD FlagsAndAttributes = - FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; - DWORD DesiredAccess = FILE_READ_ATTRIBUTES; - DWORD ShareMode = (env->me_flags & MDBX_EXCLUSIVE) - ? 0 - : (FILE_SHARE_READ | FILE_SHARE_WRITE); - - switch (purpose) { - default: - return ERROR_INVALID_PARAMETER; - case MDBX_OPEN_LCK: - CreationDisposition = OPEN_ALWAYS; - DesiredAccess |= GENERIC_READ | GENERIC_WRITE; - FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY; - break; - case MDBX_OPEN_DXB_READ: - CreationDisposition = OPEN_EXISTING; - DesiredAccess |= GENERIC_READ; - ShareMode |= FILE_SHARE_READ; - break; - case MDBX_OPEN_DXB_LAZY: - DesiredAccess |= GENERIC_READ | GENERIC_WRITE; - break; - case MDBX_OPEN_DXB_DSYNC: - CreationDisposition = OPEN_EXISTING; - DesiredAccess |= GENERIC_WRITE; - FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH; - break; - case MDBX_OPEN_COPY: - CreationDisposition = CREATE_NEW; - ShareMode = 0; - DesiredAccess |= GENERIC_WRITE; - FlagsAndAttributes |= - (env->me_psize < env->me_os_psize) ? 0 : FILE_FLAG_NO_BUFFERING; - break; - } - - *fd = CreateFileW(pathnameW, DesiredAccess, ShareMode, NULL, - CreationDisposition, FlagsAndAttributes, NULL); - if (*fd == INVALID_HANDLE_VALUE) - return GetLastError(); - - BY_HANDLE_FILE_INFORMATION info; - if (!GetFileInformationByHandle(*fd, &info)) { - int err = GetLastError(); - CloseHandle(*fd); - *fd = INVALID_HANDLE_VALUE; - return err; - } - const DWORD AttributesDiff = - (info.dwFileAttributes ^ FlagsAndAttributes) & - (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | - FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED); - if (AttributesDiff) - (void)SetFileAttributesW(pathnameW, info.dwFileAttributes ^ AttributesDiff); - -#else - int flags = unix_mode_bits ? O_CREAT : 0; - switch (purpose) { - default: - return EINVAL; - case MDBX_OPEN_LCK: - flags |= O_RDWR; - break; - case MDBX_OPEN_DXB_READ: - flags = O_RDONLY; - break; - case MDBX_OPEN_DXB_LAZY: - flags |= O_RDWR; - break; - case MDBX_OPEN_COPY: - flags = O_CREAT | O_WRONLY | O_EXCL; - break; - case MDBX_OPEN_DXB_DSYNC: - flags |= O_WRONLY; -#if defined(O_DSYNC) - flags |= O_DSYNC; -#elif defined(O_SYNC) - flags |= O_SYNC; -#elif defined(O_FSYNC) - flags |= O_FSYNC; -#endif - break; - } - - const bool direct_nocache_for_copy = - env->me_psize >= env->me_os_psize && purpose == MDBX_OPEN_COPY; - if (direct_nocache_for_copy) { -#if defined(O_DIRECT) - flags |= O_DIRECT; -#endif /* O_DIRECT */ -#if defined(O_NOCACHE) - flags |= O_NOCACHE; -#endif /* O_NOCACHE */ - } - -#ifdef O_CLOEXEC - flags |= O_CLOEXEC; -#endif /* O_CLOEXEC */ - - *fd = open(pathname, flags, unix_mode_bits); -#if defined(O_DIRECT) - if (*fd < 0 && (flags & O_DIRECT) && - (errno == EINVAL || errno == EAFNOSUPPORT)) { - flags &= ~(O_DIRECT | O_EXCL); - *fd = open(pathname, flags, unix_mode_bits); - } -#endif /* O_DIRECT */ - if (*fd < 0) - return errno; - -#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC) - const int fd_flags = fcntl(*fd, F_GETFD); - if (fd_flags != -1) - (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC); -#endif /* FD_CLOEXEC && !O_CLOEXEC */ - - if (direct_nocache_for_copy) { -#if defined(F_NOCACHE) && !defined(O_NOCACHE) - (void)fcntl(*fd, F_NOCACHE, 1); -#endif /* F_NOCACHE */ - } - -#endif - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd) { -#if defined(_WIN32) || defined(_WIN64) - return CloseHandle(fd) ? MDBX_SUCCESS : GetLastError(); -#else - return (close(fd) == 0) ? MDBX_SUCCESS : errno; -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, - uint64_t offset) { - if (bytes > MAX_WRITE) - return MDBX_EINVAL; -#if defined(_WIN32) || defined(_WIN64) - OVERLAPPED ov; - ov.hEvent = 0; - ov.Offset = (DWORD)offset; - ov.OffsetHigh = HIGH_DWORD(offset); - - DWORD read = 0; - if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) { - int rc = GetLastError(); - return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc; - } -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - intptr_t read = pread(fd, buf, bytes, offset); - if (read < 0) { - int rc = errno; - return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc; - } -#endif - return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA; -} - -MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t bytes, uint64_t offset) { - while (true) { -#if defined(_WIN32) || defined(_WIN64) - OVERLAPPED ov; - ov.hEvent = 0; - ov.Offset = (DWORD)offset; - ov.OffsetHigh = HIGH_DWORD(offset); - - DWORD written; - if (unlikely(!WriteFile( - fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, - &written, &ov))) - return GetLastError(); - if (likely(bytes == written)) - return MDBX_SUCCESS; -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - const intptr_t written = - pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset); - if (likely(bytes == (size_t)written)) - return MDBX_SUCCESS; - if (written < 0) { - const int rc = errno; - if (rc != EINTR) - return rc; - continue; - } -#endif - bytes -= written; - offset += written; - buf = (char *)buf + written; - } -} - -MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, - size_t bytes) { - while (true) { -#if defined(_WIN32) || defined(_WIN64) - DWORD written; - if (unlikely(!WriteFile( - fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, - &written, nullptr))) - return GetLastError(); - if (likely(bytes == written)) - return MDBX_SUCCESS; -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - const intptr_t written = - write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE); - if (likely(bytes == (size_t)written)) - return MDBX_SUCCESS; - if (written < 0) { - const int rc = errno; - if (rc != EINTR) - return rc; - continue; - } -#endif - bytes -= written; - buf = (char *)buf + written; - } -} - -int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, int iovcnt, - uint64_t offset, size_t expected_written) { -#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ < 24) - size_t written = 0; - for (int i = 0; i < iovcnt; ++i) { - int rc = mdbx_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - written += iov[i].iov_len; - offset += iov[i].iov_len; - } - return (expected_written == written) ? MDBX_SUCCESS - : MDBX_EIO /* ERROR_WRITE_FAULT */; -#else - int rc; - intptr_t written; - do { - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - written = pwritev(fd, iov, iovcnt, offset); - if (likely(expected_written == (size_t)written)) - return MDBX_SUCCESS; - rc = errno; - } while (rc == EINTR); - return (written < 0) ? rc : MDBX_EIO /* Use which error code? */; -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_filesync(mdbx_filehandle_t fd, - enum mdbx_syncmode_bits mode_bits) { -#if defined(_WIN32) || defined(_WIN64) - return ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) == 0 || - FlushFileBuffers(fd)) - ? MDBX_SUCCESS - : GetLastError(); -#else - -#if defined(__APPLE__) && \ - MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY - if (mode_bits & MDBX_SYNC_IODQ) - return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno; -#endif /* MacOS */ -#if defined(__linux__) || defined(__gnu_linux__) - if (mode_bits == MDBX_SYNC_SIZE && mdbx_linux_kernel_version >= 0x03060000) - return MDBX_SUCCESS; -#endif /* Linux */ - int rc; - do { -#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0 - /* LY: This code is always safe and without appreciable performance - * degradation, even on a kernel with fdatasync's bug. - * - * For more info about of a corresponding fdatasync() bug - * see http://www.spinics.net/lists/linux-ext4/msg33714.html */ - if ((mode_bits & MDBX_SYNC_SIZE) == 0) { - if (fdatasync(fd) == 0) - return MDBX_SUCCESS; - } else -#else - (void)mode_bits; -#endif - if (fsync(fd) == 0) - return MDBX_SUCCESS; - rc = errno; - } while (rc == EINTR); - return rc; -#endif -} - -int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length) { -#if defined(_WIN32) || defined(_WIN64) - BY_HANDLE_FILE_INFORMATION info; - if (!GetFileInformationByHandle(fd, &info)) - return GetLastError(); - *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; -#else - struct stat st; - - STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - if (fstat(fd, &st)) - return errno; - - *length = st.st_size; -#endif - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd) { -#if defined(_WIN32) || defined(_WIN64) - switch (GetFileType(fd)) { - case FILE_TYPE_DISK: - return MDBX_RESULT_FALSE; - case FILE_TYPE_CHAR: - case FILE_TYPE_PIPE: - return MDBX_RESULT_TRUE; - default: - return GetLastError(); - } -#else - struct stat info; - if (fstat(fd, &info)) - return errno; - switch (info.st_mode & S_IFMT) { - case S_IFBLK: - case S_IFREG: - return MDBX_RESULT_FALSE; - case S_IFCHR: - case S_IFIFO: - case S_IFSOCK: - return MDBX_RESULT_TRUE; - case S_IFDIR: - case S_IFLNK: - default: - return MDBX_INCOMPATIBLE; - } -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length) { -#if defined(_WIN32) || defined(_WIN64) - if (mdbx_SetFileInformationByHandle) { - FILE_END_OF_FILE_INFO EndOfFileInfo; - EndOfFileInfo.EndOfFile.QuadPart = length; - return mdbx_SetFileInformationByHandle(fd, FileEndOfFileInfo, - &EndOfFileInfo, - sizeof(FILE_END_OF_FILE_INFO)) - ? MDBX_SUCCESS - : GetLastError(); - } else { - LARGE_INTEGER li; - li.QuadPart = length; - return (SetFilePointerEx(fd, li, NULL, FILE_BEGIN) && SetEndOfFile(fd)) - ? MDBX_SUCCESS - : GetLastError(); - } -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno; -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos) { -#if defined(_WIN32) || defined(_WIN64) - LARGE_INTEGER li; - li.QuadPart = pos; - return SetFilePointerEx(fd, li, NULL, FILE_BEGIN) ? MDBX_SUCCESS - : GetLastError(); -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS; -#endif -} - -/*----------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC int -mdbx_thread_create(mdbx_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg) { -#if defined(_WIN32) || defined(_WIN64) - *thread = CreateThread(NULL, 0, start_routine, arg, 0, NULL); - return *thread ? MDBX_SUCCESS : GetLastError(); -#else - return pthread_create(thread, NULL, start_routine, arg); -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread) { -#if defined(_WIN32) || defined(_WIN64) - DWORD code = WaitForSingleObject(thread, INFINITE); - return waitstatus2errcode(code); -#else - void *unused_retval = &unused_retval; - return pthread_join(thread, &unused_retval); -#endif -} - -/*----------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, - size_t length, int async) { - uint8_t *ptr = (uint8_t *)map->address + offset; -#if defined(_WIN32) || defined(_WIN64) - if (FlushViewOfFile(ptr, length) && (async || FlushFileBuffers(map->fd))) - return MDBX_SUCCESS; - return GetLastError(); -#else -#if defined(__linux__) || defined(__gnu_linux__) - if (async && mdbx_linux_kernel_version > 0x02061300) - /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op, - since the kernel properly tracks dirty pages and flushes them to storage - as necessary. */ - return MDBX_SUCCESS; -#endif /* Linux */ - const int mode = async ? MS_ASYNC : MS_SYNC; - int rc = (msync(ptr, length, mode) == 0) ? MDBX_SUCCESS : errno; - return rc; -#endif -} - -MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, - const char *pathname, int err) { -#if defined(_WIN32) || defined(_WIN64) - (void)pathname; - (void)err; - if (!mdbx_GetVolumeInformationByHandleW) - return MDBX_ENOSYS; - DWORD unused, flags; - if (!mdbx_GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, - &flags, nullptr, 0)) - return GetLastError(); - if ((flags & FILE_READ_ONLY_VOLUME) == 0) - return MDBX_EACCESS; -#else - struct statvfs info; - if (err != MDBX_ENOFILE) { - if (statvfs(pathname, &info)) - return errno; - if ((info.f_flag & ST_RDONLY) == 0) - return err; - } - if (fstatvfs(handle, &info)) - return errno; - if ((info.f_flag & ST_RDONLY) == 0) - return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err; -#endif /* !Windows */ - return MDBX_SUCCESS; -} - -static int mdbx_check_fs_local(mdbx_filehandle_t handle, int flags) { - if (flags & MDBX_EXCLUSIVE) - return 0; - -#if defined(_WIN32) || defined(_WIN64) - if (mdbx_RunningUnderWine() && !(flags & MDBX_EXCLUSIVE)) - return ERROR_NOT_CAPABLE /* workaround for Wine */; - - if (GetFileType(handle) != FILE_TYPE_DISK) - return ERROR_FILE_OFFLINE; - - if (mdbx_GetFileInformationByHandleEx) { - FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo; - if (mdbx_GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, - &RemoteProtocolInfo, - sizeof(RemoteProtocolInfo))) { - if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && - !(flags & MDBX_RDONLY)) - return ERROR_FILE_OFFLINE; - if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && - !(flags & MDBX_EXCLUSIVE)) - return ERROR_REMOTE_STORAGE_MEDIA_ERROR; - } - } - - if (mdbx_NtFsControlFile) { - NTSTATUS rc; - struct { - WOF_EXTERNAL_INFO wof_info; - union { - WIM_PROVIDER_EXTERNAL_INFO wim_info; - FILE_PROVIDER_EXTERNAL_INFO_V1 file_info; - }; - size_t reserved_for_microsoft_madness[42]; - } GetExternalBacking_OutputBuffer; - IO_STATUS_BLOCK StatusBlock; - rc = mdbx_NtFsControlFile(handle, NULL, NULL, NULL, &StatusBlock, - FSCTL_GET_EXTERNAL_BACKING, NULL, 0, - &GetExternalBacking_OutputBuffer, - sizeof(GetExternalBacking_OutputBuffer)); - if (NT_SUCCESS(rc)) { - if (!(flags & MDBX_EXCLUSIVE)) - return ERROR_REMOTE_STORAGE_MEDIA_ERROR; - } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && - rc != STATUS_INVALID_DEVICE_REQUEST) - return ntstatus2errcode(rc); - } - - if (mdbx_GetVolumeInformationByHandleW && mdbx_GetFinalPathNameByHandleW) { - WCHAR *PathBuffer = mdbx_malloc(sizeof(WCHAR) * INT16_MAX); - if (!PathBuffer) - return MDBX_ENOMEM; - - int rc = MDBX_SUCCESS; - DWORD VolumeSerialNumber, FileSystemFlags; - if (!mdbx_GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, - &VolumeSerialNumber, NULL, - &FileSystemFlags, NULL, 0)) { - rc = GetLastError(); - goto bailout; - } - - if ((flags & MDBX_RDONLY) == 0) { - if (FileSystemFlags & - (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | - FILE_VOLUME_IS_COMPRESSED)) { - rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; - goto bailout; - } - } - - if (!mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, - FILE_NAME_NORMALIZED | - VOLUME_NAME_NT)) { - rc = GetLastError(); - goto bailout; - } - - if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) { - if (!(flags & MDBX_EXCLUSIVE)) { - rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; - goto bailout; - } - } else if (mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, - FILE_NAME_NORMALIZED | - VOLUME_NAME_DOS)) { - UINT DriveType = GetDriveTypeW(PathBuffer); - if (DriveType == DRIVE_NO_ROOT_DIR && - _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 && - _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) { - PathBuffer[7] = 0; - DriveType = GetDriveTypeW(PathBuffer + 4); - } - switch (DriveType) { - case DRIVE_CDROM: - if (flags & MDBX_RDONLY) - break; - // fall through - case DRIVE_UNKNOWN: - case DRIVE_NO_ROOT_DIR: - case DRIVE_REMOTE: - default: - if (!(flags & MDBX_EXCLUSIVE)) - rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; - // fall through - case DRIVE_REMOVABLE: - case DRIVE_FIXED: - case DRIVE_RAMDISK: - break; - } - } - bailout: - mdbx_free(PathBuffer); - return rc; - } - -#else - - struct statvfs statvfs_info; - if (fstatvfs(handle, &statvfs_info)) - return errno; -#if defined(ST_LOCAL) || defined(ST_EXPORTED) - const unsigned long st_flags = statvfs_info.f_flag; -#endif /* ST_LOCAL || ST_EXPORTED */ - -#if defined(__NetBSD__) - const unsigned type = 0; - const char *const name = statvfs_info.f_fstypename; - const size_t name_len = VFS_NAMELEN; -#elif defined(_AIX) || defined(__OS400__) - const char *const name = statvfs_info.f_basetype; - const size_t name_len = sizeof(statvfs_info.f_basetype); - struct stat st; - if (fstat(handle, &st)) - return errno; - const unsigned type = st.st_vfstype; - if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; -#elif defined(FSTYPSZ) || defined(_FSTYPSZ) - const unsigned type = 0; - const char *const name = statvfs_info.f_basetype; - const size_t name_len = sizeof(statvfs_info.f_basetype); -#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ) - const unsigned type = 0; - struct stat st; - if (fstat(handle, &st)) - return errno; - const char *const name = st.st_fstype; - const size_t name_len = strlen(name); -#else - struct statfs statfs_info; - if (fstatfs(handle, &statfs_info)) - return errno; -#if defined(__OpenBSD__) - const unsigned type = 0; -#else - const unsigned type = statfs_info.f_type; -#endif -#if defined(MNT_LOCAL) || defined(MNT_EXPORTED) - const unsigned long mnt_flags = statfs_info.f_flags; -#endif /* MNT_LOCAL || MNT_EXPORTED */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ - defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) - const char *const name = statfs_info.f_fstypename; - const size_t name_len = sizeof(statfs_info.f_fstypename); -#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21 - const char *const name = ""; - const unsigned name_len = 0; -#else - - const char *name = ""; - unsigned name_len = 0; - - struct stat st; - if (fstat(handle, &st)) - return errno; - - char pathbuf[PATH_MAX]; - FILE *mounted = nullptr; -#if defined(__linux__) || defined(__gnu_linux__) - mounted = setmntent("/proc/mounts", "r"); -#endif /* Linux */ - if (!mounted) - mounted = setmntent("/etc/mtab", "r"); - if (mounted) { - const struct mntent *ent; -#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \ - (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19)) - struct mntent entbuf; - const bool should_copy = false; - while (nullptr != - (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf)))) -#else - const bool should_copy = true; - while (nullptr != (ent = getmntent(mounted))) -#endif - { - struct stat mnt; - if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) { - if (should_copy) { - name = - strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1); - pathbuf[name_len] = 0; - } else { - name = ent->mnt_fsname; - name_len = strlen(name); - } - break; - } - } - endmntent(mounted); - } -#endif /* !xBSD && !Android/Bionic */ -#endif - - if (name_len) { - if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || - strncasecmp("cifs", name, name_len) == 0 || - strncasecmp("ncpfs", name, name_len) == 0 || - strncasecmp("smbfs", name, name_len) == 0 || - ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && - strncasecmp("fuseblk", name, name_len) != 0)) && - !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; - if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || - strcasecmp("sshfs", name) == 0) - return MDBX_EREMOTE; - } - -#ifdef ST_LOCAL - if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; -#elif defined(MNT_LOCAL) - if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; -#endif /* ST/MNT_LOCAL */ - -#ifdef ST_EXPORTED - if ((st_flags & ST_EXPORTED) != 0 && !(flags & MDBX_RDONLY)) - return MDBX_EREMOTE; -#elif defined(MNT_EXPORTED) - if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & MDBX_RDONLY)) - return MDBX_EREMOTE; -#endif /* ST/MNT_EXPORTED */ - - switch (type) { - case 0xFF534D42 /* CIFS_MAGIC_NUMBER */: - case 0x6969 /* NFS_SUPER_MAGIC */: - case 0x564c /* NCP_SUPER_MAGIC */: - case 0x517B /* SMB_SUPER_MAGIC */: -#if defined(__digital__) || defined(__osf__) || defined(__osf) - case 0x0E /* Tru64 NFS */: -#endif -#ifdef ST_FST_NFS - case ST_FST_NFS: -#endif - if ((flags & MDBX_EXCLUSIVE) == 0) - return MDBX_EREMOTE; - case 0: - default: - break; - } -#endif /* Unix */ - - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, - const size_t size, const size_t limit, - const unsigned options) { - assert(size <= limit); - map->limit = 0; - map->current = 0; - map->address = nullptr; -#if defined(_WIN32) || defined(_WIN64) - map->section = NULL; - map->filesize = 0; -#endif /* Windows */ - - int err = mdbx_check_fs_local(map->fd, flags); - if (unlikely(err != MDBX_SUCCESS)) - return err; - - if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) { - err = mdbx_ftruncate(map->fd, size); - if (err != MDBX_SUCCESS) - return err; -#if defined(_WIN32) || defined(_WIN64) - map->filesize = size; -#else - map->current = size; -#endif - } else { - uint64_t filesize = 0; - err = mdbx_filesize(map->fd, &filesize); - if (err != MDBX_SUCCESS) - return err; -#if defined(_WIN32) || defined(_WIN64) - map->filesize = filesize; -#else - map->current = (filesize > limit) ? limit : (size_t)filesize; -#endif - } - -#if defined(_WIN32) || defined(_WIN64) - LARGE_INTEGER SectionSize; - SectionSize.QuadPart = size; - err = NtCreateSection( - &map->section, - /* DesiredAccess */ - (flags & MDBX_WRITEMAP) - ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | - SECTION_MAP_WRITE - : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, - /* ObjectAttributes */ NULL, /* MaximumSize (InitialSize) */ &SectionSize, - /* SectionPageProtection */ - (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, - /* AllocationAttributes */ SEC_RESERVE, map->fd); - if (!NT_SUCCESS(err)) - return ntstatus2errcode(err); - - SIZE_T ViewSize = - (flags & MDBX_RDONLY) ? 0 : mdbx_RunningUnderWine() ? size : limit; - err = NtMapViewOfSection( - map->section, GetCurrentProcess(), &map->address, - /* ZeroBits */ 0, - /* CommitSize */ 0, - /* SectionOffset */ NULL, &ViewSize, - /* InheritDisposition */ ViewUnmap, - /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, - /* Win32Protect */ - (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); - if (!NT_SUCCESS(err)) { - NtClose(map->section); - map->section = 0; - map->address = nullptr; - return ntstatus2errcode(err); - } - assert(map->address != MAP_FAILED); - - map->current = (size_t)SectionSize.QuadPart; - map->limit = ViewSize; - -#else - -#ifndef MAP_TRYFIXED -#define MAP_TRYFIXED 0 -#endif - -#ifndef MAP_HASSEMAPHORE -#define MAP_HASSEMAPHORE 0 -#endif - -#ifndef MAP_CONCEAL -#define MAP_CONCEAL 0 -#endif - -#ifndef MAP_NOSYNC -#define MAP_NOSYNC 0 -#endif - - map->address = mmap( - NULL, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, - MAP_SHARED | MAP_FILE | - (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) | - ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC - : MAP_CONCEAL), - map->fd, 0); - - if (unlikely(map->address == MAP_FAILED)) { - map->limit = 0; - map->current = 0; - map->address = nullptr; - return errno; - } - map->limit = limit; - -#ifdef MADV_DONTFORK - if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0)) - return errno; -#endif -#ifdef MADV_NOHUGEPAGE - (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE); -#endif - -#endif - - VALGRIND_MAKE_MEM_DEFINED(map->address, map->current); - ASAN_UNPOISON_MEMORY_REGION(map->address, map->current); - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map) { - VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current); - /* Unpoisoning is required for ASAN to avoid false-positive diagnostic - * when this memory will re-used by malloc or another mmaping. - * See https://github.com/erthink/libmdbx/pull/93#issuecomment-613687203 */ - ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit); -#if defined(_WIN32) || defined(_WIN64) - if (map->section) - NtClose(map->section); - NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->address); - if (!NT_SUCCESS(rc)) - ntstatus2errcode(rc); -#else - if (unlikely(munmap(map->address, map->limit))) - return errno; -#endif - - map->limit = 0; - map->current = 0; - map->address = nullptr; - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t size, - size_t limit, const bool may_move) { - assert(size <= limit); -#if defined(_WIN32) || defined(_WIN64) - assert(size != map->current || limit != map->limit || size < map->filesize); - - NTSTATUS status; - LARGE_INTEGER SectionSize; - int err, rc = MDBX_SUCCESS; - - if (!(flags & MDBX_RDONLY) && limit == map->limit && size > map->current) { - /* growth rw-section */ - if (!mdbx_NtExtendSection) - return MDBX_UNABLE_EXTEND_MAPSIZE /* workaround for Wine */; - SectionSize.QuadPart = size; - status = mdbx_NtExtendSection(map->section, &SectionSize); - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); - map->current = size; - if (map->filesize < size) - map->filesize = size; - return MDBX_SUCCESS; - } - - if (limit > map->limit) { - /* check ability of address space for growth before unmap */ - PVOID BaseAddress = (PBYTE)map->address + map->limit; - SIZE_T RegionSize = limit - map->limit; - status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, - &RegionSize, MEM_RESERVE, PAGE_NOACCESS); - if (status == /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) - return MDBX_UNABLE_EXTEND_MAPSIZE; - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); - - status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, - MEM_RELEASE); - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); - } - - /* Windows unable: - * - shrink a mapped file; - * - change size of mapped view; - * - extend read-only mapping; - * Therefore we should unmap/map entire section. */ - status = NtUnmapViewOfSection(GetCurrentProcess(), map->address); - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); - status = NtClose(map->section); - map->section = NULL; - PVOID ReservedAddress = NULL; - SIZE_T ReservedSize = limit; - - if (!NT_SUCCESS(status)) { - bailout_ntstatus: - err = ntstatus2errcode(status); - bailout: - map->address = NULL; - map->current = map->limit = 0; - if (ReservedAddress) { - ReservedSize = 0; - status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, - &ReservedSize, MEM_RELEASE); - assert(NT_SUCCESS(status)); - (void)status; - } - return err; - } - - /* resizing of the file may take a while, - * therefore we reserve address space to avoid occupy it by other threads */ - ReservedAddress = map->address; - status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, - &ReservedSize, MEM_RESERVE, PAGE_NOACCESS); - if (!NT_SUCCESS(status)) { - ReservedAddress = NULL; - if (status != /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) - goto bailout_ntstatus /* no way to recovery */; - - if (may_move) - /* the base address could be changed */ - map->address = NULL; - } - -retry_file_and_section: - err = mdbx_filesize(map->fd, &map->filesize); - if (err != MDBX_SUCCESS) - goto bailout; - - if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) { - err = mdbx_ftruncate(map->fd, size); - if (err == MDBX_SUCCESS) - map->filesize = size; - /* ignore error, because Windows unable shrink file - * that already mapped (by another process) */ - } - - SectionSize.QuadPart = size; - status = NtCreateSection( - &map->section, - /* DesiredAccess */ - (flags & MDBX_WRITEMAP) - ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | - SECTION_MAP_WRITE - : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, - /* ObjectAttributes */ NULL, - /* MaximumSize (InitialSize) */ &SectionSize, - /* SectionPageProtection */ - (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, - /* AllocationAttributes */ SEC_RESERVE, map->fd); - - if (!NT_SUCCESS(status)) - goto bailout_ntstatus; - - if (ReservedAddress) { - /* release reserved address space */ - ReservedSize = 0; - status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, - &ReservedSize, MEM_RELEASE); - ReservedAddress = NULL; - if (!NT_SUCCESS(status)) - goto bailout_ntstatus; - } - -retry_mapview:; - SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit; - status = NtMapViewOfSection( - map->section, GetCurrentProcess(), &map->address, - /* ZeroBits */ 0, - /* CommitSize */ 0, - /* SectionOffset */ NULL, &ViewSize, - /* InheritDisposition */ ViewUnmap, - /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, - /* Win32Protect */ - (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); - - if (!NT_SUCCESS(status)) { - if (status == /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && - map->address && may_move) { - /* try remap at another base address */ - map->address = NULL; - goto retry_mapview; - } - NtClose(map->section); - map->section = NULL; - - if (map->address && (size != map->current || limit != map->limit)) { - /* try remap with previously size and limit, - * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */ - rc = MDBX_UNABLE_EXTEND_MAPSIZE; - size = map->current; - limit = map->limit; - goto retry_file_and_section; - } - - /* no way to recovery */ - goto bailout_ntstatus; - } - assert(map->address != MAP_FAILED); - - map->current = (size_t)SectionSize.QuadPart; - map->limit = ViewSize; - -#else - - uint64_t filesize = 0; - int rc = mdbx_filesize(map->fd, &filesize); - if (rc != MDBX_SUCCESS) - return rc; - - if (flags & MDBX_RDONLY) { - map->current = (filesize > limit) ? limit : (size_t)filesize; - if (map->current != size) - rc = MDBX_UNABLE_EXTEND_MAPSIZE; - } else if (filesize != size) { - rc = mdbx_ftruncate(map->fd, size); - if (rc != MDBX_SUCCESS) - return rc; - map->current = size; - } - - if (limit != map->limit) { -#if defined(MREMAP_MAYMOVE) - void *ptr = - mremap(map->address, map->limit, limit, may_move ? MREMAP_MAYMOVE : 0); - if (ptr == MAP_FAILED) { - rc = errno; - switch (rc) { - case EAGAIN: - case ENOMEM: - case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */: - rc = MDBX_UNABLE_EXTEND_MAPSIZE; - } - return rc; - } -#else - if (!may_move) - /* TODO: Perhaps here it is worth to implement suspend/resume threads - * and perform unmap/map as like for Windows. */ - return MDBX_UNABLE_EXTEND_MAPSIZE; - - if (unlikely(munmap(map->address, map->limit))) - return errno; - - unsigned mmap_flags = - MAP_CONCEAL | MAP_SHARED | MAP_FILE | - (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0); -#ifdef MAP_FIXED - if (!may_move) - mmap_flags |= MAP_FIXED; -#endif - - void *ptr = - mmap(map->address, limit, - (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, - mmap_flags, map->fd, 0); - if (unlikely(ptr == MAP_FAILED)) { - ptr = mmap(map->address, map->limit, - (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, - mmap_flags, map->fd, 0); - if (unlikely(ptr == MAP_FAILED)) { - VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current); - /* Unpoisoning is required for ASAN to avoid false-positive diagnostic - * when this memory will re-used by malloc or another mmaping. - * See https://github.com/erthink/libmdbx/pull/93#issuecomment-613687203 - */ - ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit); - map->limit = 0; - map->current = 0; - map->address = nullptr; - return errno; - } - return MDBX_UNABLE_EXTEND_MAPSIZE; - } -#endif /* !MREMAP_MAYMOVE */ - - if (map->address != ptr) { - VALGRIND_MAKE_MEM_NOACCESS(map->address, map->current); - /* Unpoisoning is required for ASAN to avoid false-positive diagnostic - * when this memory will re-used by malloc or another mmaping. - * See https://github.com/erthink/libmdbx/pull/93#issuecomment-613687203 - */ - ASAN_UNPOISON_MEMORY_REGION(map->address, map->limit); - - VALGRIND_MAKE_MEM_DEFINED(ptr, map->current); - ASAN_UNPOISON_MEMORY_REGION(ptr, map->current); - map->address = ptr; - } - map->limit = limit; - -#ifdef MADV_DONTFORK - if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0)) - return errno; -#endif /* MADV_DONTFORK */ - -#ifdef MADV_NOHUGEPAGE - (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE); -#endif /* MADV_NOHUGEPAGE */ - } -#endif - - return rc; -} - -/*----------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC __cold void mdbx_osal_jitter(bool tiny) { - for (;;) { -#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \ - defined(__x86_64__) - const unsigned salt = 277u * (unsigned)__rdtsc(); -#else - const unsigned salt = rand(); -#endif - - const unsigned coin = salt % (tiny ? 29u : 43u); - if (coin < 43 / 3) - break; -#if defined(_WIN32) || defined(_WIN64) - SwitchToThread(); - if (coin > 43 * 2 / 3) - Sleep(1); -#else - sched_yield(); - if (coin > 43 * 2 / 3) - usleep(coin); -#endif - } -} - -#if defined(_WIN32) || defined(_WIN64) -#elif defined(__APPLE__) || defined(__MACH__) -#include <mach/mach_time.h> -#elif defined(__linux__) || defined(__gnu_linux__) -static __cold clockid_t choice_monoclock(void) { - struct timespec probe; -#if defined(CLOCK_BOOTTIME) - if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0) - return CLOCK_BOOTTIME; -#elif defined(CLOCK_MONOTONIC_RAW) - if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0) - return CLOCK_MONOTONIC_RAW; -#elif defined(CLOCK_MONOTONIC_COARSE) - if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0) - return CLOCK_MONOTONIC_COARSE; -#endif - return CLOCK_MONOTONIC; -} -#endif - -/*----------------------------------------------------------------------------*/ - -#if defined(_WIN32) || defined(_WIN64) -static LARGE_INTEGER performance_frequency; -#elif defined(__APPLE__) || defined(__MACH__) -static uint64_t ratio_16dot16_to_monotine; -#endif - -MDBX_INTERNAL_FUNC uint64_t -mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16) { -#if defined(_WIN32) || defined(_WIN64) - if (unlikely(performance_frequency.QuadPart == 0)) - QueryPerformanceFrequency(&performance_frequency); - const uint64_t ratio = performance_frequency.QuadPart; -#elif defined(__APPLE__) || defined(__MACH__) - if (unlikely(ratio_16dot16_to_monotine == 0)) { - mach_timebase_info_data_t ti; - mach_timebase_info(&ti); - ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer; - } - const uint64_t ratio = ratio_16dot16_to_monotine; -#else - const uint64_t ratio = UINT64_C(1000000000); -#endif - return (ratio * seconds_16dot16 + 32768) >> 16; -} - -MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime) { - static uint64_t limit; - if (unlikely(monotime > limit)) { - if (limit != 0) - return UINT32_MAX; - limit = mdbx_osal_16dot16_to_monotime(UINT32_MAX - 1); - if (monotime > limit) - return UINT32_MAX; - } -#if defined(_WIN32) || defined(_WIN64) - return (uint32_t)((monotime << 16) / performance_frequency.QuadPart); -#elif defined(__APPLE__) || defined(__MACH__) - return (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine); -#else - return (uint32_t)(monotime * 128 / 1953125); -#endif -} - -MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void) { -#if defined(_WIN32) || defined(_WIN64) - LARGE_INTEGER counter; - counter.QuadPart = 0; - QueryPerformanceCounter(&counter); - return counter.QuadPart; -#elif defined(__APPLE__) || defined(__MACH__) - return mach_absolute_time(); -#else - -#if defined(__linux__) || defined(__gnu_linux__) - static clockid_t posix_clockid = -1; - if (unlikely(posix_clockid < 0)) - posix_clockid = choice_monoclock(); -#elif defined(CLOCK_MONOTONIC) -#define posix_clockid CLOCK_MONOTONIC -#else -#define posix_clockid CLOCK_REALTIME -#endif - - struct timespec ts; - if (unlikely(clock_gettime(posix_clockid, &ts) != 0)) { - ts.tv_nsec = 0; - ts.tv_sec = 0; - } - return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec; -#endif -} - -/*----------------------------------------------------------------------------*/ - -static void bootid_shake(bin128_t *p) { - /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */ - const uint32_t e = p->a - (p->b << 23 | p->b >> 9); - p->a = p->b ^ (p->c << 16 | p->c >> 16); - p->b = p->c + (p->d << 11 | p->d >> 21); - p->c = p->d + e; - p->d = e + p->a; -} - -static void bootid_collect(bin128_t *p, const void *s, size_t n) { - p->y += UINT64_C(64526882297375213); - bootid_shake(p); - for (size_t i = 0; i < n; ++i) { - bootid_shake(p); - p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i]; - bootid_shake(p); - p->y += 14621231; - } - bootid_shake(p); - - /* minor non-linear tomfoolery */ - const unsigned z = p->x % 61; - p->y = p->y << z | p->y >> (64 - z); - bootid_shake(p); - bootid_shake(p); - const unsigned q = p->x % 59; - p->y = p->y << q | p->y >> (64 - q); - bootid_shake(p); - bootid_shake(p); - bootid_shake(p); -} - -#if defined(_WIN32) || defined(_WIN64) - -static uint64_t windows_systemtime_ms() { - FILETIME ft; - GetSystemTimeAsFileTime(&ft); - return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul; -} - -static uint64_t windows_bootime(void) { - unsigned confirmed = 0; - uint64_t boottime = 0; - uint64_t up0 = mdbx_GetTickCount64(); - uint64_t st0 = windows_systemtime_ms(); - for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) { - YieldProcessor(); - const uint64_t up1 = mdbx_GetTickCount64(); - const uint64_t st1 = windows_systemtime_ms(); - if (st1 > fuse && st1 == st0 && up1 == up0) { - uint64_t diff = st1 - up1; - if (boottime == diff) { - if (++confirmed > 4) - return boottime; - } else { - confirmed = 0; - boottime = diff; - } - fuse = st1; - Sleep(1); - } - st0 = st1; - up0 = up1; - } - return 0; -} - -typedef LSTATUS (APIENTRY *pfnRegGetValueA)(HKEY, LPCSTR, LPCSTR, DWORD, LPDWORD, PVOID, LPDWORD); -static pfnRegGetValueA fnRegGetValueA = nullptr; - -static LSTATUS APIENTRY stubRegGetValueA(HKEY hkey, LPCSTR lpSubKey, LPCSTR lpValue, DWORD dwFlags, LPDWORD pdwType, PVOID pvData, LPDWORD pcbData) -{ - HKEY tmp; - LSTATUS rc = RegOpenKeyA(hkey, lpSubKey, &tmp); - if (rc != 0) - return rc; - - DWORD dwType = (dwFlags == RRF_RT_ANY) ? REG_SZ : REG_DWORD; - rc = RegQueryValueExA(tmp, lpValue, 0, &dwType, pvData, pcbData); - if (rc != 0 && dwFlags == RRF_RT_DWORD) { - rc = 0; - *(DWORD *)pvData = 0xBABAEBA; - *pcbData = sizeof(DWORD); - } - RegCloseKey(tmp); - return rc; -} - -static LSTATUS mdbx_RegGetValue(HKEY hkey, LPCSTR lpSubKey, LPCSTR lpValue, - DWORD dwFlags, LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData) { - if (fnRegGetValueA == nullptr) { - fnRegGetValueA = (pfnRegGetValueA)GetProcAddress(GetModuleHandleA("advapi32.dll"), "RegGetValueA"); - if (fnRegGetValueA == nullptr) - fnRegGetValueA = stubRegGetValueA; - } - - LSTATUS rc = - fnRegGetValueA(hkey, lpSubKey, lpValue, dwFlags, pdwType, pvData, pcbData); - if (rc != ERROR_FILE_NOT_FOUND) - return rc; - - rc = fnRegGetValueA(hkey, lpSubKey, lpValue, - dwFlags | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, pdwType, - pvData, pcbData); - if (rc != ERROR_FILE_NOT_FOUND) - return rc; - return fnRegGetValueA(hkey, lpSubKey, lpValue, - dwFlags | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, pdwType, - pvData, pcbData); -} -#endif - -static __cold __maybe_unused bool bootid_parse_uuid(bin128_t *s, const void *p, - const size_t n) { - if (n > 31) { - unsigned bits = 0; - for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ { - uint8_t c = ((const uint8_t *)p)[i]; - if (c >= '0' && c <= '9') - c -= '0'; - else if (c >= 'a' && c <= 'f') - c -= 'a' - 10; - else if (c >= 'A' && c <= 'F') - c -= 'A' - 10; - else - continue; - assert(c <= 15); - c ^= s->y >> 60; - s->y = s->y << 4 | s->x >> 60; - s->x = s->x << 4 | c; - bits += 4; - } - if (bits > 42 * 3) - /* UUID parsed successfully */ - return true; - } - - if (n > 15) /* is enough handle it as a binary? */ { - if (n == sizeof(bin128_t)) { - bin128_t aligned; - memcpy(&aligned, p, sizeof(bin128_t)); - s->x += aligned.x; - s->y += aligned.y; - } else - bootid_collect(s, p, n); - return true; - } - - if (n) - bootid_collect(s, p, n); - return false; -} - -__cold MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void) { - bin128_t bin = {{0, 0}}; - bool got_machineid = false, got_boottime = false, got_bootseq = false; - -#if defined(__linux__) || defined(__gnu_linux__) - { - const int fd = - open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_NOFOLLOW); - if (fd != -1) { - struct statfs fs; - char buf[42]; - const ssize_t len = - (fstatfs(fd, &fs) == 0 && fs.f_type == /* procfs */ 0x9FA0) - ? read(fd, buf, sizeof(buf)) - : -1; - close(fd); - if (len > 0 && bootid_parse_uuid(&bin, buf, len)) - return bin; - } - } -#endif /* Linux */ - -#if defined(__APPLE__) || defined(__MACH__) - { - char buf[42]; - size_t len = sizeof(buf); - if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && - bootid_parse_uuid(&bin, buf, len)) - return bin; - -#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \ - __MAC_OS_X_VERSION_MIN_REQUIRED > 1050 - uuid_t uuid; - struct timespec wait = {0, 1000000000u / 42}; - if (!gethostuuid(uuid, &wait) && - bootid_parse_uuid(&bin, uuid, sizeof(uuid))) - got_machineid = true; -#endif /* > 10.5 */ - - struct timeval boottime; - len = sizeof(boottime); - if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && - len == sizeof(boottime) && boottime.tv_sec) - got_boottime = true; - } -#endif /* Apple/Darwin */ - -#if defined(_WIN32) || defined(_WIN64) - { - union buf { - DWORD BootId; - DWORD BaseTime; - SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo; - struct { - LARGE_INTEGER BootTime; - LARGE_INTEGER CurrentTime; - LARGE_INTEGER TimeZoneBias; - ULONG TimeZoneId; - ULONG Reserved; - ULONGLONG BootTimeBias; - ULONGLONG SleepTimeBias; - } SysTimeOfDayInfoHacked; - wchar_t MachineGuid[42]; - char DigitalProductId[248]; - } buf; - - static const char HKLM_MicrosoftCryptography[] = - "SOFTWARE\\Microsoft\\Cryptography"; - DWORD len = sizeof(buf); - /* Windows is madness and must die */ - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, - "MachineGuid", RRF_RT_ANY, NULL, &buf.MachineGuid, - &len) == ERROR_SUCCESS && - len > 42 && len < sizeof(buf)) - got_machineid = bootid_parse_uuid(&bin, &buf.MachineGuid, len); - - if (!got_machineid) { - /* again, Windows is madness */ - static const char HKLM_WindowsNT[] = - "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; - static const char HKLM_WindowsNT_DPK[] = - "SOFTWARE\\Microsoft\\Windows " - "NT\\CurrentVersion\\DefaultProductKey"; - static const char HKLM_WindowsNT_DPK2[] = - "SOFTWARE\\Microsoft\\Windows " - "NT\\CurrentVersion\\DefaultProductKey2"; - - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, - "DigitalProductId", RRF_RT_ANY, NULL, - &buf.DigitalProductId, &len) == ERROR_SUCCESS && - len > 42 && len < sizeof(buf)) { - bootid_collect(&bin, &buf.DigitalProductId, len); - got_machineid = true; - } - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, - "DigitalProductId", RRF_RT_ANY, NULL, - &buf.DigitalProductId, &len) == ERROR_SUCCESS && - len > 42 && len < sizeof(buf)) { - bootid_collect(&bin, &buf.DigitalProductId, len); - got_machineid = true; - } - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, - "DigitalProductId", RRF_RT_ANY, NULL, - &buf.DigitalProductId, &len) == ERROR_SUCCESS && - len > 42 && len < sizeof(buf)) { - bootid_collect(&bin, &buf.DigitalProductId, len); - got_machineid = true; - } - } - - static const char HKLM_PrefetcherParams[] = - "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory " - "Management\\PrefetchParameters"; - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", - RRF_RT_DWORD, NULL, &buf.BootId, - &len) == ERROR_SUCCESS && - len > 1 && len < sizeof(buf)) { - bootid_collect(&bin, &buf.BootId, len); - got_bootseq = true; - } - - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", - RRF_RT_DWORD, NULL, &buf.BaseTime, - &len) == ERROR_SUCCESS && - len >= sizeof(buf.BaseTime) && buf.BaseTime) { - bootid_collect(&bin, &buf.BaseTime, len); - got_boottime = true; - } - - /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */ - NTSTATUS status = NtQuerySystemInformation( - 0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo, - sizeof(buf.SysTimeOfDayInfo), &len); - if (NT_SUCCESS(status) && - len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTime) + - sizeof(buf.SysTimeOfDayInfoHacked.BootTime) && - buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) { - bootid_collect(&bin, &buf.SysTimeOfDayInfoHacked.BootTime, - sizeof(buf.SysTimeOfDayInfoHacked.BootTime)); - got_boottime = true; - } - - if (!got_boottime) { - uint64_t boottime = windows_bootime(); - if (boottime) { - bootid_collect(&bin, &boottime, sizeof(boottime)); - got_boottime = true; - } - } - } -#endif /* Windows */ - -#if defined(CTL_HW) && defined(HW_UUID) - if (!got_machineid) { - static const int mib[] = {CTL_HW, HW_UUID}; - char buf[42]; - size_t len = sizeof(buf); - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0) - got_machineid = bootid_parse_uuid(&bin, buf, len); - } -#endif /* CTL_HW && HW_UUID */ - -#if defined(CTL_KERN) && defined(KERN_HOSTUUID) - if (!got_machineid) { - static const int mib[] = {CTL_KERN, KERN_HOSTUUID}; - char buf[42]; - size_t len = sizeof(buf); - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0) - got_machineid = bootid_parse_uuid(&bin, buf, len); - } -#endif /* CTL_KERN && KERN_HOSTUUID */ - -#if defined(__NetBSD__) - if (!got_machineid) { - char buf[42]; - size_t len = sizeof(buf); - if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, NULL, 0) == 0) - got_machineid = bootid_parse_uuid(&bin, buf, len); - } -#endif /* __NetBSD__ */ - -#if _XOPEN_SOURCE_EXTENDED - if (!got_machineid) { - const int hostid = gethostid(); - if (hostid > 0) { - bootid_collect(&bin, &hostid, sizeof(hostid)); - got_machineid = true; - } - } -#endif /* _XOPEN_SOURCE_EXTENDED */ - - if (!got_machineid) { - lack: - bin.x = bin.y = 0; - return bin; - } - - /*--------------------------------------------------------------------------*/ - -#if defined(CTL_KERN) && defined(KERN_BOOTTIME) - if (!got_boottime) { - static const int mib[] = {CTL_KERN, KERN_BOOTTIME}; - struct timeval boottime; - size_t len = sizeof(boottime); - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &boottime, &len, NULL, 0) == 0 && - len == sizeof(boottime) && boottime.tv_sec) { - bootid_collect(&bin, &boottime, len); - got_boottime = true; - } - } -#endif /* CTL_KERN && KERN_BOOTTIME */ - -#if defined(__sun) || defined(__SVR4) || defined(__svr4__) - if (!got_boottime) { - kstat_ctl_t *kc = kstat_open(); - if (kc) { - kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc"); - if (kp && kstat_read(kc, kp, 0) != -1) { - kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time"); - if (kn) { - switch (kn->data_type) { - case KSTAT_DATA_INT32: - case KSTAT_DATA_UINT32: - bootid_collect(&bin, &kn->value, sizeof(int32_t)); - got_boottime = true; - case KSTAT_DATA_INT64: - case KSTAT_DATA_UINT64: - bootid_collect(&bin, &kn->value, sizeof(int64_t)); - got_boottime = true; - } - } - } - kstat_close(kc); - } - } -#endif /* SunOS / Solaris */ - -#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME) - if (!got_boottime) { - setutxent(); - const struct utmpx id = {.ut_type = BOOT_TIME}; - const struct utmpx *entry = getutxid(&id); - if (entry) { - bootid_collect(&bin, entry, sizeof(*entry)); - got_boottime = true; - while (unlikely((entry = getutxid(&id)) != nullptr)) { - /* have multiple reboot records, assuming we can distinguish next - * bootsession even if RTC is wrong or absent */ - bootid_collect(&bin, entry, sizeof(*entry)); - got_bootseq = true; - } - } - endutxent(); - } -#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */ - - if (!got_bootseq) { - if (!got_boottime || !MDBX_TRUST_RTC) - goto lack; - -#if defined(_WIN32) || defined(_WIN64) - FILETIME now; - GetSystemTimeAsFileTime(&now); - if (0x1CCCCCC > now.dwHighDateTime) -#else - struct timespec mono, real; - if (clock_gettime(CLOCK_MONOTONIC, &mono) || - clock_gettime(CLOCK_REALTIME, &real) || - /* wrong time, RTC is mad or absent */ - 1555555555l > real.tv_sec || - /* seems no adjustment by RTC/NTP, i.e. a fake time */ - real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || - (real.tv_sec - mono.tv_sec) % 900u == 0) -#endif - goto lack; - } - - return bin; -} diff --git a/libs/libmdbx/src/src/osal.h b/libs/libmdbx/src/src/osal.h deleted file mode 100644 index ff1c4af1bf..0000000000 --- a/libs/libmdbx/src/src/osal.h +++ /dev/null @@ -1,945 +0,0 @@ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -/*----------------------------------------------------------------------------*/ -/* Microsoft compiler generates a lot of warning for self includes... */ - -#ifdef _MSC_VER -#pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ - expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ - * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ - * not guaranteed. Specify /EHsc */ -#endif /* _MSC_VER (warnings) */ - -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(MDBX_TOOLS) && MDBX_AVOID_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - -#ifdef __cplusplus -extern "C" { -#endif - -/*----------------------------------------------------------------------------*/ -/* C99 includes */ -#include <inttypes.h> -#include <stddef.h> -#include <stdint.h> -#include <stdlib.h> - -#include <assert.h> -#include <fcntl.h> -#include <limits.h> -#include <stdio.h> -#include <string.h> -#include <time.h> - -/* C11 stdalign.h */ -#if __has_include(<stdalign.h>) -#include <stdalign.h> -#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -#define alignas(N) _Alignas(N) -#elif defined(_MSC_VER) -#define alignas(N) __declspec(align(N)) -#elif __has_attribute(__aligned__) || defined(__GNUC__) -#define alignas(N) __attribute__((__aligned__(N))) -#else -#error "FIXME: Required _alignas() or equivalent." -#endif - -/*----------------------------------------------------------------------------*/ -/* Systems includes */ - -#ifdef __APPLE__ -#include <TargetConditionals.h> -#endif /* Apple OSX & iOS */ - -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) -#include <sys/cdefs.h> -#include <sys/mount.h> -#include <sys/sysctl.h> -#include <sys/types.h> -#if defined(__FreeBSD__) || defined(__DragonFly__) -#include <vm/vm_param.h> -#elif defined(__OpenBSD__) || defined(__NetBSD__) -#include <uvm/uvm_param.h> -#else -#define SYSCTL_LEGACY_NONCONST_MIB -#endif -#ifndef __MACH__ -#include <sys/vmmeter.h> -#endif -#else -#include <malloc.h> -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) -#include <mntent.h> -#endif /* !Solaris */ -#endif /* !xBSD */ - -#if defined(__FreeBSD__) || __has_include(<malloc_np.h>) -#include <malloc_np.h> -#endif - -#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>) -#include <malloc/malloc.h> -#endif /* MacOS */ - -#if defined(__MACH__) -#include <mach/host_info.h> -#include <mach/mach_host.h> -#include <mach/mach_port.h> -#include <uuid/uuid.h> -#undef P_DIRTY -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -#include <linux/sysctl.h> -#include <sys/sendfile.h> -#include <sys/statfs.h> -#endif /* Linux */ - -#ifndef _XOPEN_SOURCE -#define _XOPEN_SOURCE 0 -#endif - -#ifndef _XOPEN_SOURCE_EXTENDED -#define _XOPEN_SOURCE_EXTENDED 0 -#else -#include <utmpx.h> -#endif /* _XOPEN_SOURCE_EXTENDED */ - -#if defined(__sun) || defined(__SVR4) || defined(__svr4__) -#include <kstat.h> -#include <sys/mnttab.h> -/* On Solaris, it's easier to add a missing prototype rather than find a - * combination of #defines that break nothing. */ -__extern_C key_t ftok(const char *, int); -#endif /* SunOS/Solaris */ - -#if defined(_WIN32) || defined(_WIN64) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include <tlhelp32.h> -#include <windows.h> -#include <winnt.h> -#include <winternl.h> -#define HAVE_SYS_STAT_H -#define HAVE_SYS_TYPES_H -typedef HANDLE mdbx_thread_t; -typedef unsigned mdbx_thread_key_t; -#define MDBX_OSAL_SECTION HANDLE -#define MAP_FAILED NULL -#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) -#define THREAD_CALL WINAPI -#define THREAD_RESULT DWORD -typedef struct { - HANDLE mutex; - HANDLE event[2]; -} mdbx_condpair_t; -typedef CRITICAL_SECTION mdbx_fastmutex_t; - -#if MDBX_AVOID_CRT -#ifndef mdbx_malloc -static inline void *mdbx_malloc(size_t bytes) { - return LocalAlloc(LMEM_FIXED, bytes); -} -#endif /* mdbx_malloc */ - -#ifndef mdbx_calloc -static inline void *mdbx_calloc(size_t nelem, size_t size) { - return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); -} -#endif /* mdbx_calloc */ - -#ifndef mdbx_realloc -static inline void *mdbx_realloc(void *ptr, size_t bytes) { - return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); -} -#endif /* mdbx_realloc */ - -#ifndef mdbx_free -#define mdbx_free LocalFree -#endif /* mdbx_free */ -#else -#define mdbx_malloc malloc -#define mdbx_calloc calloc -#define mdbx_realloc realloc -#define mdbx_free free -#define mdbx_strdup _strdup -#endif /* MDBX_AVOID_CRT */ - -#ifndef snprintf -#define snprintf _snprintf /* ntdll */ -#endif - -#ifndef vsnprintf -#define vsnprintf _vsnprintf /* ntdll */ -#endif - -#else /*----------------------------------------------------------------------*/ - -#include <pthread.h> -#include <semaphore.h> -#include <signal.h> -#include <sys/file.h> -#include <sys/ipc.h> -#include <sys/mman.h> -#include <sys/param.h> -#include <sys/stat.h> -#include <sys/statvfs.h> -#include <sys/uio.h> -#include <unistd.h> -typedef pthread_t mdbx_thread_t; -typedef pthread_key_t mdbx_thread_key_t; -#define INVALID_HANDLE_VALUE (-1) -#define THREAD_CALL -#define THREAD_RESULT void * -typedef struct { - pthread_mutex_t mutex; - pthread_cond_t cond[2]; -} mdbx_condpair_t; -typedef pthread_mutex_t mdbx_fastmutex_t; -#define mdbx_malloc malloc -#define mdbx_calloc calloc -#define mdbx_realloc realloc -#define mdbx_free free -#define mdbx_strdup strdup -#endif /* Platform */ - -#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ -#elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) -#elif defined(_MSC_VER) && !MDBX_AVOID_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ - -#ifdef __ANDROID_API__ -#include <android/log.h> -#if __ANDROID_API__ >= 21 -#include <sys/sendfile.h> -#endif -#endif /* Android */ - -/* *INDENT-OFF* */ -/* clang-format off */ -#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>) -#include <sys/stat.h> -#endif -#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>) -#include <sys/types.h> -#endif -#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>) -#include <sys/file.h> -#endif -/* *INDENT-ON* */ -/* clang-format on */ - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if !defined(MADV_DODUMP) && defined(MADV_CORE) -#define MADV_DODUMP MADV_CORE -#endif /* MADV_CORE -> MADV_DODUMP */ - -#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) -#define MADV_DONTDUMP MADV_NOCORE -#endif /* MADV_NOCORE -> MADV_DONTDUMP */ - -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || \ - defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) -#ifndef __ia32__ -/* LY: define neutral __ia32__ for x86 and x86-64 archs */ -#define __ia32__ 1 -#endif /* __ia32__ */ -#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64) || defined(_M_X64)) -/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ -#define __amd64__ 1 -#endif /* __amd64__ */ -#endif /* all x86 */ - -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - -/*----------------------------------------------------------------------------*/ -/* Compiler's includes for builtins/intrinsics */ - -#if defined(_MSC_VER) || defined(__INTEL_COMPILER) -#include <intrin.h> -#elif __GNUC_PREREQ(4, 4) || defined(__clang__) -#if defined(__ia32__) || defined(__e2k__) -#include <x86intrin.h> -#endif /* __ia32__ */ -#if defined(__ia32__) -#include <cpuid.h> -#endif /* __ia32__ */ -#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) -#include <mbarrier.h> -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) -#include <machine/sys/inline.h> -#elif defined(__IBMC__) && defined(__powerpc) -#include <atomic.h> -#elif defined(_AIX) -#include <builtins.h> -#include <sys/atomic_op.h> -#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) -#include <c_asm.h> -#include <machine/builtins.h> -#elif defined(__MWERKS__) -/* CodeWarrior - troubles ? */ -#pragma gcc_extensions -#elif defined(__SNC__) -/* Sony PS3 - troubles ? */ -#elif defined(__hppa__) || defined(__hppa) -#include <machine/inline.h> -#else -#error Unsupported C compiler, please use GNU C 4.4 or newer -#endif /* Compiler */ - -/*----------------------------------------------------------------------------*/ -/* Byteorder */ - -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) - -/* *INDENT-OFF* */ -/* clang-format off */ -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || \ - defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) -#include <endian.h> -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) -#include <machine/endian.h> -#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) -#include <sys/isa_defs.h> -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ - (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) -#include <sys/endian.h> -#include <sys/types.h> -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || \ - defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) -#include <sys/param.h> -#endif /* OS */ -/* *INDENT-ON* */ -/* clang-format on */ - -#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) -#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN -#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN -#define __BYTE_ORDER__ __BYTE_ORDER -#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) -#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN -#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN -#define __BYTE_ORDER__ _BYTE_ORDER -#else -#define __ORDER_LITTLE_ENDIAN__ 1234 -#define __ORDER_BIG_ENDIAN__ 4321 - -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) -#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ - -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) -#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ - -#else -#error __BYTE_ORDER__ should be defined. -#endif /* Arch */ - -#endif -#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ - -#if __has_include(<sys/cachectl.h>) -#include <sys/cachectl.h> -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) -/* MIPS should have explicit cache control */ -#include <sys/cachectl.h> -#endif - -static __maybe_unused __inline void mdbx_compiler_barrier(void) { -#if defined(__clang__) || defined(__GNUC__) - __asm__ __volatile__("" ::: "memory"); -#elif defined(_MSC_VER) - _ReadWriteBarrier(); -#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ - __memory_barrier(); - if (type > MDBX_BARRIER_COMPILER) -#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) - __mf(); -#elif defined(__i386__) || defined(__x86_64__) - _mm_mfence(); -#else -#error "Unknown target for Intel Compiler, please report to us." -#endif -#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) - __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) - _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) - __fence(); -#else -#error "Could not guess the kind of compiler, please report to us." -#endif -} - -static __maybe_unused __inline void mdbx_memory_barrier(void) { -#if __has_extension(c_atomic) || __has_extension(cxx_atomic) - __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); -#elif defined(__ATOMIC_SEQ_CST) - __atomic_thread_fence(__ATOMIC_SEQ_CST); -#elif defined(__clang__) || defined(__GNUC__) - __sync_synchronize(); -#elif defined(_MSC_VER) - MemoryBarrier(); -#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ -#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) - __mf(); -#elif defined(__i386__) || defined(__x86_64__) - _mm_mfence(); -#else -#error "Unknown target for Intel Compiler, please report to us." -#endif -#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) - __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) - _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) - __lwsync(); -#else -#error "Could not guess the kind of compiler, please report to us." -#endif -} - -/*----------------------------------------------------------------------------*/ -/* libc compatibility stuff */ - -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) -#define mdbx_asprintf asprintf -#define mdbx_vasprintf vasprintf -#else -MDBX_INTERNAL_FUNC __printf_args(2, 3) int __maybe_unused - mdbx_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); -#endif - -/*----------------------------------------------------------------------------*/ -/* OS abstraction layer stuff */ - -/* max bytes to write in one call */ -#if defined(_WIN32) || defined(_WIN64) -#define MAX_WRITE UINT32_C(0x01000000) -#else -#define MAX_WRITE UINT32_C(0x3fff0000) -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; -MDBX_INTERNAL_VAR bool - mdbx_RunningOnWSL /* Windows Subsystem for Linux is mad and trouble-full */; -#endif /* Linux */ - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -static __maybe_unused __inline size_t mdbx_syspagesize(void) { -#if defined(_WIN32) || defined(_WIN64) - SYSTEM_INFO si; - GetSystemInfo(&si); - return si.dwPageSize; -#else - return sysconf(_SC_PAGE_SIZE); -#endif -} - -#ifndef mdbx_strdup -LIBMDBX_API char *mdbx_strdup(const char *str); -#endif - -static __maybe_unused __inline int mdbx_get_errno(void) { -#if defined(_WIN32) || defined(_WIN64) - DWORD rc = GetLastError(); -#else - int rc = errno; -#endif - return rc; -} - -#ifndef mdbx_memalign_alloc -MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, - void **result); -#endif -#ifndef mdbx_memalign_free -MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int mdbx_condpair_init(mdbx_condpair_t *condpair); -MDBX_INTERNAL_FUNC int mdbx_condpair_lock(mdbx_condpair_t *condpair); -MDBX_INTERNAL_FUNC int mdbx_condpair_unlock(mdbx_condpair_t *condpair); -MDBX_INTERNAL_FUNC int mdbx_condpair_signal(mdbx_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int mdbx_condpair_wait(mdbx_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int mdbx_condpair_destroy(mdbx_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - int iovcnt, uint64_t offset, - size_t expected_written); -MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -mdbx_thread_create(mdbx_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); - -enum mdbx_syncmode_bits { - MDBX_SYNC_DATA = 1, - MDBX_SYNC_SIZE = 2, - MDBX_SYNC_IODQ = 4 -}; - -MDBX_INTERNAL_FUNC int mdbx_filesync(mdbx_filehandle_t fd, - const enum mdbx_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); - -enum mdbx_openfile_purpose { - MDBX_OPEN_DXB_READ = 0, - MDBX_OPEN_DXB_LAZY = 1, - MDBX_OPEN_DXB_DSYNC = 2, - MDBX_OPEN_LCK = 3, - MDBX_OPEN_COPY = 4 -}; - -MDBX_INTERNAL_FUNC int mdbx_openfile(const enum mdbx_openfile_purpose purpose, - const MDBX_env *env, const char *pathname, - mdbx_filehandle_t *fd, - mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); -MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); - -typedef struct mdbx_mmap_param { - union { - void *address; - uint8_t *dxb; - struct MDBX_lockinfo *lck; - }; - mdbx_filehandle_t fd; - size_t limit; /* mapping length, but NOT a size of file nor DB */ - size_t current; /* mapped region size, i.e. the size of file and DB */ -#if defined(_WIN32) || defined(_WIN64) - uint64_t filesize /* in-process cache of a file size. */; -#endif -#ifdef MDBX_OSAL_SECTION - MDBX_OSAL_SECTION section; -#endif -} mdbx_mmap_t; - -#define MMAP_OPTION_TRUNCATE 1 -#define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, - const size_t must, const size_t limit, - const unsigned options); -MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); -MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, - size_t wanna, const bool may_move); -#if defined(_WIN32) || defined(_WIN64) -typedef struct { - unsigned limit, count; - HANDLE handles[31]; -} mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); -#endif /* Windows */ -MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, - size_t length, int async); -MDBX_INTERNAL_FUNC int mdbx_check_fs_rdonly(mdbx_filehandle_t handle, - const char *pathname, int err); - -static __maybe_unused __inline uint32_t mdbx_getpid(void) { - STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); -#if defined(_WIN32) || defined(_WIN64) - return GetCurrentProcessId(); -#else - return getpid(); -#endif -} - -static __maybe_unused __inline uintptr_t mdbx_thread_self(void) { - mdbx_tid_t thunk; - STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); -#if defined(_WIN32) || defined(_WIN64) - thunk = GetCurrentThreadId(); -#else - thunk = pthread_self(); -#endif - return (uintptr_t)thunk; -} - -MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); -MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t -mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); - -typedef union bin128 { - __anonymous_struct_extension__ struct { uint64_t x, y; }; - __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; -} bin128_t; - -MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); -/*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of mdbx_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by agrument. The reson for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (expecially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of mdbx_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of mdbx_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); - -#if defined(_WIN32) || defined(_WIN64) - -typedef union MDBX_srwlock { - struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} MDBX_srwlock; - -typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); -MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, - mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, - mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -#if 0 /* LY: unused for now */ -#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 -typedef enum OFFER_PRIORITY { - VmOfferPriorityVeryLow = 1, - VmOfferPriorityLow, - VmOfferPriorityBelowNormal, - VmOfferPriorityNormal -} OFFER_PRIORITY; -#endif /* Windows 8.1 */ - -typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, - SIZE_T Size); -MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; - -typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, - SIZE_T Size); -MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; - -typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( - PVOID VirtualAddress, - SIZE_T Size, - OFFER_PRIORITY Priority -); -MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; -#endif /* unused for now */ - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -#endif /* Windows */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include <stdatomic.h> -#elif defined(__GNUC__) || defined(__clang__) -/* LY: nothing required */ -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include <libkern/OSAtomic.h> -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef __cplusplus -} -#endif - -#ifdef _MSC_VER -#pragma warning(pop) -#endif diff --git a/libs/libmdbx/src/src/version.c.in b/libs/libmdbx/src/src/version.c.in deleted file mode 100644 index 2854bd5d9d..0000000000 --- a/libs/libmdbx/src/src/version.c.in +++ /dev/null @@ -1,46 +0,0 @@ -/* This is CMake-template for libmdbx's version.c - ******************************************************************************/ - -#include "internals.h" - -#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || \ - MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR} -#error "API version mismatch! Had `git fetch --tags` done?" -#endif - -static const char sourcery[] = STRINGIFY(MDBX_BUILD_SOURCERY); - -__dll_export -#ifdef __attribute_used__ - __attribute_used__ -#elif defined(__GNUC__) || __has_attribute(__used__) - __attribute__((__used__)) -#endif -#ifdef __attribute_externally_visible__ - __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) - __attribute__((__externally_visible__)) -#endif - const mdbx_version_info mdbx_version = { - ${MDBX_VERSION_MAJOR}, - ${MDBX_VERSION_MINOR}, - ${MDBX_VERSION_RELEASE}, - ${MDBX_VERSION_REVISION}, - {"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@", - "@MDBX_GIT_DESCRIBE@"}, - sourcery}; - -__dll_export -#ifdef __attribute_used__ - __attribute_used__ -#elif defined(__GNUC__) || __has_attribute(__used__) - __attribute__((__used__)) -#endif -#ifdef __attribute_externally_visible__ - __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) - __attribute__((__externally_visible__)) -#endif - const char *const mdbx_sourcery_anchor = sourcery; diff --git a/libs/libmdbx/src/src/wingetopt.c b/libs/libmdbx/src/src/wingetopt.c deleted file mode 100644 index 7feb223fc1..0000000000 --- a/libs/libmdbx/src/src/wingetopt.c +++ /dev/null @@ -1,95 +0,0 @@ -/* - * POSIX getopt for Windows - * - * AT&T Public License - * - * Code given out at the 1985 UNIFORUM conference in Dallas. - */ - -/*----------------------------------------------------------------------------*/ -/* Microsoft compiler generates a lot of warning for self includes... */ - -#ifdef _MSC_VER -#pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ - expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ - * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ - * not guaranteed. Specify /EHsc */ -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif -#endif /* _MSC_VER (warnings) */ - -#include "wingetopt.h" -#include <stdio.h> -#include <string.h> - -#ifdef _MSC_VER -#pragma warning(pop) -#endif -/*----------------------------------------------------------------------------*/ - -#ifndef NULL -#define NULL 0 -#endif - -#ifndef EOF -#define EOF (-1) -#endif - -#define ERR(s, c) \ - if (opterr) { \ - fputs(argv[0], stderr); \ - fputs(s, stderr); \ - fputc(c, stderr); \ - } - -int opterr = 1; -int optind = 1; -int optopt; -char *optarg; - -int getopt(int argc, char *const argv[], const char *opts) { - static int sp = 1; - int c; - const char *cp; - - if (sp == 1) { - if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') - return EOF; - else if (strcmp(argv[optind], "--") == 0) { - optind++; - return EOF; - } - } - optopt = c = argv[optind][sp]; - if (c == ':' || (cp = strchr(opts, c)) == NULL) { - ERR(": illegal option -- ", c); - if (argv[optind][++sp] == '\0') { - optind++; - sp = 1; - } - return '?'; - } - if (*++cp == ':') { - if (argv[optind][sp + 1] != '\0') - optarg = &argv[optind++][sp + 1]; - else if (++optind >= argc) { - ERR(": option requires an argument -- ", c); - sp = 1; - return '?'; - } else - optarg = argv[optind++]; - sp = 1; - } else { - if (argv[optind][++sp] == '\0') { - sp = 1; - optind++; - } - optarg = NULL; - } - return c; -} diff --git a/libs/libmdbx/src/src/wingetopt.h b/libs/libmdbx/src/src/wingetopt.h deleted file mode 100644 index d328e38ccc..0000000000 --- a/libs/libmdbx/src/src/wingetopt.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * POSIX getopt for Windows - * - * AT&T Public License - * - * Code given out at the 1985 UNIFORUM conference in Dallas. - */ - -#ifndef _WINGETOPT_H_ -#define _WINGETOPT_H_ - -/* Bit of madness for Windows console */ -#define mdbx_strerror mdbx_strerror_ANSI2OEM -#define mdbx_strerror_r mdbx_strerror_r_ANSI2OEM - -#ifdef __cplusplus -extern "C" { -#endif - -extern int opterr; -extern int optind; -extern int optopt; -extern char *optarg; -int getopt(int argc, char *const argv[], const char *optstring); - -#ifdef __cplusplus -} -#endif - -#endif /* _GETOPT_H_ */ diff --git a/libs/libmdbx/src/test/CMakeLists.txt b/libs/libmdbx/src/test/CMakeLists.txt deleted file mode 100644 index 3e0a929b76..0000000000 --- a/libs/libmdbx/src/test/CMakeLists.txt +++ /dev/null @@ -1,72 +0,0 @@ -add_executable(mdbx_test - base.h - cases.cc - chrono.cc - chrono.h - config.cc - config.h - copy.cc - dead.cc - hill.cc - jitter.cc - keygen.cc - keygen.h - log.cc - log.h - main.cc - osal.h - osal-unix.cc - osal-windows.cc - test.cc - test.h - try.cc - utils.cc - utils.h - append.cc - ttl.cc - nested.cc - ) - -list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_20 HAS_CXX20) -list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_17 HAS_CXX17) -list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_14 HAS_CXX14) -list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_11 HAS_CXX11) -if(NOT DEFINED MDBX_CXX_STANDARD) - if(DEFINED CMAKE_CXX_STANDARD) - set(MDBX_CXX_STANDARD ${CMAKE_CXX_STANDARD}) - elseif(NOT HAS_CXX20 LESS 0) - set(MDBX_CXX_STANDARD 20) - elseif(NOT HAS_CXX17 LESS 0) - set(MDBX_CXX_STANDARD 17) - elseif(NOT HAS_CXX14 LESS 0) - set(MDBX_CXX_STANDARD 14) - elseif(NOT HAS_CXX11 LESS 0) - set(MDBX_CXX_STANDARD 11) - endif() -endif() -if(MDBX_CXX_STANDARD) - message(STATUS "Use C++${MDBX_CXX_STANDARD} for libmdbx") - if(NOT SUBPROJECT OR NOT DEFINED CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD ${MDBX_CXX_STANDARD}) - endif() -endif() - -if(MDBX_CXX_STANDARD) - set_target_properties(mdbx_test PROPERTIES - CXX_STANDARD ${MDBX_CXX_STANDARD} CXX_STANDARD_REQUIRED ON) -endif() - -set_target_properties(mdbx_test PROPERTIES - INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>) -target_setup_options(mdbx_test) - -target_link_libraries(mdbx_test ${TOOL_MDBX_LIB} ${LIB_MATH} ${CMAKE_THREAD_LIBS_INIT}) -if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - target_link_libraries(mdbx_test winmm.lib) -endif() - -if(UNIX AND NOT SUBPROJECT) - add_executable(pcrf_test pcrf/pcrf_test.c) - target_include_directories(pcrf_test PRIVATE "${PROJECT_SOURCE_DIR}") - target_link_libraries(pcrf_test ${TOOL_MDBX_LIB}) -endif() diff --git a/libs/libmdbx/src/test/append.cc b/libs/libmdbx/src/test/append.cc deleted file mode 100644 index 2fc8e429da..0000000000 --- a/libs/libmdbx/src/test/append.cc +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -bool testcase_append::run() { - int err = db_open__begin__table_create_open_clean(dbi); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("append: bailout-prepare due '%s'", mdbx_strerror(err)); - return true; - } - - keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); - /* LY: тест наполнения таблиц в append-режиме, - * при котором записи добавляются строго в конец (в порядке сортировки) */ - const unsigned flags = (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_APPEND | MDBX_APPENDDUP - : MDBX_APPEND; - keyvalue_maker.make_ordered(); - - key = keygen::alloc(config.params.keylen_max); - data = keygen::alloc(config.params.datalen_max); - keygen::buffer last_key = keygen::alloc(config.params.keylen_max); - keygen::buffer last_data = keygen::alloc(config.params.datalen_max); - last_key->value.iov_base = last_key->bytes; - last_key->value.iov_len = 0; - last_data->value.iov_base = last_data->bytes; - last_data->value.iov_len = 0; - - simple_checksum inserted_checksum; - uint64_t inserted_number = 0; - uint64_t serial_count = 0; - - unsigned txn_nops = 0; - uint64_t commited_inserted_number = inserted_number; - simple_checksum commited_inserted_checksum = inserted_checksum; - while (should_continue()) { - const keygen::serial_t serial = serial_count; - if (!keyvalue_maker.increment(serial_count, 1)) { - // дошли до границы пространства ключей - break; - } - - log_trace("append: append-a %" PRIu64, serial); - generate_pair(serial); - int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value, - &last_key->value) - : 1; - if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT)) - cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value); - - err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags); - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("append: bailout-insert due '%s'", mdbx_strerror(err)); - txn_end(true); - inserted_number = commited_inserted_number; - inserted_checksum = commited_inserted_checksum; - break; - } - - if (cmp > 0) { - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_put(appenda-a)", err); - - memcpy(last_key->value.iov_base, key->value.iov_base, - last_key->value.iov_len = key->value.iov_len); - memcpy(last_data->value.iov_base, data->value.iov_base, - last_data->value.iov_len = data->value.iov_len); - ++inserted_number; - inserted_checksum.push((uint32_t)inserted_number, key->value); - inserted_checksum.push(10639, data->value); - } else { - if (unlikely(err != MDBX_EKEYMISMATCH)) - failure_perror("mdbx_put(appenda-a) != MDBX_EKEYMISMATCH", err); - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("append: bailout-commit due '%s'", mdbx_strerror(err)); - inserted_number = commited_inserted_number; - inserted_checksum = commited_inserted_checksum; - break; - } - commited_inserted_number = inserted_number; - commited_inserted_checksum = inserted_checksum; - txn_nops = 0; - } - - report(1); - } - - if (txn_guard) { - err = breakable_commit(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("append: bailout-commit due '%s'", mdbx_strerror(err)); - inserted_number = commited_inserted_number; - inserted_checksum = commited_inserted_checksum; - } - } - //---------------------------------------------------------------------------- - txn_begin(true); - cursor_open(dbi); - - MDBX_val check_key, check_data; - err = - mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST); - if (likely(inserted_number)) { - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_cursor_get(MDBX_FIRST)", err); - } - - simple_checksum read_checksum; - uint64_t read_count = 0; - while (err == MDBX_SUCCESS) { - ++read_count; - read_checksum.push((uint32_t)read_count, check_key); - read_checksum.push(10639, check_data); - - err = - mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_NEXT); - } - - if (unlikely(err != MDBX_NOTFOUND)) - failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err); - - if (unlikely(read_count != inserted_number)) - failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")", - read_count, inserted_number); - - if (unlikely(read_checksum.value != inserted_checksum.value)) - failure("read_checksum(0x%016" PRIu64 ") " - "!= inserted_checksum(0x%016" PRIu64 ")", - read_checksum.value, inserted_checksum.value); - - cursor_close(); - txn_end(true); - //---------------------------------------------------------------------------- - - if (dbi) { - if (config.params.drop_table && !mode_readonly()) { - txn_begin(false); - db_table_drop(dbi); - err = breakable_commit(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("append: bailout-clean due '%s'", mdbx_strerror(err)); - return true; - } - } else - db_table_close(dbi); - } - return true; -} diff --git a/libs/libmdbx/src/test/base.h b/libs/libmdbx/src/test/base.h deleted file mode 100644 index 7fc6cac68c..0000000000 --- a/libs/libmdbx/src/test/base.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -#ifndef NOMINMAX -#define NOMINMAX -#endif - -#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) -#ifdef _MSC_VER -#define _CRT_SECURE_NO_WARNINGS -#pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ - expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ - semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - mode specified; termination on exception \ - is not guaranteed. Specify /EHsc */ -#endif /* _MSC_VER (warnings) */ - -/* If you wish to build your application for a previous Windows platform, - * include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you - * wish to support before including SDKDDKVer.h. - * - * TODO: #define _WIN32_WINNT WIN32_MUSTDIE */ -#include <SDKDDKVer.h> -#endif /* WINDOWS */ - -#ifdef __APPLE__ -#define _DARWIN_C_SOURCE -#endif - -#include <errno.h> -#include <limits.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> - -#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) -#include <io.h> -#else -#include <fcntl.h> -#include <sys/param.h> -#include <sys/stat.h> -#include <sys/time.h> -#include <sys/types.h> -#include <unistd.h> -#endif - -#ifdef _BSD_SOURCE -#include <endian.h> -#endif - -#include <algorithm> -#include <cassert> -#include <cinttypes> // for PRId64, PRIu64 -#include <cstdarg> -#include <cstddef> -#include <cstdint> -#include <map> -#include <memory> -#include <set> -#include <string> -#include <type_traits> -#include <unordered_map> -#include <unordered_set> -#include <vector> - -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "../mdbx.h" -#include "../src/defs.h" -#include "../src/osal.h" - -#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) -#define __thread __declspec(thread) -#endif /* __thread */ - -#include "../src/options.h" - -#ifdef _MSC_VER -#pragma warning(pop) -#pragma warning(disable : 4201) /* nonstandard extension used : \ - nameless struct / union */ -#pragma warning(disable : 4127) /* conditional expression is constant */ -#if _MSC_VER < 1900 -#pragma warning(disable : 4510) /* default constructor could \ - not be generated */ -#pragma warning(disable : 4512) /* assignment operator could \ - not be generated */ -#pragma warning(disable : 4610) /* user-defined constructor required */ -#ifndef snprintf -#define snprintf(buffer, buffer_size, format, ...) \ - _snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__) -#endif -#ifndef vsnprintf -#define vsnprintf(buffer, buffer_size, format, args) \ - _vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args) -#endif -#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \ - may be unsafe */ -#endif -#endif /* _MSC_VER */ diff --git a/libs/libmdbx/src/test/cases.cc b/libs/libmdbx/src/test/cases.cc deleted file mode 100644 index 98255f52ca..0000000000 --- a/libs/libmdbx/src/test/cases.cc +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -void configure_actor(unsigned &last_space_id, const actor_testcase testcase, - const char *space_id_cstr, const actor_params ¶ms) { - unsigned wait4id = 0; - if (params.waitfor_nops) { - for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) { - if (i->is_waitable(params.waitfor_nops)) { - if (i->signal_nops && i->signal_nops != params.waitfor_nops) - failure("Previous waitable actor (id=%u) already linked on %u-ops\n", - i->actor_id, i->signal_nops); - wait4id = i->actor_id; - i->signal_nops = params.waitfor_nops; - break; - } - } - if (!wait4id) - failure("No previous waitable actor for %u-ops\n", params.waitfor_nops); - } - - unsigned space_id = 0; - if (!space_id_cstr || strcmp(space_id_cstr, "auto") == 0) - space_id = last_space_id + 1; - else { - char *end = nullptr; - errno = 0; - space_id = strtoul(space_id_cstr, &end, 0); - if (errno) - failure_perror("Expects an integer value for space-id\n", errno); - if (end && *end) - failure("The '%s' is unexpected for space-id\n", end); - } - - if (space_id > ACTOR_ID_MAX) - failure("Invalid space-id %u\n", space_id); - last_space_id = space_id; - - log_trace("configure_actor: space %u for %s", space_id, - testcase2str(testcase)); - global::actors.emplace_back( - actor_config(testcase, params, space_id, wait4id)); - global::databases.insert(params.pathname_db); -} - -void testcase_setup(const char *casename, actor_params ¶ms, - unsigned &last_space_id) { - if (strcmp(casename, "basic") == 0) { - log_notice(">>> testcase_setup(%s)", casename); - configure_actor(last_space_id, ac_nested, nullptr, params); - configure_actor(last_space_id, ac_hill, nullptr, params); - configure_actor(last_space_id, ac_ttl, nullptr, params); - configure_actor(last_space_id, ac_copy, nullptr, params); - configure_actor(last_space_id, ac_append, nullptr, params); - configure_actor(last_space_id, ac_jitter, nullptr, params); - configure_actor(last_space_id, ac_try, nullptr, params); - configure_actor(last_space_id, ac_jitter, nullptr, params); - configure_actor(last_space_id, ac_try, nullptr, params); - log_notice("<<< testcase_setup(%s): done", casename); - } else { - failure("unknown testcase `%s`", casename); - } -} - -void keycase_setup(const char *casename, actor_params ¶ms) { - if (strcmp(casename, "random") == 0 || strcmp(casename, "prng") == 0) { - log_notice(">>> keycase_setup(%s)", casename); - params.keygen.keycase = kc_random; - // TODO - log_notice("<<< keycase_setup(%s): done", casename); - } else if (strcmp(casename, "dashes") == 0 || - strcmp(casename, "aside") == 0) { - log_notice(">>> keycase_setup(%s)", casename); - params.keygen.keycase = kc_dashes; - // TODO - log_notice("<<< keycase_setup(%s): done", casename); - } else if (strcmp(casename, "custom") == 0) { - log_notice("=== keycase_setup(%s): skip", casename); - params.keygen.keycase = kc_custom; - } else { - failure("unknown keycase `%s`", casename); - } -} - -/* TODO */ diff --git a/libs/libmdbx/src/test/chrono.cc b/libs/libmdbx/src/test/chrono.cc deleted file mode 100644 index 43cd2a370c..0000000000 --- a/libs/libmdbx/src/test/chrono.cc +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -namespace chrono { - -#ifndef NSEC_PER_SEC -#define NSEC_PER_SEC 1000000000u -#endif /* NSEC_PER_SEC */ - -uint32_t ns2fractional(uint32_t ns) { - assert(ns < NSEC_PER_SEC); - /* LY: здесь и далее используется "длинное деление", которое - * для ясности кода оставлено как есть (без ручной оптимизации). Так как - * GCC, Clang и даже MSVC сами давно умеют конвертировать деление на - * константу в быструю reciprocal-форму. */ - return ((uint64_t)ns << 32) / NSEC_PER_SEC; -} - -uint32_t fractional2ns(uint32_t fractional) { - return (fractional * (uint64_t)NSEC_PER_SEC) >> 32; -} - -#ifndef USEC_PER_SEC -#define USEC_PER_SEC 1000000u -#endif /* USEC_PER_SEC */ -uint32_t us2fractional(uint32_t us) { - assert(us < USEC_PER_SEC); - return ((uint64_t)us << 32) / USEC_PER_SEC; -} - -uint32_t fractional2us(uint32_t fractional) { - return (fractional * (uint64_t)USEC_PER_SEC) >> 32; -} - -#ifndef MSEC_PER_SEC -#define MSEC_PER_SEC 1000u -#endif /* MSEC_PER_SEC */ -uint32_t ms2fractional(uint32_t ms) { - assert(ms < MSEC_PER_SEC); - return ((uint64_t)ms << 32) / MSEC_PER_SEC; -} - -uint32_t fractional2ms(uint32_t fractional) { - return (fractional * (uint64_t)MSEC_PER_SEC) >> 32; -} - -time from_ns(uint64_t ns) { - time result; - result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) | - ns2fractional((uint32_t)(ns % NSEC_PER_SEC)); - return result; -} - -time from_us(uint64_t us) { - time result; - result.fixedpoint = ((us / USEC_PER_SEC) << 32) | - us2fractional((uint32_t)(us % USEC_PER_SEC)); - return result; -} - -time from_ms(uint64_t ms) { - time result; - result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) | - ms2fractional((uint32_t)(ms % MSEC_PER_SEC)); - return result; -} - -time now_realtime() { -#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) - static void(WINAPI * query_time)(LPFILETIME); - if (!query_time) { - query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress( - GetModuleHandle(TEXT("kernel32.dll")), - "GetSystemTimePreciseAsFileTime"); - if (!query_time) - query_time = GetSystemTimeAsFileTime; - } - - FILETIME filetime; - query_time(&filetime); - uint64_t ns100 = - (uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime; - return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u); -#else - struct timespec ts; - if (unlikely(clock_gettime(CLOCK_REALTIME, &ts))) - failure_perror("clock_gettime(CLOCK_REALTIME", errno); - - return from_timespec(ts); -#endif -} - -time now_motonic() { -#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) - static uint64_t reciprocal; - static LARGE_INTEGER Frequency; - if (reciprocal == 0) { - if (!QueryPerformanceFrequency(&Frequency)) - failure_perror("QueryPerformanceFrequency()", GetLastError()); - reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) / - Frequency.QuadPart); - assert(reciprocal); - } - - LARGE_INTEGER Counter; - if (!QueryPerformanceCounter(&Counter)) - failure_perror("QueryPerformanceCounter()", GetLastError()); - - time result; - result.fixedpoint = (Counter.QuadPart / Frequency.QuadPart) << 32; - uint64_t mod = Counter.QuadPart % Frequency.QuadPart; - result.fixedpoint += (mod * reciprocal) >> 16; - return result; -#else - struct timespec ts; - if (unlikely(clock_gettime(CLOCK_MONOTONIC, &ts))) - failure_perror("clock_gettime(CLOCK_MONOTONIC)", errno); - - return from_timespec(ts); -#endif -} - -} /* namespace chrono */ diff --git a/libs/libmdbx/src/test/chrono.h b/libs/libmdbx/src/test/chrono.h deleted file mode 100644 index f3d8debe74..0000000000 --- a/libs/libmdbx/src/test/chrono.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -#include "base.h" -#include "utils.h" - -namespace chrono { - -#pragma pack(push, 1) - -typedef union time { - uint64_t fixedpoint; - __anonymous_struct_extension__ struct { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - uint32_t fractional; - union { - uint32_t utc; - uint32_t integer; - }; -#else - union { - uint32_t utc; - uint32_t integer; - }; - uint32_t fractional; -#endif - }; - - void reset() { fixedpoint = 0; } - uint32_t seconds() const { return utc; } -} time; - -#pragma pack(pop) - -uint32_t ns2fractional(uint32_t); -uint32_t fractional2ns(uint32_t); -uint32_t us2fractional(uint32_t); -uint32_t fractional2us(uint32_t); -uint32_t ms2fractional(uint32_t); -uint32_t fractional2ms(uint32_t); - -time from_ns(uint64_t us); -time from_us(uint64_t ns); -time from_ms(uint64_t ms); - -inline time from_seconds(uint64_t seconds) { - assert(seconds < UINT32_MAX); - time result; - result.fixedpoint = seconds << 32; - return result; -} - -inline time from_utc(time_t utc) { - assert(utc >= 0); - return from_seconds((uint64_t)utc); -} - -inline time infinite() { - time result; - result.fixedpoint = UINT64_MAX; - return result; -} - -#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \ - defined(CLOCK_REALTIME) -inline time from_timespec(const struct timespec &ts) { - time result; - result.fixedpoint = - ((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec); - return result; -} -#endif /* HAVE_TIMESPEC_TV_NSEC */ - -#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL) -inline time from_timeval(const struct timeval &tv) { - time result; - result.fixedpoint = - ((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec); - return result; -} -#endif /* HAVE_TIMEVAL_TV_USEC */ - -time now_realtime(); -time now_motonic(); - -} /* namespace chrono */ diff --git a/libs/libmdbx/src/test/config.cc b/libs/libmdbx/src/test/config.cc deleted file mode 100644 index f8e6866f94..0000000000 --- a/libs/libmdbx/src/test/config.cc +++ /dev/null @@ -1,603 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -#if defined(_MSC_VER) && !defined(strcasecmp) -#define strcasecmp(str, len) _stricmp(str, len) -#endif /* _MSC_VER && strcasecmp() */ - -namespace config { - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - const char **value, const char *default_value) { - assert(narg < argc); - const char *current = argv[narg]; - const size_t optlen = strlen(option); - - if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen)) - return false; - - if (!value) { - if (current[optlen + 2] == '=') - failure("Option '--%s' doen't accept any value\n", option); - return true; - } - - *value = nullptr; - if (current[optlen + 2] == '=') { - *value = ¤t[optlen + 3]; - return true; - } - - if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) { - *value = argv[narg + 1]; - if (strcmp(*value, "default") == 0) { - if (!default_value) - failure("Option '--%s' doen't accept default value\n", option); - *value = default_value; - } - ++narg; - return true; - } - - if (default_value) { - *value = default_value; - return true; - } - - failure("No value given for '--%s' option\n", option); -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty) { - return parse_option(argc, argv, narg, option, value, allow_empty, - allow_empty ? "" : nullptr); -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty, - const char *default_value) { - const char *value_cstr; - if (!parse_option(argc, argv, narg, option, &value_cstr, default_value)) - return false; - - if (!allow_empty && strlen(value_cstr) == 0) - failure("Value for option '--%s' could't be empty\n", option); - - value = value_cstr; - return true; -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - unsigned &mask, const option_verb *verbs) { - const char *list; - if (!parse_option(argc, argv, narg, option, &list)) - return false; - - unsigned clear = 0; - while (*list) { - if (*list == ',' || *list == ' ' || *list == '\t') { - ++list; - continue; - } - - const char *const comma = strchr(list, ','); - const bool strikethrough = *list == '-' || *list == '~'; - if (strikethrough || *list == '+') - ++list; - else - mask = clear; - const size_t len = (comma) ? comma - list : strlen(list); - const option_verb *scan = verbs; - - while (true) { - if (!scan->verb) - failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list, - option); - if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) { - mask = strikethrough ? mask & ~scan->mask : mask | scan->mask; - clear = strikethrough ? clear & ~scan->mask : clear | scan->mask; - list += len; - break; - } - ++scan; - } - } - - return true; -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint64_t &value, const scale_mode scale, - const uint64_t minval, const uint64_t maxval, - const uint64_t default_value) { - - const char *value_cstr; - if (!parse_option(argc, argv, narg, option, &value_cstr)) - return false; - - if (default_value && strcmp(value_cstr, "default") == 0) { - value = default_value; - return true; - } - - if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0) { - value = minval; - return true; - } - - if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0) { - value = maxval; - return true; - } - - char *suffix = nullptr; - errno = 0; - unsigned long long raw = strtoull(value_cstr, &suffix, 0); - if ((suffix && *suffix) || errno) { - suffix = nullptr; - errno = 0; - raw = strtoull(value_cstr, &suffix, 10); - } - if (errno) - failure("Option '--%s' expects a numeric value (%s)\n", option, - test_strerror(errno)); - - uint64_t multipler = 1; - if (suffix && *suffix) { - if (scale == no_scale) - failure("Option '--%s' doen't accepts suffixes, so '%s' is unexpected\n", - option, suffix); - if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0) - multipler = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024); - else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0) - multipler = - (scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024; - else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0) - multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 - : UINT64_C(1024) * 1024 * 1024; - else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0) - multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000 - : UINT64_C(1024) * 1024 * 1024 * 1024; - else if (scale == duration && - (strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0)) - multipler = 1; - else if (scale == duration && - (strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0)) - multipler = 60; - else if (scale == duration && - (strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0)) - multipler = 3600; - else if (scale == duration && - (strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0)) - multipler = 3600 * 24; - else - failure( - "Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s" - "suffixes, but '%s' is unexpected\n", - option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "", - suffix); - } - - if (raw >= UINT64_MAX / multipler) - failure("The value for option '--%s' is too huge\n", option); - - value = raw * multipler; - if (maxval && value > maxval) - failure("The maximal value for option '--%s' is %" PRIu64 "\n", option, - maxval); - if (value < minval) - failure("The minimal value for option '--%s' is %" PRIu64 "\n", option, - minval); - return true; -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - unsigned &value, const scale_mode scale, - const unsigned minval, const unsigned maxval, - const unsigned default_value) { - - uint64_t huge; - if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval, - default_value)) - return false; - value = (unsigned)huge; - return true; -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint8_t &value, const uint8_t minval, const uint8_t maxval, - const uint8_t default_value) { - - uint64_t huge; - if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval, - default_value)) - return false; - value = (uint8_t)huge; - return true; -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int64_t &value, const int64_t minval, const int64_t maxval, - const int64_t default_value) { - uint64_t proxy = (uint64_t)value; - if (parse_option(argc, argv, narg, option, proxy, config::binary, - (uint64_t)minval, (uint64_t)maxval, - (uint64_t)default_value)) { - value = (int64_t)proxy; - return true; - } - return false; -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int32_t &value, const int32_t minval, const int32_t maxval, - const int32_t default_value) { - uint64_t proxy = (uint64_t)value; - if (parse_option(argc, argv, narg, option, proxy, config::binary, - (uint64_t)minval, (uint64_t)maxval, - (uint64_t)default_value)) { - value = (int32_t)proxy; - return true; - } - return false; -} - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - bool &value) { - const char *value_cstr = nullptr; - if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) { - const char *current = argv[narg]; - if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) { - value = false; - return true; - } - if (strncmp(current, "--dont-", 7) == 0 && - strcmp(current + 7, option) == 0) { - value = false; - return true; - } - return false; - } - - if (!value_cstr) { - value = true; - return true; - } - - if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) { - value = true; - return true; - } - - if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) { - value = false; - return true; - } - - failure( - "Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n", - option, value_cstr); -} - -//----------------------------------------------------------------------------- - -const struct option_verb mode_bits[] = {{"rdonly", MDBX_RDONLY}, - {"mapasync", MDBX_MAPASYNC}, - {"nosync-utterly", MDBX_UTTERLY_NOSYNC}, - {"nosubdir", MDBX_NOSUBDIR}, - {"nosync-safe", MDBX_SAFE_NOSYNC}, - {"nometasync", MDBX_NOMETASYNC}, - {"writemap", MDBX_WRITEMAP}, - {"notls", MDBX_NOTLS}, - {"nordahead", MDBX_NORDAHEAD}, - {"nomeminit", MDBX_NOMEMINIT}, - {"coalesce", MDBX_COALESCE}, - {"lifo", MDBX_LIFORECLAIM}, - {"perturb", MDBX_PAGEPERTURB}, - {"accede", MDBX_ACCEDE}, - {nullptr, 0}}; - -const struct option_verb table_bits[] = { - {"key.reverse", MDBX_REVERSEKEY}, - {"key.integer", MDBX_INTEGERKEY}, - {"data.integer", MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT}, - {"data.fixed", MDBX_DUPFIXED | MDBX_DUPSORT}, - {"data.reverse", MDBX_REVERSEDUP | MDBX_DUPSORT}, - {"data.dups", MDBX_DUPSORT}, - {nullptr, 0}}; - -static void dump_verbs(const char *caption, size_t bits, - const struct option_verb *verbs) { - log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits); - - const char *comma = ""; - while (verbs->mask && bits) { - if ((bits & verbs->mask) == verbs->mask) { - logging::feed("%s%s", comma, verbs->verb); - bits -= verbs->mask; - comma = ", "; - } - ++verbs; - } - - logging::feed("%s\n", (*comma == '\0') ? "none" : ""); -} - -static void dump_duration(const char *caption, unsigned duration) { - log_verbose("%s: ", caption); - if (duration) { - if (duration > 24 * 3600) - logging::feed("%u_", duration / (24 * 3600)); - if (duration > 3600) - logging::feed("%02u:", (duration % (24 * 3600)) / 3600); - logging::feed("%02u:%02u", (duration % 3600) / 60, duration % 60); - } else { - logging::feed("INFINITE"); - } - logging::feed("\n"); -} - -void dump(const char *title) { - logging::local_suffix indent(title); - - for (auto i = global::actors.begin(); i != global::actors.end(); ++i) { - log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id, - testcase2str(i->testcase), i->space_id); - indent.push(); - - if (i->params.loglevel) { - log_verbose("log: level %u, %s\n", i->params.loglevel, - i->params.pathname_log.empty() - ? "console" - : i->params.pathname_log.c_str()); - } - - log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR - ", %i %i, %i]\n", - i->params.pathname_db.c_str(), i->params.size_now, - i->params.size_lower, i->params.size_upper, - i->params.shrink_threshold, i->params.growth_step, - i->params.pagesize); - - dump_verbs("mode", i->params.mode_flags, mode_bits); - log_verbose("random-writemap: %s\n", - i->params.random_writemap ? "Yes" : "No"); - dump_verbs("table", i->params.table_flags, table_bits); - - if (i->params.test_nops) - log_verbose("iterations/records %u\n", i->params.test_nops); - else - dump_duration("duration", i->params.test_duration); - - if (i->params.nrepeat) - log_verbose("repeat %u\n", i->params.nrepeat); - else - log_verbose("repeat ETERNALLY\n"); - - log_verbose("threads %u\n", i->params.nthreads); - - log_verbose( - "keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64 - ", split %u/%u\n", - keygencase2str(i->params.keygen.keycase), i->params.keygen.width, - i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset, - i->params.keygen.split, - i->params.keygen.width - i->params.keygen.split); - log_verbose("keygen.seed: %u\n", i->params.keygen.seed); - log_verbose("keygen.zerofill: %s\n", - i->params.keygen.zero_fill ? "Yes" : "No"); - log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min, - i->params.keylen_max); - log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min, - i->params.datalen_max); - - log_verbose("batch: read %u, write %u\n", i->params.batch_read, - i->params.batch_write); - - if (i->params.waitfor_nops) - log_verbose("wait: actor %u for %u ops\n", i->wait4id, - i->params.waitfor_nops); - else if (i->params.delaystart) - dump_duration("delay", i->params.delaystart); - else - log_verbose("no-delay\n"); - - if (i->params.inject_writefaultn) - log_verbose("inject-writefault on %u ops\n", - i->params.inject_writefaultn); - else - log_verbose("no-inject-writefault\n"); - - log_verbose("limits: readers %u, tables %u, txn-bytes %zu\n", - i->params.max_readers, i->params.max_tables, - mdbx_limits_txnsize_max(i->params.pagesize)); - - log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No"); - log_verbose("ignore MDBX_MAP_FULL error: %s\n", - i->params.ignore_dbfull ? "Yes" : "No"); - log_verbose("verifying by speculum: %s\n", - i->params.speculum ? "Yes" : "No"); - - indent.pop(); - } - - dump_duration("timeout", global::config::timeout_duration_seconds); - log_verbose("cleanup: before %s, after %s\n", - global::config::cleanup_before ? "Yes" : "No", - global::config::cleanup_after ? "Yes" : "No"); - - log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No"); - log_verbose("progress indicator: %s\n", - global::config::progress_indicator ? "Yes" : "No"); - log_verbose("console mode: %s\n", - global::config::console_mode ? "Yes" : "No"); -} - -} /* namespace config */ - -//----------------------------------------------------------------------------- - -using namespace config; - -actor_config::actor_config(actor_testcase testcase, const actor_params ¶ms, - unsigned space_id, unsigned wait4id) - : actor_config_pod(1 + unsigned(global::actors.size()), testcase, space_id, - wait4id), - params(params) {} - -const std::string actor_config::serialize(const char *prefix) const { - simple_checksum checksum; - std::string result; - - if (prefix) - result.append(prefix); - - checksum.push(params.pathname_db); - result.append(params.pathname_db); - result.push_back('|'); - - checksum.push(params.pathname_log); - result.append(params.pathname_log); - result.push_back('|'); - - static_assert(std::is_trivially_copyable<actor_params_pod>::value, - "actor_params_pod should by POD"); - result.append(data2hex(static_cast<const actor_params_pod *>(¶ms), - sizeof(actor_params_pod), checksum)); - result.push_back('|'); - - static_assert(std::is_trivially_copyable<actor_config_pod>::value, - "actor_config_pod should by POD"); - result.append(data2hex(static_cast<const actor_config_pod *>(this), - sizeof(actor_config_pod), checksum)); - result.push_back('|'); - result.push_back(global::config::progress_indicator ? 'Y' : 'N'); - checksum.push(global::config::progress_indicator); - result.push_back(global::config::console_mode ? 'Y' : 'N'); - checksum.push(global::config::console_mode); - result.push_back('|'); - - result.append(osal_serialize(checksum)); - result.push_back('|'); - - result.append(std::to_string(checksum.value)); - return result; -} - -bool actor_config::deserialize(const char *str, actor_config &config) { - simple_checksum checksum; - - TRACE(">> actor_config::deserialize: %s\n", str); - - const char *slash = strchr(str, '|'); - if (!slash) { - TRACE("<< actor_config::deserialize: slash-1\n"); - return false; - } - config.params.pathname_db.assign(str, slash - str); - checksum.push(config.params.pathname_db); - str = slash + 1; - - slash = strchr(str, '|'); - if (!slash) { - TRACE("<< actor_config::deserialize: slash-2\n"); - return false; - } - config.params.pathname_log.assign(str, slash - str); - checksum.push(config.params.pathname_log); - str = slash + 1; - - slash = strchr(str, '|'); - if (!slash) { - TRACE("<< actor_config::deserialize: slash-3\n"); - return false; - } - static_assert(std::is_trivially_copyable<actor_params_pod>::value, - "actor_params_pod should by POD"); - if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params), - sizeof(actor_params_pod), checksum)) { - TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n", - (int)(slash - str), str); - return false; - } - str = slash + 1; - - slash = strchr(str, '|'); - if (!slash) { - TRACE("<< actor_config::deserialize: slash-4\n"); - return false; - } - static_assert(std::is_trivially_copyable<actor_config_pod>::value, - "actor_config_pod should by POD"); - if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config), - sizeof(actor_config_pod), checksum)) { - TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n", - (int)(slash - str), str); - return false; - } - str = slash + 1; - - slash = strchr(str, '|'); - if (!slash) { - TRACE("<< actor_config::deserialize: slash-5\n"); - return false; - } - if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N')) { - global::config::progress_indicator = str[0] == 'Y'; - checksum.push(global::config::progress_indicator); - global::config::console_mode = str[1] == 'Y'; - checksum.push(global::config::console_mode); - str = slash + 1; - - slash = strchr(str, '|'); - if (!slash) { - TRACE("<< actor_config::deserialize: slash-6\n"); - return false; - } - } - - if (!config.osal_deserialize(str, slash, checksum)) { - TRACE("<< actor_config::deserialize: osal\n"); - return false; - } - str = slash + 1; - - uint64_t verify = std::stoull(std::string(str)); - if (checksum.value != verify) { - TRACE("<< actor_config::deserialize: checksum mismatch\n"); - return false; - } - - TRACE("<< actor_config::deserialize: OK\n"); - return true; -} - -unsigned actor_params::mdbx_keylen_min() const { - return (table_flags & MDBX_INTEGERKEY) ? 4 : 0; -} - -unsigned actor_params::mdbx_keylen_max() const { - return (unsigned)mdbx_limits_keysize_max(pagesize, table_flags); -} - -unsigned actor_params::mdbx_datalen_min() const { - return (table_flags & MDBX_INTEGERDUP) ? 4 : 0; -} - -unsigned actor_params::mdbx_datalen_max() const { - return std::min((unsigned)UINT16_MAX, - (unsigned)mdbx_limits_valsize_max(pagesize, table_flags)); -} diff --git a/libs/libmdbx/src/test/config.h b/libs/libmdbx/src/test/config.h deleted file mode 100644 index 5f0331291b..0000000000 --- a/libs/libmdbx/src/test/config.h +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -#include "base.h" -#include "log.h" -#include "utils.h" - -#define ACTOR_ID_MAX INT16_MAX - -enum actor_testcase { - ac_none, - ac_hill, - ac_deadread, - ac_deadwrite, - ac_jitter, - ac_try, - ac_copy, - ac_append, - ac_ttl, - ac_nested -}; - -enum actor_status { - as_unknown, - as_debugging, - as_running, - as_successful, - as_killed, - as_failed, - as_coredump, -}; - -const char *testcase2str(const actor_testcase); -const char *status2str(actor_status status); - -enum keygen_case { - kc_random, /* [ 6.. 2.. 7.. 4.. 0.. 1.. 5.. 3.. ] */ - kc_dashes, /* [ 0123.. 4567.. ] */ - kc_custom, - /* TODO: more cases */ -}; - -const char *keygencase2str(const keygen_case); - -//----------------------------------------------------------------------------- - -namespace config { - -enum scale_mode { no_scale, decimal, binary, duration }; - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - const char **value, const char *default_value = nullptr); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty = false); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty, - const char *default_value); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - bool &value); - -struct option_verb { - const char *const verb; - unsigned mask; -}; - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - unsigned &mask, const option_verb *verbs); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint64_t &value, const scale_mode scale, - const uint64_t minval = 0, const uint64_t maxval = INT64_MAX, - const uint64_t default_value = 0); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - unsigned &value, const scale_mode scale, - const unsigned minval = 0, const unsigned maxval = INT32_MAX, - const unsigned default_value = 0); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint8_t &value, const uint8_t minval = 0, - const uint8_t maxval = 255, const uint8_t default_value = 0); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int64_t &value, const int64_t minval, const int64_t maxval, - const int64_t default_value = -1); - -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int32_t &value, const int32_t minval, const int32_t maxval, - const int32_t default_value = -1); - -inline bool parse_option_intptr(int argc, char *const argv[], int &narg, - const char *option, intptr_t &value, - const intptr_t minval, const intptr_t maxval, - const intptr_t default_value = -1) { - static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?"); - if (sizeof(intptr_t) == 8) - return parse_option(argc, argv, narg, option, - *reinterpret_cast<int64_t *>(&value), int64_t(minval), - int64_t(maxval), int64_t(default_value)); - else - return parse_option(argc, argv, narg, option, - *reinterpret_cast<int32_t *>(&value), int32_t(minval), - int32_t(maxval), int32_t(default_value)); -} - -//----------------------------------------------------------------------------- - -#pragma pack(push, 1) - -struct keygen_params_pod { - /* Параметры генератора пар key-value. Также может быть полезным описание - * алгоритма генерации в keygen.h - * - * Ключи и значения генерируются по задаваемым параметрам на основе "плоской" - * исходной координаты. При этом, в общем случае, в процессе тестов исходная - * координата последовательно итерируется в заданном диапазоне, а необходимые - * паттерны/последовательности/узоры получаются за счет преобразования - * исходной координаты, согласно описанным ниже параметрам. - * - * Стоит отметить, что порядок описания параметров для удобства совпадает с - * порядком их использования, т.е. с порядком соответствующих преобразований. - * - * Второе важное замечание касается ограничений одновременной координированной - * генерации паттеров как для ключей, так и для значений. Суть в том, что - * такая возможность не нужна по следующим причинам: - * - libmdbx поддерживает два существенно различающихся вида таблиц, - * "уникальные" (без дубликатов и без multi-value), и так называемые - * "с дубликатами" (c multi-value). - * - Для таблиц "без дубликатов" только размер связанных с ключами значений - * (данных) оказывает влияния на работу движка, непосредственно содержимое - * данных не анализируется движком и не оказывает влияния на его работу. - * - Для таблиц "с дубликатами", при наличии более одного значения для - * некоторого ключа, формируется дочернее btree-поддерево. Это дерево - * формируется во вложенной странице или отдельном "кусте" страниц, - * и обслуживается независимо от окружения родительского ключа. - * - Таким образом, паттерн генерации значений имеет смысл только для - * таблиц "с дубликатами" и только в контексте одного значения ключа. - * Иначе говоря, не имеет смысла взаимная координация при генерации - * значений для разных ключей. Поэтому генерацию значений следует - * рассматривать только в контексте связки с одним значением ключа. - * - Тем не менее, во всех случаях достаточно важным является равновероятное - * распределение всех возможных сочетаний длин ключей и данных. - * - * width: - * Большинство тестов предполагают создание или итерирование некоторого - * количества записей. При этом требуется итерирование или генерация - * значений и ключей из некоторого ограниченного пространства вариантов. - * - * Параметр width задает такую ширину пространства вариантов в битах. - * Таким образом мощность пространства вариантов (пока) всегда равна - * степени двойки. Это ограничение можно снять, но ценой увеличения - * вычислительной сложности, включая потерю простоты и прозрачности. - * - * С другой стороны, не-n-битовый width может быть полезен: - * - Позволит генерировать ключи/значения в точно задаваемом диапазоне. - * Например, перебрать в псевдо-случайном порядке 10001 значение. - * - Позволит поровну разделять заданное пространство (диапазон) - * ключей/значений между количеством потоков некратным степени двойки. - * - * mesh и seed: - * Позволяют получить псевдо-случайные последовательности ключей/значений. - * Параметр mesh задает сколько младших бит исходной плоской координаты - * будет "перемешано" (инъективно отображено), а параметр seed позволяет - * выбрать конкретный вариант "перемешивания". - * - * Перемешивание выполняется при ненулевом значении mesh. Перемешивание - * реализуется посредством применения двух инъективных функций для - * заданного количества бит: - * - применяется первая инъективная функция; - * - к результату добавляется salt полученный из seed; - * - применяется вторая инъективная функция; - * - * Следует отметить, что mesh умышленно позволяет перемешать только младшую - * часть, что при ненулевом значении split (см далее) не позволяет получать - * псевдо-случайные значений ключей без псевдо-случайности в значениях. - * - * Такое ограничение соответствуют внутренней алгоритмике libmdbx. Проще - * говоря, мы можем проверить движок псевдо-случайной последовательностью - * ключей на таблицах без дубликатов (без multi-value), а затем проверить - * корректность работу псевдо-случайной последовательностью значений на - * таблицах с дубликатами (с multi-value), опционально добавляя - * псевдо-случайности к последовательности ключей. Однако, нет смысла - * генерировать псевдо-случайные ключи, одновременно с формированием - * какого-либо паттерна в значениях, так как содержимое в данных либо - * не будет иметь значения (для таблиц без дубликатов), либо будет - * обрабатываться в отдельных btree-поддеревьях. - * - * rotate и offset: - * Для проверки слияния и разделения страниц внутри движка требуются - * генерация ключей/значений в виде не-смежных последовательностей, как-бы - * в виде "пунктира", который постепенно заполняет весь заданный диапазон. - * - * Параметры позволяют генерировать такой "пунктир". Соответственно rotate - * задает циклический сдвиг вправо, а offset задает смещение, точнее говоря - * сложение по модулю внутри диапазона заданного посредством width. - * - * Например, при rotate равном 1 (циклический сдвиг вправо на 1 бит), - * четные и нечетные исходные значения сложатся в две линейные - * последовательности, которые постепенно закроют старшую и младшую - * половины диапазона. - * - * split: - * Для таблиц без дубликатов (без multi-value ключей) фактически требуется - * генерация только ключей, а данные могут быть постоянным. Но для таблиц с - * дубликатами (с multi-value ключами) также требуется генерация значений. - * - * Ненулевое значение параметра split фактически включает генерацию значений, - * при этом значение split определяет сколько бит исходного абстрактного - * номера будет отрезано для генерации значения. - */ - - uint8_t width{0}; - uint8_t mesh{0}; - uint8_t rotate{0}; - uint8_t split{0}; - uint32_t seed{0}; - uint64_t offset{0}; - keygen_case keycase{kc_random}; - bool zero_fill{false}; -}; - -struct actor_params_pod { - unsigned mode_flags{0}; - unsigned table_flags{0}; - intptr_t size_lower{0}; - intptr_t size_now{0}; - intptr_t size_upper{0}; - int shrink_threshold{0}; - int growth_step{0}; - int pagesize{0}; - - unsigned test_duration{0}; - unsigned test_nops{0}; - unsigned nrepeat{0}; - unsigned nthreads{0}; - - unsigned keylen_min{0}, keylen_max{0}; - unsigned datalen_min{0}, datalen_max{0}; - - unsigned batch_read{0}; - unsigned batch_write{0}; - - unsigned delaystart{0}; - unsigned waitfor_nops{0}; - unsigned inject_writefaultn{0}; - - unsigned max_readers{0}; - unsigned max_tables{0}; - keygen_params_pod keygen; - - uint8_t loglevel{0}; - bool drop_table{false}; - bool ignore_dbfull{false}; - bool speculum{false}; - bool random_writemap{true}; -}; - -struct actor_config_pod { - unsigned actor_id{0}, space_id{0}; - actor_testcase testcase{ac_none}; - unsigned wait4id{0}; - unsigned signal_nops{0}; - - actor_config_pod() = default; - actor_config_pod(unsigned actor_id, actor_testcase testcase, - unsigned space_id, unsigned wait4id) - : actor_id(actor_id), space_id(space_id), testcase(testcase), - wait4id(wait4id) {} -}; - -#pragma pack(pop) - -extern const struct option_verb mode_bits[]; -extern const struct option_verb table_bits[]; -void dump(const char *title = "config-dump: "); - -} /* namespace config */ - -struct actor_params : public config::actor_params_pod { - std::string pathname_log; - std::string pathname_db; - actor_params() = default; - - void set_defaults(const std::string &tmpdir); - unsigned mdbx_keylen_min() const; - unsigned mdbx_keylen_max() const; - unsigned mdbx_datalen_min() const; - unsigned mdbx_datalen_max() const; -}; - -struct actor_config : public config::actor_config_pod { - actor_params params; - - bool wanna_event4signalling() const { return true /* TODO ? */; } - - actor_config() = default; - actor_config(actor_testcase testcase, const actor_params ¶ms, - unsigned space_id, unsigned wait4id); - - actor_config(const char *str) : actor_config() { - if (!deserialize(str, *this)) - failure("Invalid internal parameter '%s'\n", str); - } - - const std::string osal_serialize(simple_checksum &) const; - bool osal_deserialize(const char *str, const char *end, simple_checksum &); - - const std::string serialize(const char *prefix) const; - static bool deserialize(const char *str, actor_config &config); - - bool is_waitable(size_t nops) const { - switch (testcase) { - case ac_hill: - if (!params.test_nops || params.test_nops >= nops) - return true; - __fallthrough; - default: - return false; - } - } -}; diff --git a/libs/libmdbx/src/test/copy.cc b/libs/libmdbx/src/test/copy.cc deleted file mode 100644 index ff53153e1a..0000000000 --- a/libs/libmdbx/src/test/copy.cc +++ /dev/null @@ -1,26 +0,0 @@ -#include "test.h" - -void testcase_copy::copy_db(const bool with_compaction) { - int err = osal_removefile(copy_pathname); - if (err != MDBX_SUCCESS && err != MDBX_ENOFILE) - failure_perror("mdbx_removefile()", err); - - err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(), - with_compaction ? MDBX_CP_COMPACT : 0); - if (unlikely(err != MDBX_SUCCESS)) - failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)" - : "mdbx_env_copy(MDBX_CP_ASIS)", - err); -} - -bool testcase_copy::run() { - jitter_delay(); - db_open(); - assert(!txn_guard); - const bool order = flipcoin(); - jitter_delay(); - copy_db(order); - jitter_delay(); - copy_db(!order); - return true; -} diff --git a/libs/libmdbx/src/test/darwin/LICENSE b/libs/libmdbx/src/test/darwin/LICENSE deleted file mode 100644 index 6a0dd3066b..0000000000 --- a/libs/libmdbx/src/test/darwin/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -Copyright (c) 2015, Aleksey Demakov -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/libs/libmdbx/src/test/darwin/README.md b/libs/libmdbx/src/test/darwin/README.md deleted file mode 100644 index a6a8fd1a91..0000000000 --- a/libs/libmdbx/src/test/darwin/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# DarwinPthreadBarrier - -A pthread_barrier_t implementation for Mac OS/X - -There is no pthread_barrier_t in Mac OS/X pthreads. This project fixes -this omission by providing a simple-minded barrier implementation based -on a pair of pthread_mutex_t and pthread_cond_t. - diff --git a/libs/libmdbx/src/test/darwin/pthread_barrier.c b/libs/libmdbx/src/test/darwin/pthread_barrier.c deleted file mode 100644 index 054aa00708..0000000000 --- a/libs/libmdbx/src/test/darwin/pthread_barrier.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2015, Aleksey Demakov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "pthread_barrier.h" - -#include <errno.h> - -#ifdef __APPLE__ - -int pthread_barrierattr_init(pthread_barrierattr_t *attr) { - memset(attr, 0, sizeof(pthread_barrierattr_t)); - int m = pthread_mutexattr_init(&attr->mattr); - int c = pthread_condattr_init(&attr->cattr); - return m ? m : c; -} - -int pthread_barrierattr_destroy(pthread_barrierattr_t *attr) { - int c = pthread_condattr_destroy(&attr->cattr); - int m = pthread_mutexattr_destroy(&attr->mattr); - return m ? m : c; -} - -int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, - int *__restrict pshared) { - return pthread_condattr_getpshared(&attr->cattr, pshared); -} - -int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) { - int m = pthread_mutexattr_setpshared(&attr->mattr, pshared); - int c = pthread_condattr_setpshared(&attr->cattr, pshared); - return m ? m : c; -} - -int pthread_barrier_init(pthread_barrier_t *__restrict barrier, - const pthread_barrierattr_t *__restrict attr, - unsigned count) { - if (count == 0) - return errno = EINVAL; - - int rc = pthread_mutex_init(&barrier->mutex, attr ? &attr->mattr : 0); - if (rc) - return rc; - - rc = pthread_cond_init(&barrier->cond, attr ? &attr->cattr : 0); - if (rc) { - int errno_save = errno; - pthread_mutex_destroy(&barrier->mutex); - errno = errno_save; - return rc; - } - - barrier->limit = count; - barrier->count = 0; - barrier->phase = 0; - return 0; -} - -int pthread_barrier_destroy(pthread_barrier_t *barrier) { - pthread_mutex_destroy(&barrier->mutex); - pthread_cond_destroy(&barrier->cond); - return 0; -} - -int pthread_barrier_wait(pthread_barrier_t *barrier) { - int rc = pthread_mutex_lock(&barrier->mutex); - if (rc) - return rc; - - barrier->count++; - if (barrier->count >= barrier->limit) { - barrier->phase++; - barrier->count = 0; - pthread_cond_broadcast(&barrier->cond); - pthread_mutex_unlock(&barrier->mutex); - return PTHREAD_BARRIER_SERIAL_THREAD; - } else { - unsigned phase = barrier->phase; - do - pthread_cond_wait(&barrier->cond, &barrier->mutex); - while (phase == barrier->phase); - pthread_mutex_unlock(&barrier->mutex); - return 0; - } -} - -#endif /* __APPLE__ */ diff --git a/libs/libmdbx/src/test/darwin/pthread_barrier.h b/libs/libmdbx/src/test/darwin/pthread_barrier.h deleted file mode 100644 index efa9b9b751..0000000000 --- a/libs/libmdbx/src/test/darwin/pthread_barrier.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2015, Aleksey Demakov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef PTHREAD_BARRIER_H -#define PTHREAD_BARRIER_H - -#include <pthread.h> - -#ifdef __APPLE__ - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(PTHREAD_BARRIER_SERIAL_THREAD) -#define PTHREAD_BARRIER_SERIAL_THREAD (1) -#endif - -#if !defined(PTHREAD_PROCESS_PRIVATE) -#define PTHREAD_PROCESS_PRIVATE (42) -#endif -#if !defined(PTHREAD_PROCESS_SHARED) -#define PTHREAD_PROCESS_SHARED (43) -#endif - -typedef struct { - pthread_mutexattr_t mattr; - pthread_condattr_t cattr; -} pthread_barrierattr_t; - -typedef struct { - pthread_mutex_t mutex; - pthread_cond_t cond; - unsigned int limit; - unsigned int count; - unsigned int phase; -} pthread_barrier_t; - -int pthread_barrierattr_init(pthread_barrierattr_t *attr); -int pthread_barrierattr_destroy(pthread_barrierattr_t *attr); - -int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, - int *__restrict pshared); -int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared); - -int pthread_barrier_init(pthread_barrier_t *__restrict barrier, - const pthread_barrierattr_t *__restrict attr, - unsigned int count); -int pthread_barrier_destroy(pthread_barrier_t *barrier); - -int pthread_barrier_wait(pthread_barrier_t *barrier); - -#ifdef __cplusplus -} -#endif - -#endif /* __APPLE__ */ - -#endif /* PTHREAD_BARRIER_H */ diff --git a/libs/libmdbx/src/test/dead.cc b/libs/libmdbx/src/test/dead.cc deleted file mode 100644 index 97f326a2f4..0000000000 --- a/libs/libmdbx/src/test/dead.cc +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -bool testcase_deadread::run() { - db_open(); - txn_begin(true); - cursor_guard.reset(); - txn_guard.reset(); - db_guard.reset(); - return true; -} - -//----------------------------------------------------------------------------- - -bool testcase_deadwrite::run() { - db_open(); - txn_begin(false); - cursor_guard.reset(); - txn_guard.reset(); - db_guard.reset(); - return true; -} diff --git a/libs/libmdbx/src/test/dump-load.sh b/libs/libmdbx/src/test/dump-load.sh deleted file mode 100644 index 55fa5c7f33..0000000000 --- a/libs/libmdbx/src/test/dump-load.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -echo "------------------------------------------------------------------------------" - -if [ -z "$1" ]; then - echo "No mdbx-db pathname given"; - exit 2 -elif [ ! -e "$1" ]; then - echo "The mdbx-db '$1' don't exists"; - exit 2 -else - echo ">>>>>>>>>> $1" - RECO="$1.recovered" - rm -f dump1.txt dump2.txt "$RECO" - if ./mdbx_chk "$1"; then - echo ">>>>>>>>>> SOURCE VALID" - (./mdbx_dump -a "$1" > dump1.txt && \ - ./mdbx_load -nf dump1.txt "$RECO" && \ - ./mdbx_chk "$RECO" && \ - echo ">>>>>>>>>> DUMP/LOAD/CHK OK") || (echo ">>>>>>>>>> DUMP/LOAD/CHK FAILED"; exit 1) - REMOVE_RECO=1 - elif ./mdbx_chk -i "$1"; then - echo ">>>>>>>>>> SOURCE HAS WRONG-ORDER, TRY RECOVERY" - (./mdbx_dump -a "$1" > dump1.txt && \ - ./mdbx_load -anf dump1.txt "$RECO" && \ - ./mdbx_chk -i "$RECO" && \ - echo ">>>>>>>>>> DUMP/LOAD/CHK OK") || (echo ">>>>>>>>>> DUMP/LOAD/CHK FAILED"; exit 1) - REMOVE_RECO=0 - else - echo ">>>>>>>>>> SOURCE CORRUPTED, TRY RECOVERY" - (./mdbx_dump -ar "$1" > dump1.txt && \ - ./mdbx_load -ranf dump1.txt "$RECO" && \ - ./mdbx_chk -i "$RECO" && \ - echo ">>>>>>>>>> DUMP/LOAD/CHK OK") || (echo ">>>>>>>>>> DUMP/LOAD/CHK FAILED"; exit 1) - REMOVE_RECO=0 - fi - ./mdbx_dump -a "$RECO" > dump2.txt && diff -u dump1.txt dump2.txt && \ - rm -f dump1.txt dump2.txt && [ $REMOVE_RECO -ne 0 ] && rm -f "$RECO" - exit 0 -fi diff --git a/libs/libmdbx/src/test/hill.cc b/libs/libmdbx/src/test/hill.cc deleted file mode 100644 index efc43abe2a..0000000000 --- a/libs/libmdbx/src/test/hill.cc +++ /dev/null @@ -1,450 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -bool testcase_hill::run() { - int err = db_open__begin__table_create_open_clean(dbi); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err)); - return false; - } - speculum.clear(); - speculum_commited.clear(); - - /* LY: тест "холмиком": - * - сначала наполняем таблицу циклическими CRUD-манипуляциями, - * которые в каждом цикле делают несколько операций, включая удаление, - * но в результате добавляют записи. - * - затем очищаем таблицу также CRUD-манипуляциями, но уже с другой - * пропорцией удалений. - * - * При этом очень многое зависит от порядка перебора ключей: - * - (псевдо)случайное распределение требуется лишь для полноты картины, - * но в целом не покрывает важных кейсов. - * - кроме (псевдо)случайного перебора требуется последовательное - * итерирование ключей интервалами различной ширины, с тем чтобы - * проверить различные варианты как разделения, так и слияния страниц - * внутри движка. - * - при не-уникальных ключах (MDBX_DUPSORT с подвариантами), для каждого - * повтора внутри движка формируется вложенное btree-дерево, - * соответственно требуется соблюдение аналогичных принципов - * итерирования для значений. - */ - - /* TODO: работа в несколько потоков */ - keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); - - keygen::buffer a_key = keygen::alloc(config.params.keylen_max); - keygen::buffer a_data_0 = keygen::alloc(config.params.datalen_max); - keygen::buffer a_data_1 = keygen::alloc(config.params.datalen_max); - keygen::buffer b_key = keygen::alloc(config.params.keylen_max); - keygen::buffer b_data = keygen::alloc(config.params.datalen_max); - - const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_NODUPDATA - : MDBX_NODUPDATA | MDBX_NOOVERWRITE; - const unsigned update_flags = - (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE - : MDBX_NODUPDATA; - - uint64_t serial_count = 0; - uint64_t commited_serial = serial_count; - unsigned txn_nops = 0; - - bool rc = false; - while (should_continue()) { - const keygen::serial_t a_serial = serial_count; - if (unlikely(!keyvalue_maker.increment(serial_count, 1))) { - log_notice("uphill: unexpected key-space overflow"); - break; - } - - const keygen::serial_t b_serial = serial_count; - assert(b_serial > a_serial); - - // создаем первую запись из пары - const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31); - log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift, - a_serial); - generate_pair(a_serial, a_key, a_data_1, age_shift); - - err = insert(a_key, a_data_1, insert_flags); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err)); - txn_restart(true, false); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - failure_perror("mdbx_put(insert-a.1)", err); - } - if (!speculum_verify()) { - log_notice("uphill: bailout after insert-a, before commit"); - goto bailout; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - commited_serial = a_serial; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("uphill: bailout after insert-a, after commit"); - goto bailout; - } - } - - // создаем вторую запись из пары - log_trace("uphill: insert-b %" PRIu64, b_serial); - generate_pair(b_serial, b_key, b_data, 0); - err = insert(b_key, b_data, insert_flags); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err)); - txn_restart(true, false); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - failure_perror("mdbx_put(insert-b)", err); - } - if (!speculum_verify()) { - log_notice("uphill: bailout after insert-b, before commit"); - goto bailout; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - commited_serial = a_serial; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("uphill: bailout after insert-b, after commit"); - goto bailout; - } - } - - // обновляем данные в первой записи - log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift, - a_serial); - generate_pair(a_serial, a_key, a_data_0, 0); - checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value); - err = replace(a_key, a_data_0, a_data_1, update_flags); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err)); - txn_restart(true, false); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - failure_perror("mdbx_replace(update-a: 1->0)", err); - } - if (!speculum_verify()) { - log_notice("uphill: bailout after update-a, before commit"); - goto bailout; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - commited_serial = a_serial; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("uphill: bailout after update-a, after commit"); - goto bailout; - } - } - - // удаляем вторую запись - log_trace("uphill: delete-b %" PRIu64, b_serial); - checkdata("uphill: delete-b", dbi, b_key->value, b_data->value); - err = remove(b_key, b_data); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err)); - txn_restart(true, false); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - failure_perror("mdbx_del(b)", err); - } - if (!speculum_verify()) { - log_notice("uphill: bailout after delete-b, before commit"); - goto bailout; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); - serial_count = commited_serial; - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - commited_serial = a_serial; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("uphill: bailout after delete-b, after commit"); - goto bailout; - } - } - - report(1); - if (!keyvalue_maker.increment(serial_count, 1)) { - // дошли до границы пространства ключей - serial_count = a_serial; - goto overflow; - } - } - - if (txn_guard) { - MDBX_stat stat; - err = mdbx_dbi_stat(txn_guard.get(), dbi, &stat, sizeof(stat)); - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_dbi_stat()", err); - - uint32_t nested_deepmask; - err = mdbx_dbi_dupsort_depthmask(txn_guard.get(), dbi, &nested_deepmask); - if (unlikely(err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE)) - failure_perror("mdbx_dbi_stat_nested_deepmask()", err); - - if (err != MDBX_SUCCESS) { - log_notice("hill: reached %d tree depth", stat.ms_depth); - } else { - std::string str; - int prev = -2, i = 0; - do { - while (!(nested_deepmask & 1)) - ++i, nested_deepmask >>= 1; - if (prev + 1 == i) { - if (str.back() != '-') - str.push_back('-'); - prev = i; - continue; - } - if (!str.empty()) { - if (str.back() == '-') - str.append(std::to_string(prev)); - str.push_back(','); - } - str.append(std::to_string(i)); - prev = i; - } while (++i, nested_deepmask >>= 1); - if (str.back() == '-') - str.append(std::to_string(prev)); - - log_notice("hill: reached %d tree depth & %s sub-tree depth(s)", - stat.ms_depth, str.c_str()); - } - } - - while (serial_count > 1) { - if (unlikely(!keyvalue_maker.increment(serial_count, -2))) - failure("downhill: unexpected key-space underflow"); - - overflow: - const keygen::serial_t a_serial = serial_count; - const keygen::serial_t b_serial = a_serial + 1; - assert(b_serial > a_serial); - - // обновляем первую запись из пары - const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31); - log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift, - a_serial); - generate_pair(a_serial, a_key, a_data_0, 0); - generate_pair(a_serial, a_key, a_data_1, age_shift); - checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value); - err = replace(a_key, a_data_1, a_data_0, update_flags); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at update-a due '%s'", - mdbx_strerror(err)); - txn_end(true); - speculum = speculum_commited; - break; - } - failure_perror("mdbx_put(update-a: 0->1)", err); - } - if (!speculum_verify()) { - log_notice("downhill: bailout after update-a, before commit"); - break; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("downhill: bailout after update-a, after commit"); - break; - } - } - - // создаем вторую запись из пары - log_trace("downhill: insert-b %" PRIu64, b_serial); - generate_pair(b_serial, b_key, b_data, 0); - err = insert(b_key, b_data, insert_flags); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at insert-a due '%s'", - mdbx_strerror(err)); - txn_end(true); - speculum = speculum_commited; - break; - } - failure_perror("mdbx_put(insert-b)", err); - } - if (!speculum_verify()) { - log_notice("downhill: bailout after insert-b, before commit"); - break; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("downhill: bailout after insert-b, after commit"); - break; - } - } - - // удаляем первую запись - log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift, - a_serial); - checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value); - err = remove(a_key, a_data_1); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at delete-a due '%s'", - mdbx_strerror(err)); - txn_end(true); - speculum = speculum_commited; - break; - } - failure_perror("mdbx_del(a)", err); - } - if (!speculum_verify()) { - log_notice("downhill: bailout after delete-a, before commit"); - break; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("downhill: bailout after delete-a, after commit"); - break; - } - } - - // удаляем вторую запись - log_trace("downhill: delete-b %" PRIu64, b_serial); - checkdata("downhill: delete-b", dbi, b_key->value, b_data->value); - err = remove(b_key, b_data); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at delete-b due '%s'", - mdbx_strerror(err)); - txn_end(true); - speculum = speculum_commited; - break; - } - failure_perror("mdbx_del(b)", err); - } - if (!speculum_verify()) { - log_notice("downhill: bailout after delete-b, before commit"); - break; - } - - if (++txn_nops >= config.params.batch_write) { - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - speculum = speculum_commited; - break; - } - speculum_commited = speculum; - txn_nops = 0; - if (!speculum_verify()) { - log_notice("downhill: bailout after delete-b, after commit"); - goto bailout; - } - } - - report(1); - } - - rc = speculum_verify(); -bailout: - if (txn_guard) { - err = breakable_commit(); - if (unlikely(err != MDBX_SUCCESS)) - log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - } - - if (dbi) { - if (config.params.drop_table && !mode_readonly()) { - txn_begin(false); - db_table_drop(dbi); - err = breakable_commit(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err)); - return rc; - } - } else - db_table_close(dbi); - } - return rc; -} diff --git a/libs/libmdbx/src/test/jitter.cc b/libs/libmdbx/src/test/jitter.cc deleted file mode 100644 index 71d58699de..0000000000 --- a/libs/libmdbx/src/test/jitter.cc +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -bool testcase_jitter::run() { - int err; - size_t upper_limit = config.params.size_upper; - if (upper_limit < 1) - upper_limit = config.params.size_now * 2; - - while (should_continue()) { - jitter_delay(); - db_open(); - - if (upper_limit < 1) { - MDBX_envinfo info; - err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &info, - sizeof(info)); - if (err) - failure_perror("mdbx_env_info_ex()", err); - upper_limit = (info.mi_geo.upper < INTPTR_MAX) - ? (intptr_t)info.mi_geo.upper - : INTPTR_MAX; - } - - if (flipcoin()) { - jitter_delay(); - txn_begin(true); - fetch_canary(); - jitter_delay(); - txn_end(flipcoin()); - } - - const bool coin4size = flipcoin(); - jitter_delay(); - txn_begin(mode_readonly()); - jitter_delay(); - if (!mode_readonly()) { - fetch_canary(); - update_canary(1); - err = mdbx_env_set_geometry( - db_guard.get(), -1, -1, - coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1); - if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && - err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE) - failure_perror("mdbx_env_set_geometry-1", err); - } - txn_end(flipcoin()); - - err = mdbx_env_set_geometry( - db_guard.get(), -1, -1, - !coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1); - if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && - err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE) - failure_perror("mdbx_env_set_geometry-2", err); - - if (flipcoin()) { - jitter_delay(); - txn_begin(true); - jitter_delay(); - txn_end(flipcoin()); - } - - jitter_delay(); - err = - mdbx_env_set_geometry(db_guard.get(), -1, -1, upper_limit, -1, -1, -1); - if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && - err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE) - failure_perror("mdbx_env_set_geometry-3", err); - - db_close(); - - /* just 'align' nops with other tests with batching */ - const auto batching = - std::max(config.params.batch_read, config.params.batch_write); - report(std::max(1u, batching / 2)); - } - return true; -} diff --git a/libs/libmdbx/src/test/keygen.cc b/libs/libmdbx/src/test/keygen.cc deleted file mode 100644 index 05070afe02..0000000000 --- a/libs/libmdbx/src/test/keygen.cc +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -namespace keygen { - -static inline __pure_function serial_t mask(unsigned bits) { - assert(bits > 0 && bits <= serial_maxwith); - return serial_allones >> (serial_maxwith - bits); -} - -/* LY: https://en.wikipedia.org/wiki/Injective_function */ -serial_t injective(const serial_t serial, - const unsigned bits /* at least serial_minwith (8) */, - const serial_t salt) { - assert(bits > serial_minwith && bits <= serial_maxwith); - - /* LY: All these "magic" prime numbers were found - * and verified with a bit of brute force. */ - - static const uint64_t m[64 - serial_minwith + 1] = { - /* 8 - 24 */ - 113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937, - 309277, 834497, 1499933, 4373441, 10184137, - /* 25 - 64 */ - 10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573, - 6924051577, 19204053433, 45840188887, 53625693977, 73447827913, - 141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197, - 5871903036137, 11238507001417, 45264020802263, 105008404482889, - 81921776907059, 199987980256399, 307207457507641, 946769023178273, - 2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153, - 23171543400565993, 53353226456762893, 155627817337932409, - 227827205384840249, 816509268558278821, 576933057762605689, - 2623957345935638441, 5048241705479929949, 4634245581946485653, - 4613509448041658233, 4952535426879925961}; - static const uint8_t s[64 - serial_minwith + 1] = { - /* 8 - 24 */ - 2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11, - /* 25 - 64 */ - 11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20, - 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20, - 19}; - - const auto mult = m[bits - 8]; - const auto shift = s[bits - 8]; - serial_t result = serial * mult; - if (salt) { - const unsigned left = bits / 2; - const unsigned right = bits - left; - result = (result << left) | ((result & mask(bits)) >> right); - result = (result ^ salt) * mult; - } - - result ^= result << shift; - result &= mask(bits); - log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64 - " => %" PRIu64 "/%u", - serial, bits, mult, shift, salt, result, bits); - return result; -} - -void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, - serial_t value_age, const bool keylen_changeable) { - assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith); - assert(mapping.split <= mapping.width); - assert(mapping.mesh <= mapping.width); - assert(mapping.rotate <= mapping.width); - assert(mapping.offset <= mask(mapping.width)); - assert( - !(key_essentials.flags & ~(essentials::prng_fill_flag | MDBX_INTEGERKEY | - MDBX_REVERSEKEY | MDBX_DUPSORT))); - assert(!(value_essentials.flags & - ~(essentials::prng_fill_flag | MDBX_INTEGERDUP | MDBX_REVERSEDUP))); - - log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial, - value_age); - - if (mapping.mesh >= serial_minwith) { - serial = - (serial & ~mask(mapping.mesh)) | injective(serial, mapping.mesh, salt); - log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial); - } - - if (mapping.rotate) { - const unsigned right = mapping.rotate; - const unsigned left = mapping.width - right; - serial = (serial << left) | ((serial & mask(mapping.width)) >> right); - log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64, - mapping.rotate, serial, serial); - } - - if (mapping.offset) { - serial = (serial + mapping.offset) & mask(mapping.width); - log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset, - serial); - } - if (base) { - serial += base; - log_trace("keygen-pair: base@%" PRIu64 " => %" PRIu64, base, serial); - } - - serial_t key_serial = serial; - serial_t value_serial = value_age << mapping.split; - if (mapping.split) { - if (key_essentials.flags & MDBX_DUPSORT) { - key_serial >>= mapping.split; - value_serial += serial & mask(mapping.split); - } else { - /* Без MDBX_DUPSORT требуется уникальность ключей, а для этого нельзя - * отбрасывать какие-либо биты serial после инъективного преобразования. - * Поэтому key_serial не трогаем, а в value_serial нелинейно вмешиваем - * запрошенное количество бит из serial */ - value_serial += - (serial ^ (serial >> mapping.split)) & mask(mapping.split); - } - - value_serial |= value_age << mapping.split; - log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split, - key_serial, value_serial); - } - - log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial, - value_serial); - mk_begin(key_serial, key_essentials, *key); - mk_begin(value_serial, value_essentials, *value); - -#if 0 /* unused for now */ - if (key->value.iov_len + value->value.iov_len > pair_maxlen) { - unsigned extra = key->value.iov_len + value->value.iov_len - pair_maxlen; - if (keylen_changeable && - key->value.iov_len > std::max(8u, (unsigned)key_essentials.minlen)) { -#if defined(__GNUC__) || defined(__clang__) - const bool coin = __builtin_parityll(serial) != 0; -#else - const bool coin = INT64_C(0xF2CEECA9989BD96A) * int64_t(serial) < 0; -#endif - if (coin) { - const unsigned gap = - key->value.iov_len - std::max(8u, (unsigned)key_essentials.minlen); - const unsigned chop = std::min(gap, extra); - log_trace("keygen-pair: chop %u key-len %u -> %u", chop, - (unsigned)key->value.iov_len, - (unsigned)key->value.iov_len - chop); - key->value.iov_len -= chop; - extra -= chop; - } - } - if (extra && value->value.iov_len > - std::max(8u, (unsigned)value_essentials.minlen)) { - const unsigned gap = value->value.iov_len - - std::max(8u, (unsigned)value_essentials.minlen); - const unsigned chop = std::min(gap, extra); - log_trace("keygen-pair: chop %u value-len %u -> %u", chop, - (unsigned)value->value.iov_len, - (unsigned)value->value.iov_len - chop); - value->value.iov_len -= chop; - extra -= chop; - } - if (keylen_changeable && extra && - key->value.iov_len > std::max(8u, (unsigned)key_essentials.minlen)) { - const unsigned gap = - key->value.iov_len - std::max(8u, (unsigned)key_essentials.minlen); - const unsigned chop = std::min(gap, extra); - log_trace("keygen-pair: chop %u key-len %u -> %u", chop, - (unsigned)key->value.iov_len, - (unsigned)key->value.iov_len - chop); - key->value.iov_len -= chop; - extra -= chop; - } - } -#else - (void)keylen_changeable; -#endif /* unused for now */ - - mk_continue(key_serial, key_essentials, *key); - mk_continue(value_serial, value_essentials, *value); - - if (log_enabled(logging::trace)) { - char dump_key[4096], dump_value[4096]; - log_trace("keygen-pair: key %s, value %s", - mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)), - mdbx_dump_val(&value->value, dump_value, sizeof(dump_value))); - } -} - -void maker::setup(const config::actor_params_pod &actor, unsigned actor_id, - unsigned thread_number) { - key_essentials.flags = - actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT); - assert(actor.keylen_min <= UINT16_MAX); - key_essentials.minlen = (uint16_t)actor.keylen_min; - assert(actor.keylen_max <= UINT32_MAX); - key_essentials.maxlen = std::min( - (uint32_t)actor.keylen_max, - (uint32_t)mdbx_limits_keysize_max(actor.pagesize, key_essentials.flags)); - - value_essentials.flags = - actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP); - assert(actor.datalen_min <= UINT16_MAX); - value_essentials.minlen = (uint16_t)actor.datalen_min; - assert(actor.datalen_max <= UINT32_MAX); - value_essentials.maxlen = std::min( - (uint32_t)actor.datalen_max, - (uint32_t)mdbx_limits_valsize_max(actor.pagesize, key_essentials.flags)); - - if (!actor.keygen.zero_fill) { - key_essentials.flags |= essentials::prng_fill_flag; - value_essentials.flags |= essentials::prng_fill_flag; - } - - (void)thread_number; - mapping = actor.keygen; - salt = (actor.keygen.seed + actor_id) * UINT64_C(14653293970879851569); - - // FIXME: TODO - base = 0; -} - -void maker::make_ordered() { - mapping.mesh = 0; - mapping.rotate = 0; -} - -bool maker::is_unordered() const { - return (mapping.mesh >= serial_minwith || mapping.rotate) != 0; -} - -bool maker::increment(serial_t &serial, int delta) const { - if (serial > mask(mapping.width)) { - log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial, - mask(mapping.width)); - return false; - } - - serial_t target = serial + (int64_t)delta; - if (target > mask(mapping.width) || - ((delta > 0) ? target < serial : target > serial)) { - log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", overflow", - serial, delta, target); - return false; - } - - log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", continue", serial, - delta, target); - serial = target; - return true; -} - -//----------------------------------------------------------------------------- - -static size_t length(serial_t serial) { - size_t n = 0; - if (serial > UINT32_MAX) { - n = 4; - serial >>= 32; - } - if (serial > UINT16_MAX) { - n += 2; - serial >>= 16; - } - if (serial > UINT8_MAX) { - n += 1; - serial >>= 8; - } - return (serial > 0) ? n + 1 : n; -} - -buffer alloc(size_t limit) { - result *ptr = (result *)malloc(sizeof(result) + limit); - if (unlikely(ptr == nullptr)) - failure_perror("malloc(keyvalue_buffer)", errno); - ptr->value.iov_base = ptr->bytes; - ptr->value.iov_len = 0; - ptr->limit = limit; - return buffer(ptr); -} - -void __hot maker::mk_begin(const serial_t serial, const essentials ¶ms, - result &out) { - assert(out.limit >= params.maxlen); - assert(params.maxlen >= params.minlen); - assert(params.maxlen >= length(serial)); - - out.value.iov_len = - (params.maxlen > params.minlen) - ? params.minlen + serial % (params.maxlen - params.minlen) - : params.minlen; - - if ((params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) == 0 && - out.value.iov_len < 8) - out.value.iov_len = std::max(length(serial), out.value.iov_len); -} - -void __hot maker::mk_continue(const serial_t serial, const essentials ¶ms, - result &out) { - static_assert((essentials::prng_fill_flag & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY | - MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0, - "WTF?"); - out.value.iov_base = out.bytes; - if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) { - assert(params.maxlen == params.minlen); - assert(params.minlen == 4 || params.minlen == 8); - if (is_byteorder_le() || params.minlen == 8) - out.u64 = serial; - else - out.u32 = (uint32_t)serial; - } else if (params.flags & (MDBX_REVERSEKEY | MDBX_REVERSEDUP)) { - if (out.value.iov_len > 8) { - if (params.flags & essentials::prng_fill_flag) { - uint64_t state = serial ^ UINT64_C(0x41803711c9b75f19); - prng_fill(state, out.bytes, out.value.iov_len - 8); - } else - memset(out.bytes, '\0', out.value.iov_len - 8); - unaligned::store(out.bytes + out.value.iov_len - 8, htobe64(serial)); - } else { - out.u64 = htobe64(serial); - if (out.value.iov_len < 8) - out.value.iov_base = out.bytes + 8 - out.value.iov_len; - } - } else { - out.u64 = htole64(serial); - if (out.value.iov_len > 8) { - if (params.flags & essentials::prng_fill_flag) { - uint64_t state = serial ^ UINT64_C(0x923ab47b7ee6f6e4); - prng_fill(state, out.bytes + 8, out.value.iov_len - 8); - } else - memset(out.bytes + 8, '\0', out.value.iov_len - 8); - } - } - - assert(out.value.iov_len >= params.minlen); - assert(out.value.iov_len <= params.maxlen); - assert(out.value.iov_len >= length(serial)); - assert(out.value.iov_base >= out.bytes); - assert((uint8_t *)out.value.iov_base + out.value.iov_len <= - out.bytes + out.limit); -} - -} /* namespace keygen */ diff --git a/libs/libmdbx/src/test/keygen.h b/libs/libmdbx/src/test/keygen.h deleted file mode 100644 index c36cc1a2e7..0000000000 --- a/libs/libmdbx/src/test/keygen.h +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -#include "base.h" -#include "config.h" -#include "log.h" -#include "utils.h" - -namespace keygen { - -/* Под "генерацией ключей" здесь понимается генерация обоих значений для - * пар key-value, т.е. не только ключей, но и ассоциированных с ними данных. - * - * Генерацию ключей нельзя отнести к простым задачам, так как требования - * примерно следующие: - * - генерация разного количества уникальных ключей различной длины - * в задаваемом диапазоне; - * - возможность выбора как псевдо-случайного порядка ключей, - * так и по некоторым специфическим законам (ограниченными упорядоченными - * последовательностями, в шахматном порядке по граница диапазона и т.д.); - * - возможность генерации дубликатов с задаваемым законом распределения; - * - возможность генерации непересекающимися кластерами для параллельного - * использования в нескольких потоках; - * - использовать минимум ресурсов, как CPU, так и RAM, в том числе - * включая cache pollution и ram bandwidth. - * - * При этом заведомо известно, что для MDBX не имеет значения: - * - используемый алфавит (значения байтов); - * - частотное распределение по алфавиту; - * - абсолютное значение ключей или разность между отдельными значениями; - * - * Соответственно, в общих чертах, схема генерации следующая: - * - вводится плоская одномерная "координата" serial (uint64_t); - * - генерация специфических паттернов (последовательностей) - * реализуется посредством соответствующих преобразований "координат", при - * этом все подобные преобразования выполняются только над "координатой"; - * - итоговая "координата" преобразуется в 8-байтное суррогатное значение - * ключа; - * - для получения ключей длиной МЕНЕЕ 8 байт суррогат может усекаться - * до ненулевых байт, в том числе до нулевой длины; - * - для получения ключей длиной БОЛЕЕ 8 байт суррогат дополняется - * нулями или псевдослучайной последовательностью; - * - * Механизм генерации паттернов: - * - реализованный механизм является компромиссом между скоростью/простотой - * и гибкостью, необходимой для получения последовательностей, которых - * будет достаточно для проверки сценариев разделения и слияния страниц - * с данными внутри mdbx; - * - псевдо-случайные паттерны реализуются посредством набора инъективных - * отображающих функций; - * - не-псевдо-случайные паттерны реализуются посредством параметризируемого - * трех-этапного преобразования: - * 1) смещение (сложение) по модулю; - * 2) циклический сдвиг; - * 3) добавление абсолютного смещения (базы); - * - * Также см. описание параметров генератора ключей и значений в config.h */ - -typedef uint64_t serial_t; - -enum : serial_t { - serial_minwith = 8, - serial_maxwith = sizeof(serial_t) * 8, - serial_allones = ~(serial_t)0u -}; - -struct result { - MDBX_val value; - size_t limit; - union { - uint8_t bytes[sizeof(uint64_t)]; - uint32_t u32; - uint64_t u64; - }; - - std::string as_string() const { - return std::string((const char *)value.iov_base, value.iov_len); - } -}; - -//----------------------------------------------------------------------------- - -struct buffer_deleter /* : public std::unary_function<void, result *> */ { - void operator()(result *buffer) const { free(buffer); } -}; - -typedef std::unique_ptr<result, buffer_deleter> buffer; - -buffer alloc(size_t limit); - -class maker { - config::keygen_params_pod mapping; - serial_t base{0}; - serial_t salt{0}; - - struct essentials { - uint16_t minlen{0}; - enum { prng_fill_flag = 1 }; - uint16_t flags{0}; - uint32_t maxlen{0}; - } key_essentials, value_essentials; - - static void mk_begin(const serial_t serial, const essentials ¶ms, - result &out); - static void mk_continue(const serial_t serial, const essentials ¶ms, - result &out); - static void mk(const serial_t serial, const essentials ¶ms, result &out) { - mk_begin(serial, params, out); - mk_continue(serial, params, out); - } - -public: - void pair(serial_t serial, const buffer &key, buffer &value, - serial_t value_age, const bool keylen_changeable); - void setup(const config::actor_params_pod &actor, unsigned actor_id, - unsigned thread_number); - void make_ordered(); - bool is_unordered() const; - - bool increment(serial_t &serial, int delta) const; -}; - -} /* namespace keygen */ diff --git a/libs/libmdbx/src/test/log.cc b/libs/libmdbx/src/test/log.cc deleted file mode 100644 index f568e774fc..0000000000 --- a/libs/libmdbx/src/test/log.cc +++ /dev/null @@ -1,370 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -static void fflushall() { fflush(nullptr); } - -void failure(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - fflushall(); - logging::output_nocheckloglevel_ap(logging::failure, fmt, ap); - va_end(ap); - fflushall(); - exit(EXIT_FAILURE); -} - -const char *test_strerror(int errnum) { - static __thread char buf[1024]; - return mdbx_strerror_r(errnum, buf, sizeof(buf)); -} - -void __noreturn failure_perror(const char *what, int errnum) { - failure("%s failed: %s (%d)\n", what, test_strerror(errnum), errnum); -} - -//----------------------------------------------------------------------------- - -static void mdbx_logger(int priority, const char *function, int line, - const char *msg, va_list args) { - if (!function) - function = "unknown"; - - if (priority == MDBX_LOG_FATAL) - log_error("mdbx: fatal failure: %s, %d", function, line); - - logging::output_nocheckloglevel( - logging::loglevel(priority), - strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx %s: ", function); - logging::feed_ap(msg, args); -} - -namespace logging { - -static std::string prefix; -static std::string suffix; -static loglevel level; -static FILE *last; - -void setlevel(loglevel priority) { - level = priority; - int rc = mdbx_setup_debug(priority, - MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER, - mdbx_logger); - log_trace("set mdbx debug-opts: 0x%02x", rc); -} - -void setup(loglevel priority, const std::string &_prefix) { - setlevel(priority); - prefix = _prefix; -} - -void setup(const std::string &_prefix) { prefix = _prefix; } - -const char *level2str(const loglevel alevel) { - switch (alevel) { - default: - return "invalid/unknown"; - case extra: - return "extra"; - case trace: - return "trace"; - case debug: - return "debug"; - case verbose: - return "verbose"; - case notice: - return "notice"; - case warning: - return "warning"; - case error: - return "error"; - case failure: - return "failure"; - } -} - -bool output(const loglevel priority, const char *format, ...) { - if (lower(priority, level)) - return false; - - va_list ap; - va_start(ap, format); - output_nocheckloglevel_ap(priority, format, ap); - va_end(ap); - return true; -} - -void output_nocheckloglevel_ap(const logging::loglevel priority, - const char *format, va_list ap) { - if (last) { - putc('\n', last); - fflush(last); - if (last == stderr) { - putc('\n', stdout); - fflush(stdout); - } - last = nullptr; - } - - chrono::time now = chrono::now_realtime(); - struct tm tm; -#ifdef _MSC_VER - int rc = _localtime32_s(&tm, (const __time32_t *)&now.utc); -#else - time_t time = now.utc; - int rc = localtime_r(&time, &tm) ? MDBX_SUCCESS : errno; -#endif - if (rc != MDBX_SUCCESS) - failure_perror("localtime_r()", rc); - - last = stdout; - fprintf(last, - "[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05lu %-10s %.4s ] %s" /* TODO */, - tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, - tm.tm_sec, chrono::fractional2us(now.fractional), (long)osal_getpid(), - prefix.c_str(), level2str(priority), suffix.c_str()); - - va_list ones; - memset(&ones, 0, sizeof(ones)) /* zap MSVC and other stupid compilers */; - if (same_or_higher(priority, error)) - va_copy(ones, ap); - vfprintf(last, format, ap); - - size_t len = strlen(format); - char end = len ? format[len - 1] : '\0'; - - switch (end) { - default: - putc('\n', last); - // fall through - case '\n': - fflush(last); - last = nullptr; - // fall through - case ' ': - case '_': - case ':': - case '|': - case ',': - case '\t': - case '\b': - case '\r': - case '\0': - break; - } - - if (same_or_higher(priority, error)) { - if (last != stderr) { - fprintf(stderr, "[ %05lu %-10s %.4s ] %s", (long)osal_getpid(), - prefix.c_str(), level2str(priority), suffix.c_str()); - vfprintf(stderr, format, ones); - if (end == '\n') - fflush(stderr); - else - last = stderr; - } - va_end(ones); - } -} - -bool feed_ap(const char *format, va_list ap) { - if (!last) - return false; - - if (last == stderr) { - va_list ones; - va_copy(ones, ap); - vfprintf(stdout, format, ones); - va_end(ones); - } - vfprintf(last, format, ap); - size_t len = strlen(format); - if (len && format[len - 1] == '\n') { - fflush(last); - if (last == stderr) - fflush(stdout); - last = nullptr; - } - return true; -} - -bool feed(const char *format, ...) { - if (!last) - return false; - - va_list ap; - va_start(ap, format); - feed_ap(format, ap); - va_end(ap); - return true; -} - -local_suffix::local_suffix(const char *c_str) - : trim_pos(suffix.size()), indent(0) { - suffix.append(c_str); -} - -local_suffix::local_suffix(const std::string &str) - : trim_pos(suffix.size()), indent(0) { - suffix.append(str); -} - -void local_suffix::push() { - indent += 1; - suffix.push_back('\t'); -} - -void local_suffix::pop() { - assert(indent > 0); - if (indent > 0) { - indent -= 1; - suffix.pop_back(); - } -} - -local_suffix::~local_suffix() { suffix.erase(trim_pos); } - -void progress_canary(bool active) { - static chrono::time progress_timestamp; - chrono::time now = chrono::now_motonic(); - - if (now.fixedpoint - progress_timestamp.fixedpoint < - chrono::from_ms(42).fixedpoint) - return; - - if (osal_progress_push(active)) { - progress_timestamp = now; - return; - } - - if (progress_timestamp.fixedpoint == 0) { - putc('>', stderr); - progress_timestamp = now; - } else if (global::config::console_mode) { - if (active) { - static int last_point = -1; - int point = (now.fixedpoint >> 29) & 3; - if (point != last_point) { - progress_timestamp = now; - fprintf(stderr, "%c\b", "-\\|/"[last_point = point]); - } - } else if (now.fixedpoint - progress_timestamp.fixedpoint > - chrono::from_seconds(2).fixedpoint) { - progress_timestamp = now; - fprintf(stderr, "%c\b", "@*"[now.utc & 1]); - } - } else { - static int count; - if (active && now.fixedpoint - progress_timestamp.fixedpoint > - chrono::from_seconds(1).fixedpoint) { - putc('.', stderr); - progress_timestamp = now; - ++count; - } else if (now.fixedpoint - progress_timestamp.fixedpoint > - chrono::from_seconds(5).fixedpoint) { - putc("@*"[now.utc & 1], stderr); - progress_timestamp = now; - ++count; - } - if (count == 60) { - count = 0; - putc('\n', stderr); - } - } - fflush(stderr); -} - -} // namespace logging - -void log_extra(const char *msg, ...) { - if (logging::same_or_higher(logging::extra, logging::level)) { - va_list ap; - va_start(ap, msg); - logging::output_nocheckloglevel_ap(logging::extra, msg, ap); - va_end(ap); - } else - logging::last = nullptr; -} - -void log_trace(const char *msg, ...) { - if (logging::same_or_higher(logging::trace, logging::level)) { - va_list ap; - va_start(ap, msg); - logging::output_nocheckloglevel_ap(logging::trace, msg, ap); - va_end(ap); - } else - logging::last = nullptr; -} - -void log_debug(const char *msg, ...) { - if (logging::same_or_higher(logging::debug, logging::level)) { - va_list ap; - va_start(ap, msg); - logging::output_nocheckloglevel_ap(logging::debug, msg, ap); - va_end(ap); - } else - logging::last = nullptr; -} - -void log_verbose(const char *msg, ...) { - if (logging::same_or_higher(logging::verbose, logging::level)) { - va_list ap; - va_start(ap, msg); - logging::output_nocheckloglevel_ap(logging::verbose, msg, ap); - va_end(ap); - } else - logging::last = nullptr; -} - -void log_notice(const char *msg, ...) { - if (logging::same_or_higher(logging::notice, logging::level)) { - va_list ap; - va_start(ap, msg); - logging::output_nocheckloglevel_ap(logging::notice, msg, ap); - va_end(ap); - } else - logging::last = nullptr; -} - -void log_warning(const char *msg, ...) { - if (logging::same_or_higher(logging::warning, logging::level)) { - va_list ap; - va_start(ap, msg); - logging::output_nocheckloglevel_ap(logging::warning, msg, ap); - va_end(ap); - } else - logging::last = nullptr; -} - -void log_error(const char *msg, ...) { - if (logging::same_or_higher(logging::error, logging::level)) { - va_list ap; - va_start(ap, msg); - logging::output_nocheckloglevel_ap(logging::error, msg, ap); - va_end(ap); - } else - logging::last = nullptr; -} - -void log_trouble(const char *where, const char *what, int errnum) { - log_error("%s: %s %s", where, what, test_strerror(errnum)); -} - -bool log_enabled(const logging::loglevel priority) { - return logging::same_or_higher(priority, logging::level); -} - -void log_flush(void) { fflushall(); } diff --git a/libs/libmdbx/src/test/log.h b/libs/libmdbx/src/test/log.h deleted file mode 100644 index bb24893779..0000000000 --- a/libs/libmdbx/src/test/log.h +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -#include "base.h" -#include "chrono.h" - -void __noreturn usage(void); -void __noreturn __printf_args(1, 2) failure(const char *fmt, ...); -void __noreturn failure_perror(const char *what, int errnum); -const char *test_strerror(int errnum); - -namespace logging { - -enum loglevel { - extra = MDBX_LOG_EXTRA, - trace = MDBX_LOG_TRACE, - debug = MDBX_LOG_DEBUG, - verbose = MDBX_LOG_VERBOSE, - notice = MDBX_LOG_NOTICE, - warning = MDBX_LOG_WARN, - error = MDBX_LOG_ERROR, - failure = MDBX_LOG_FATAL -}; - -inline bool lower(loglevel left, loglevel right) { - static_assert(MDBX_LOG_EXTRA > MDBX_LOG_FATAL, "WTF?"); - return left > right; -} - -inline bool same_or_higher(loglevel left, loglevel right) { - return left <= right; -} - -const char *level2str(const loglevel level); -void setup(loglevel priority, const std::string &prefix); -void setup(const std::string &prefix); -void setlevel(loglevel priority); - -void output_nocheckloglevel_ap(const loglevel priority, const char *format, - va_list ap); -bool __printf_args(2, 3) - output(const loglevel priority, const char *format, ...); -bool feed_ap(const char *format, va_list ap); -bool __printf_args(1, 2) feed(const char *format, ...); - -void inline __printf_args(2, 3) - output_nocheckloglevel(const loglevel priority, const char *format, ...) { - va_list ap; - va_start(ap, format); - output_nocheckloglevel_ap(priority, format, ap); - va_end(ap); -} - -void progress_canary(bool active); - -class local_suffix { -protected: - size_t trim_pos; - int indent; - -public: - local_suffix(const local_suffix &) = delete; - local_suffix(const local_suffix &&) = delete; - const local_suffix &operator=(const local_suffix &) = delete; - - local_suffix(const char *c_str); - local_suffix(const std::string &str); - void push(); - void pop(); - ~local_suffix(); -}; - -} // namespace logging - -void __printf_args(1, 2) log_extra(const char *msg, ...); -void __printf_args(1, 2) log_trace(const char *msg, ...); -void __printf_args(1, 2) log_debug(const char *msg, ...); -void __printf_args(1, 2) log_verbose(const char *msg, ...); -void __printf_args(1, 2) log_notice(const char *msg, ...); -void __printf_args(1, 2) log_warning(const char *msg, ...); -void __printf_args(1, 2) log_error(const char *msg, ...); - -void log_trouble(const char *where, const char *what, int errnum); -void log_flush(void); -bool log_enabled(const logging::loglevel priority); - -#ifdef _DEBUG -#define TRACE(...) log_trace(__VA_ARGS__) -#else -#define TRACE(...) __noop(__VA_ARGS__) -#endif diff --git a/libs/libmdbx/src/test/long_stochastic.sh b/libs/libmdbx/src/test/long_stochastic.sh deleted file mode 100644 index 45d1247af7..0000000000 --- a/libs/libmdbx/src/test/long_stochastic.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env bash -if ! which make cc c++ tee lz4 >/dev/null; then - echo "Please install the following prerequisites: make cc c++ tee lz4 banner" >&2 - exit 1 -fi - -BANNER="$(which banner 2>/dev/null | echo echo)" -UNAME="$(uname -s 2>/dev/null || echo Unknown)" -set -euo pipefail - -## NOTE: Valgrind could produce some false-positive warnings -## in multi-process environment with shared memory. -## For instance, when the process "A" explicitly marks a memory -## region as "undefined", the process "B" fill it, -## and after this process "A" read such region, etc. -#VALGRIND="valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt" - -############################################################################### -# 1. clean data from prev runs and examine available RAM - -if [[ -v VALGRIND && ! -z "$VALGRIND" ]]; then - rm -f valgrind-*.log -else - VALGRIND=time -fi - -WANNA_MOUNT=0 -case ${UNAME} in - Linux) - MAKE=make - if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then - for old_test_dir in $(ls -d /dev/shm/mdbx-test.[0-9]*); do - rm -rf $old_test_dir - done - TESTDB_DIR="/dev/shm/mdbx-test.$$" - fi - mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* - - if LC_ALL=C free | grep -q -i available; then - ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 7) / 1024)) - else - ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 4) / 1024)) - fi - ;; - - FreeBSD) - MAKE=gmake - if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then - for old_test_dir in $(ls -d /tmp/mdbx-test.[0-9]*); do - umount $old_test_dir && rm -r $old_test_dir - done - TESTDB_DIR="/tmp/mdbx-test.$$" - rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR - WANNA_MOUNT=1 - else - mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* - fi - - ram_avail_mb=$(($(LC_ALL=C vmstat -s | grep -ie '[0-9] pages free$' | cut -d p -f 1) * ($(LC_ALL=C vmstat -s | grep -ie '[0-9] bytes per page$' | cut -d b -f 1) / 1024) / 1024)) - ;; - - Darwin) - MAKE=make - if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then - for vol in $(ls -d /Volumes/mdx[0-9]*[0-9]tst); do - disk=$(mount | grep $vol | cut -d ' ' -f 1) - echo "umount: volume $vol disk $disk" - hdiutil unmount $vol -force - hdiutil detach $disk - done - TESTDB_DIR="/Volumes/mdx$$tst" - WANNA_MOUNT=1 - else - mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* - fi - - pagesize=$(($(LC_ALL=C vm_stat | grep -o 'page size of [0-9]\+ bytes' | cut -d' ' -f 4) / 1024)) - freepages=$(LC_ALL=C vm_stat | grep '^Pages free:' | grep -o '[0-9]\+\.$' | cut -d'.' -f 1) - ram_avail_mb=$((pagesize * freepages / 1024)) - echo "pagesize ${pagesize}K, freepages ${freepages}, ram_avail_mb ${ram_avail_mb}" - - ;; - - *) - echo "FIXME: ${UNAME} not supported by this script" - exit 2 - ;; -esac - -############################################################################### -# 2. estimate reasonable RAM space for test-db - -echo "=== ${ram_avail_mb}M RAM available" -ram_reserve4logs_mb=1234 -if [ $ram_avail_mb -lt $ram_reserve4logs_mb ]; then - echo "=== At least ${ram_reserve4logs_mb}Mb RAM required" - exit 3 -fi - -# -# В режимах отличных от MDBX_WRITEMAP изменения до записи в файл -# будут накапливаться в памяти, что может потребовать свободной -# памяти размером с БД. Кроме этого, в тест входит сценарий -# создания копия БД на ходу. Поэтому БД не может быть больше 1/3 -# от доступной памяти. Однако, следует учесть что malloc() будет -# не сразу возвращать выделенную память системе, а также -# предусмотреть места для логов. -# -# In non-MDBX_WRITEMAP modes, updates (dirty pages) will -# accumulate in memory before writing to the disk, which may -# require a free memory up to the size of a whole database. In -# addition, the test includes a script create a copy of the -# database on the go. Therefore, the database cannot be more 1/3 -# of available memory. Moreover, should be taken into account -# that malloc() will not return the allocated memory to the -# system immediately, as well some space is required for logs. -# -db_size_mb=$(((ram_avail_mb - ram_reserve4logs_mb) / 4)) -if [ $db_size_mb -gt 17408 ]; then - db_size_mb=17408 -fi -echo "=== use ${db_size_mb}M for DB" - -############################################################################### -# 3. Create test-directory in ramfs/tmpfs, i.e. create/format/mount if required -case ${UNAME} in - Linux) - ;; - - FreeBSD) - if [[ WANNA_MOUNT ]]; then - mount -t tmpfs tmpfs $TESTDB_DIR - fi - ;; - - Darwin) - if [[ WANNA_MOUNT ]]; then - ramdisk_size_mb=$((42 + db_size_mb * 2 + ram_reserve4logs_mb)) - number_of_sectors=$((ramdisk_size_mb * 2048)) - ramdev=$(hdiutil attach -nomount ram://${number_of_sectors}) - diskutil erasevolume ExFAT "mdx$$tst" ${ramdev} - fi - ;; - - *) - echo "FIXME: ${UNAME} not supported by this script" - exit 2 - ;; -esac - -############################################################################### -# 4. Run basic test, i.e. `make check` - -${MAKE} TEST_DB=${TESTDB_DIR}/smoke.db TEST_LOG=${TESTDB_DIR}/smoke.log check -rm -f ${TESTDB_DIR}/* - -############################################################################### -# 5. run stochastic iterations - -function join { local IFS="$1"; shift; echo "$*"; } -function bit2option { local -n arr=$1; (( ($2&(1<<$3)) != 0 )) && echo -n '+' || echo -n '-'; echo "${arr[$3]}"; } - -options=(writemap coalesce lifo notls) - -function bits2list { - local -n arr=$1 - local i - local list=() - for ((i=0; i<${#arr[@]}; ++i)) do - list[$i]=$(bit2option $1 $2 $i) - done - join , "${list[@]}" -} - -function probe { - echo "----------------------------------------------- $(date)" - echo "${caption}: $*" - rm -f ${TESTDB_DIR}/* \ - && ${VALGRIND} ./mdbx_test ${speculum} --ignore-dbfull --repeat=3 --pathname=${TESTDB_DIR}/long.db --cleanup-after=no "$@" \ - | tee >(lz4 > ${TESTDB_DIR}/long.log.lz4) | grep -e reach -e achieve \ - && ${VALGRIND} ./mdbx_chk ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \ - && ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ${VALGRIND} ./mdbx_chk ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \ - || (echo "FAILED"; exit 1) -} - -#------------------------------------------------------------------------------ - -count=0 -cases='?' -for nops in 10 100 1000 10000 100000 1000000 10000000 100000000 1000000000; do - echo "=======================================================================" - wbatch=$((nops / 10 + 1)) - speculum=$([ $nops -le 1000 ] && echo '--speculum' || true) - while true; do - echo "=======================================================================" - ${BANNER} "$nops / $wbatch" - subcase=0 - for ((bits=2**${#options[@]}; --bits >= 0; )); do - seed=$(($(date +%s) + RANDOM)) - - split=30 - caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - - split=24 - caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - - split=16 - caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - - split=4 - caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ - --nops=$nops --batch.write=$wbatch --mode=$(bits2list options $bits) \ - --keygen.seed=${seed} basic - done # options - cases="${subcase}" - wbatch=$(((wbatch > 9) ? wbatch / 10 : 1)) - if [ $wbatch -eq 1 -o $((nops / wbatch)) -gt 1000 ]; then break; fi - done # batch (write-ops per txn) -done # n-ops - -echo "=== ALL DONE ====================== $(date)" diff --git a/libs/libmdbx/src/test/main.cc b/libs/libmdbx/src/test/main.cc deleted file mode 100644 index 10016ab3d5..0000000000 --- a/libs/libmdbx/src/test/main.cc +++ /dev/null @@ -1,639 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -#if !(defined(_WIN32) || defined(_WIN64)) -#include <sys/resource.h> -#include <sys/time.h> -#endif /* !Windows */ - -void __noreturn usage(void) { - puts( - "usage:\n" - " --help or -h Show this text\n" - "Common parameters:\n" - " --pathname=... Path and/or name of database files\n" - " --repeat=N Set repeat counter\n" - " --threads=N Number of thread (unsunpported for now)\n" - " --timeout=N[s|m|h|d] Set timeout in seconds/minutes/hours/days\n" - " --failfast[=YES/no] Lill all actors on first failure/error\n" - " --max-readers=N See mdbx_env_set_maxreaders() description\n" - " --max-tables=N Se mdbx_env_set_maxdbs() description\n" - " --dump-config[=YES/no] Dump entire test config before run\n" - " --progress[=YES/no] Enable/disable progress `canary`\n" - " --console[=yes/no] Enable/disable console-like output\n" - " --cleanup-before[=YES/no] Cleanup/remove and re-create database\n" - " --cleanup-after[=YES/no] Cleanup/remove database after completion\n" - "Database size control:\n" - " --pagesize=... Database page size: min, max, 256..65536\n" - " --size-lower=N[K|M|G|T] Lower-bound of size in Kb/Mb/Gb/Tb\n" - " --size-upper Upper-bound of size in Kb/Mb/Gb/Tb\n" - " --size Initial size in Kb/Mb/Gb/Tb\n" - " --shrink-threshold Shrink threshold in Kb/Mb/Gb/Tb\n" - " --growth-step Grow step in Kb/Mb/Gb/Tb\n" - "Predefined complext scenarios/cases:\n" - " --case=... Only `basic` scenario implemented for now\n" - " basic == Simultaneous multi-process execution\n" - " of test-actors: nested,hill,ttl,copy,append,jitter,try\n" - "Test actors:\n" - " --hill Fill-up and empty-down\n" - " by CRUD-operation quads\n" - " --ttl Stochastic time-to-live simulation\n" - " --nested Nested transactionы\n" - " with stochastic-size bellows\n" - " --jitter Jitter/delays simulation\n" - " --try Try write-transaction, no more\n" - " --copy Online copy/backup\n" - " --append Append-mode insertions\n" - " --dead.reader Dead-reader simulator\n" - " --dead.writer Dead-writer simulator\n" - "Actor options:\n" - " --batch.read=N Read-operations batch size\n" - " --batch.write=N Write-operations batch size\n" - " --delay=N | --no-delay (no)Delay test-actor before start\n" - " --wait4ops=N | --no-wait4ops (no)Wait for previous test-actor\n" - " completes # ops before start\n" - " --duration=N[s|m|h|d] Define running duration\n" - " --nops=N[K|M|G|T] Define number of operations/steps\n" - " --inject-writefault[=yes|NO] TBD (see the source code)\n" - " --drop[=yes|NO] Drop key-value space/table on " - "completion\n" - " --ignore-dbfull[=yes|NO] Ignore MDBX_MAP_FULL error\n" - " --speculum[=yes|NO] Use internal `speculum` to check " - "dataset\n" - "Keys and Value:\n" - " --keylen.min=N Minimal keys length\n" - " --keylen.max=N Miximal keys length\n" - " --datalen.min=N Minimal data length\n" - " --datalen.max=N Miximal data length\n" - " --keygen.width=N TBD (see the source code)\n" - " --keygen.mesh=N TBD (see the source code)\n" - " --keygen.seed=N TBD (see the source code)\n" - " --keygen.zerofill=yes|NO TBD (see the source code)\n" - " --keygen.split=N TBD (see the source code)\n" - " --keygen.rotate=N TBD (see the source code)\n" - " --keygen.offset=N TBD (see the source code)\n" - " --keygen.case=random Generator case (only `random` for now)\n" - "Database operation mode:\n" - " --mode={[+-]FLAG}[,[+-]FLAG]...\n" - " nosubdir == MDBX_NOSUBDIR\n" - " rdonly == MDBX_RDONLY\n" - " nometasync == MDBX_NOMETASYNC\n" - " lifo == MDBX_LIFORECLAIM\n" - " coalesce == MDBX_COALESCE\n" - " nosync-safe == MDBX_SAFE_NOSYNC\n" - " writemap == MDBX_WRITEMAP\n" - " mapasync == MDBX_MAPASYNC\n" - " nosync-utterly == MDBX_UTTERLY_NOSYNC\n" - " perturb == MDBX_PAGEPERTURB\n" - " notls == MDBX_NOTLS\n" - " nordahead == MDBX_NORDAHEAD\n" - " nomeminit == MDBX_NOMEMINIT\n" - " --random-writemap[=YES|no] Toggle MDBX_WRITEMAP randomly\n" - "Key-value space/table options:\n" - " --table={[+-]FLAG}[,[+-]FLAG]...\n" - " key.reverse == MDBX_REVERSEKEY\n" - " key.integer == MDBX_INTEGERKEY\n" - " data.dups == MDBX_DUPSORT\n" - " data.integer == MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT\n" - " data.fixed == MDBX_DUPFIXED | MDBX_DUPSORT\n" - " data.reverse == MDBX_REVERSEDUP | MDBX_DUPSORT\n"); - exit(EXIT_FAILURE); -} - -//----------------------------------------------------------------------------- - -void actor_params::set_defaults(const std::string &tmpdir) { - pathname_log = ""; - loglevel = -#if defined(NDEBUG) || defined(_WIN32) || defined(_WIN64) - logging::verbose; -#else - logging::trace; -#endif - - pathname_db = tmpdir + "mdbx-test.db"; - mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NOMEMINIT | - MDBX_COALESCE | MDBX_LIFORECLAIM | MDBX_ACCEDE; - table_flags = MDBX_DUPSORT; - - size_lower = -1; - size_now = - intptr_t(1024) * 1024 * ((table_flags & MDBX_DUPSORT) ? 256 : 1024); - size_upper = -1; - shrink_threshold = -1; - growth_step = -1; - pagesize = -1; - - keygen.seed = 1; - keygen.zero_fill = false; - keygen.keycase = kc_random; - keygen.width = (table_flags & MDBX_DUPSORT) ? 32 : 64; - keygen.mesh = keygen.width; - keygen.split = keygen.width / 2; - keygen.rotate = 3; - keygen.offset = 41; - - test_duration = 0; - test_nops = 1000; - nrepeat = 1; - nthreads = 1; - - keylen_min = mdbx_keylen_min(); - keylen_max = mdbx_keylen_max(); - datalen_min = mdbx_datalen_min(); - datalen_max = std::min(mdbx_datalen_max(), 256u * 1024 + 42); - - batch_read = 42; - batch_write = 42; - - delaystart = 0; - waitfor_nops = 0; - inject_writefaultn = 0; - - drop_table = false; - ignore_dbfull = false; - speculum = false; - random_writemap = true; - - max_readers = 42; - max_tables = 42; - - global::config::timeout_duration_seconds = 0 /* infinite */; - global::config::dump_config = true; - global::config::cleanup_before = true; - global::config::cleanup_after = true; - global::config::failfast = true; - global::config::progress_indicator = true; - global::config::console_mode = osal_istty(STDERR_FILENO); -} - -namespace global { - -std::vector<actor_config> actors; -std::unordered_map<unsigned, actor_config *> events; -std::unordered_map<mdbx_pid_t, actor_config *> pid2actor; -std::set<std::string> databases; -unsigned nactors; -chrono::time start_motonic; -chrono::time deadline_motonic; -bool singlemode; - -namespace config { -unsigned timeout_duration_seconds; -bool dump_config; -bool cleanup_before; -bool cleanup_after; -bool failfast; -bool progress_indicator; -bool console_mode; -} /* namespace config */ - -} /* namespace global */ - -//----------------------------------------------------------------------------- - -const char global::thunk_param_prefix[] = "--execute="; - -std::string thunk_param(const actor_config &config) { - return config.serialize(global::thunk_param_prefix); -} - -void cleanup() { - log_trace(">> cleanup"); - for (const auto &db_path : global::databases) { - int err = osal_removefile(db_path); - if (err != MDBX_SUCCESS && err != MDBX_ENOFILE) - failure_perror(db_path.c_str(), err); - } - log_trace("<< cleanup"); -} - -int main(int argc, char *const argv[]) { - -#ifdef _DEBUG - log_trace("#argc = %d", argc); - for (int i = 0; i < argc; ++i) - log_trace("#argv[%d] = %s", i, argv[i]); -#endif /* _DEBUG */ - - if (argc < 2) - failure("No parameters given. Try --help\n"); - - if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix, - strlen(global::thunk_param_prefix)) == 0) - return test_execute( - actor_config(argv[1] + strlen(global::thunk_param_prefix))) - ? EXIT_SUCCESS - : EXIT_FAILURE; - - if (argc == 2 && - (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0)) - usage(); - - actor_params params; - params.set_defaults(osal_tempdir()); - global::config::dump_config = true; - logging::setup((logging::loglevel)params.loglevel, "main"); - unsigned last_space_id = 0; - - for (int narg = 1; narg < argc; ++narg) { - const char *value = nullptr; - - if (config::parse_option(argc, argv, narg, "case", &value)) { - testcase_setup(value, params, last_space_id); - continue; - } - if (config::parse_option(argc, argv, narg, "pathname", params.pathname_db)) - continue; - if (config::parse_option(argc, argv, narg, "mode", params.mode_flags, - config::mode_bits)) - continue; - if (config::parse_option(argc, argv, narg, "random-writemap", - params.random_writemap)) - continue; - if (config::parse_option(argc, argv, narg, "table", params.table_flags, - config::table_bits)) { - if ((params.table_flags & MDBX_DUPFIXED) == 0) - params.table_flags &= ~MDBX_INTEGERDUP; - if ((params.table_flags & MDBX_DUPSORT) == 0) - params.table_flags &= - ~(MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP); - continue; - } - - if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize, - int(mdbx_limits_pgsize_min()), - int(mdbx_limits_pgsize_max()))) { - const unsigned keylen_max = params.mdbx_keylen_max(); - if (params.keylen_min > keylen_max) - params.keylen_min = keylen_max; - if (params.keylen_max > keylen_max) - params.keylen_max = keylen_max; - const unsigned datalen_max = params.mdbx_datalen_max(); - if (params.datalen_min > datalen_max) - params.datalen_min = datalen_max; - if (params.datalen_max > datalen_max) - params.datalen_max = datalen_max; - continue; - } - if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat, - config::no_scale)) - continue; - if (config::parse_option(argc, argv, narg, "threads", params.nthreads, - config::no_scale, 1, 64)) - continue; - if (config::parse_option(argc, argv, narg, "timeout", - global::config::timeout_duration_seconds, - config::duration, 1)) - continue; - - if (config::parse_option_intptr(argc, argv, narg, "size-lower", - params.size_lower, - mdbx_limits_dbsize_min(params.pagesize), - mdbx_limits_dbsize_max(params.pagesize))) - continue; - if (config::parse_option_intptr(argc, argv, narg, "size-upper", - params.size_upper, - mdbx_limits_dbsize_min(params.pagesize), - mdbx_limits_dbsize_max(params.pagesize))) - continue; - if (config::parse_option_intptr(argc, argv, narg, "size", params.size_now, - mdbx_limits_dbsize_min(params.pagesize), - mdbx_limits_dbsize_max(params.pagesize))) - continue; - if (config::parse_option( - argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0, - (int)std::min((intptr_t)INT_MAX, - mdbx_limits_dbsize_max(params.pagesize) - - mdbx_limits_dbsize_min(params.pagesize)))) - continue; - if (config::parse_option( - argc, argv, narg, "growth-step", params.growth_step, 0, - (int)std::min((intptr_t)INT_MAX, - mdbx_limits_dbsize_max(params.pagesize) - - mdbx_limits_dbsize_min(params.pagesize)))) - continue; - - if (config::parse_option(argc, argv, narg, "keygen.width", - params.keygen.width, 8, 64)) - continue; - if (config::parse_option(argc, argv, narg, "keygen.mesh", - params.keygen.mesh, 0, 64)) - continue; - if (config::parse_option(argc, argv, narg, "keygen.seed", - params.keygen.seed, config::no_scale)) - continue; - if (config::parse_option(argc, argv, narg, "keygen.zerofill", - params.keygen.zero_fill)) - continue; - if (config::parse_option(argc, argv, narg, "keygen.split", - params.keygen.split, 0, 63)) - continue; - if (config::parse_option(argc, argv, narg, "keygen.rotate", - params.keygen.rotate, 0, 63)) - continue; - if (config::parse_option(argc, argv, narg, "keygen.offset", - params.keygen.offset, config::binary)) - continue; - if (config::parse_option(argc, argv, narg, "keygen.case", &value)) { - keycase_setup(value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min, - config::no_scale, params.mdbx_keylen_min(), - params.mdbx_keylen_max())) { - if ((params.table_flags & MDBX_INTEGERKEY) || - params.keylen_max < params.keylen_min) - params.keylen_max = params.keylen_min; - continue; - } - if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max, - config::no_scale, params.mdbx_keylen_min(), - params.mdbx_keylen_max())) { - if ((params.table_flags & MDBX_INTEGERKEY) || - params.keylen_min > params.keylen_max) - params.keylen_min = params.keylen_max; - continue; - } - if (config::parse_option(argc, argv, narg, "datalen.min", - params.datalen_min, config::no_scale, - params.mdbx_datalen_min(), - params.mdbx_datalen_max())) { - if ((params.table_flags & MDBX_DUPFIXED) || - params.datalen_max < params.datalen_min) - params.datalen_max = params.datalen_min; - continue; - } - if (config::parse_option(argc, argv, narg, "datalen.max", - params.datalen_max, config::no_scale, - params.mdbx_datalen_min(), - params.mdbx_datalen_max())) { - if ((params.table_flags & MDBX_DUPFIXED) || - params.datalen_min > params.datalen_max) - params.datalen_min = params.datalen_max; - continue; - } - if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read, - config::no_scale, 1)) - continue; - if (config::parse_option(argc, argv, narg, "batch.write", - params.batch_write, config::no_scale, 1)) - continue; - if (config::parse_option(argc, argv, narg, "delay", params.delaystart, - config::duration)) - continue; - if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops, - config::decimal)) - continue; - if (config::parse_option(argc, argv, narg, "inject-writefault", - params.inject_writefaultn, config::decimal)) - continue; - if (config::parse_option(argc, argv, narg, "drop", params.drop_table)) - continue; - if (config::parse_option(argc, argv, narg, "ignore-dbfull", - params.ignore_dbfull)) - continue; - if (config::parse_option(argc, argv, narg, "speculum", params.speculum)) - continue; - if (config::parse_option(argc, argv, narg, "dump-config", - global::config::dump_config)) - continue; - if (config::parse_option(argc, argv, narg, "cleanup-before", - global::config::cleanup_before)) - continue; - if (config::parse_option(argc, argv, narg, "cleanup-after", - global::config::cleanup_after)) - continue; - if (config::parse_option(argc, argv, narg, "max-readers", - params.max_readers, config::no_scale, 1, 255)) - continue; - if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables, - config::no_scale, 1, INT16_MAX)) - continue; - - if (config::parse_option(argc, argv, narg, "no-delay", nullptr)) { - params.delaystart = 0; - continue; - } - if (config::parse_option(argc, argv, narg, "no-wait4ops", nullptr)) { - params.waitfor_nops = 0; - continue; - } - if (config::parse_option(argc, argv, narg, "duration", params.test_duration, - config::duration, 1)) { - params.test_nops = 0; - continue; - } - if (config::parse_option(argc, argv, narg, "nops", params.test_nops, - config::decimal, 1)) { - params.test_duration = 0; - continue; - } - if (config::parse_option(argc, argv, narg, "hill", &value, "auto")) { - configure_actor(last_space_id, ac_hill, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "jitter", nullptr)) { - configure_actor(last_space_id, ac_jitter, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "dead.reader", nullptr)) { - configure_actor(last_space_id, ac_deadread, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "dead.writer", nullptr)) { - configure_actor(last_space_id, ac_deadwrite, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "try", nullptr)) { - configure_actor(last_space_id, ac_try, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "copy", nullptr)) { - configure_actor(last_space_id, ac_copy, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "append", nullptr)) { - configure_actor(last_space_id, ac_append, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "ttl", nullptr)) { - configure_actor(last_space_id, ac_ttl, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "nested", nullptr)) { - configure_actor(last_space_id, ac_nested, value, params); - continue; - } - if (config::parse_option(argc, argv, narg, "failfast", - global::config::failfast)) - continue; - if (config::parse_option(argc, argv, narg, "progress", - global::config::progress_indicator)) - continue; - if (config::parse_option(argc, argv, narg, "console", - global::config::console_mode)) - continue; - - if (*argv[narg] != '-') - testcase_setup(argv[narg], params, last_space_id); - else - failure("Unknown option '%s'. Try --help\n", argv[narg]); - } - - if (global::config::dump_config) - config::dump(); - - //-------------------------------------------------------------------------- - - if (global::actors.empty()) { - log_notice("no testcase(s) configured, exiting"); - return EXIT_SUCCESS; - } - - bool failed = false; - global::start_motonic = chrono::now_motonic(); - global::deadline_motonic.fixedpoint = - (global::config::timeout_duration_seconds == 0) - ? chrono::infinite().fixedpoint - : global::start_motonic.fixedpoint + - chrono::from_seconds(global::config::timeout_duration_seconds) - .fixedpoint; - - if (global::config::cleanup_before) - cleanup(); - - log_trace(">> probe entropy_ticks()"); - entropy_ticks(); - log_trace("<< probe entropy_ticks()"); - - if (global::actors.size() == 1) { - logging::setup("main"); - global::singlemode = true; - if (!test_execute(global::actors.front())) - failed = true; - } else { - logging::setup("overlord"); - - log_trace("=== preparing..."); - log_trace(">> osal_setup"); - osal_setup(global::actors); - log_trace("<< osal_setup"); - - for (auto &a : global::actors) { - mdbx_pid_t pid; - log_trace(">> actor_start"); - int rc = osal_actor_start(a, pid); - log_trace("<< actor_start"); - if (rc) { - log_trace(">> killall_actors: (%s)", "start failed"); - osal_killall_actors(); - log_trace("<< killall_actors"); - failure("Failed to start actor #%u (%s)\n", a.actor_id, - test_strerror(rc)); - } - global::pid2actor[pid] = &a; - } - - log_trace("=== ready to start..."); - atexit(osal_killall_actors); - log_trace(">> wait4barrier"); - osal_wait4barrier(); - log_trace("<< wait4barrier"); - - size_t left = global::actors.size(); - log_trace("=== polling..."); - while (left > 0) { - unsigned timeout_seconds_left = INT_MAX; - chrono::time now_motonic = chrono::now_motonic(); - if (now_motonic.fixedpoint >= global::deadline_motonic.fixedpoint) - timeout_seconds_left = 0; - else { - chrono::time left_motonic; - left_motonic.fixedpoint = - global::deadline_motonic.fixedpoint - now_motonic.fixedpoint; - timeout_seconds_left = left_motonic.seconds(); - } - - mdbx_pid_t pid; - int rc = osal_actor_poll(pid, timeout_seconds_left); - if (rc) - failure("Poll error: %s (%d)\n", test_strerror(rc), rc); - - if (pid) { - actor_status status = osal_actor_info(pid); - actor_config *actor = global::pid2actor.at(pid); - if (!actor) - continue; - - log_verbose("actor #%u, id %d, pid %ld: %s\n", actor->actor_id, - actor->space_id, (long)pid, status2str(status)); - if (status > as_running) { - left -= 1; - if (status != as_successful) { - if (global::config::failfast && !failed) { - log_trace(">> killall_actors: (%s)", "failfast"); - osal_killall_actors(); - log_trace("<< killall_actors"); - } - failed = true; - } - } - } else { - if (timeout_seconds_left == 0) - failure("Timeout\n"); - } - } - log_trace("=== done..."); - } - - log_notice("RESULT: %s\n", failed ? "Failed" : "Successful"); - if (global::config::cleanup_after) { - if (failed) - log_verbose("skip cleanup"); - else - cleanup(); - } - -#if !(defined(_WIN32) || defined(_WIN64)) - struct rusage spent; - if (!getrusage(global::singlemode ? RUSAGE_SELF : RUSAGE_CHILDREN, &spent)) { - log_notice("%6s: user %f, system %f", "CPU", - spent.ru_utime.tv_sec + spent.ru_utime.tv_usec * 1e-6, - spent.ru_stime.tv_sec + spent.ru_stime.tv_usec * 1e-6); -#if defined(__linux__) || defined(__gnu_linux__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || \ - defined(__bsdi__) || defined(__DragonFly__) || defined(__APPLE__) || \ - defined(__MACH__) || defined(__sun) - log_notice("%6s: read %ld, write %ld", "IOPs", spent.ru_inblock, - spent.ru_oublock); - if (spent.ru_maxrss > 0) - log_notice("%6s: %ld Kb", "RAM", - spent.ru_maxrss -#if defined(__sun) - * getpagesize() / 1024u -#elif defined(__APPLE__) - / 1024u -#endif - ); - log_notice("%6s: reclaims %ld, faults %ld, swaps %ld", "Paging", - spent.ru_minflt, spent.ru_majflt, spent.ru_nswap); -#endif /* Linux */ - } -#endif /* !Windows */ - - return failed ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/libs/libmdbx/src/test/nested.cc b/libs/libmdbx/src/test/nested.cc deleted file mode 100644 index 85df6fa62b..0000000000 --- a/libs/libmdbx/src/test/nested.cc +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" -#include <cmath> - -/* LY: тест "эмуляцией time-to-live" с вложенными транзакциями: - * - организуется "скользящее окно", которое каждую транзакцию сдвигается - * вперед вдоль числовой оси. - * - по переднему краю "скользящего окна" записи добавляются в таблицу, - * а по заднему удаляются. - * - количество добавляемых/удаляемых записей псевдослучайно зависит - * от номера транзакции, но с экспоненциальным распределением. - * - размер "скользящего окна" также псевдослучайно зависит от номера - * транзакции с "отрицательным" экспоненциальным распределением - * MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний - * край и удаляются записи позади него. - * - групповое добавление данных в начало окна и групповое удаление в конце, - * преимущественно выполняются во вложенных транзакциях. - * - меньшая часть запускаемых вложенных транзакций отменяется, с последующим - * продолжением итераций с состояния предыдущиего коммита. - * - * Таким образом имитируется поведение таблицы с TTL: записи стохастически - * добавляются и удаляются, и изредка происходят массивные удаления. */ - -bool testcase_nested::setup() { - if (!inherited::setup()) - return false; - int err = db_open__begin__table_create_open_clean(dbi); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("nested: bailout-prepare due '%s'", mdbx_strerror(err)); - return false; - } - - keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); - key = keygen::alloc(config.params.keylen_max); - data = keygen::alloc(config.params.datalen_max); - serial = 0; - fifo.clear(); - speculum.clear(); - assert(stack.empty()); - stack.emplace(nullptr, serial, fifo, speculum); - return true; -} - -bool testcase_nested::teardown() { - while (!stack.empty()) - pop_txn(true); - - bool ok = true; - if (dbi) { - if (config.params.drop_table && !mode_readonly()) { - txn_begin(false); - db_table_drop(dbi); - int err = breakable_commit(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("nested: bailout-clean due '%s'", mdbx_strerror(err)); - ok = false; - } - } else - db_table_close(dbi); - dbi = 0; - } - return inherited::teardown() && ok; -} - -void testcase_nested::push_txn() { - MDBX_txn *txn; - unsigned flags = - prng32() & (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC); - int err = mdbx_txn_begin(db_guard.get(), txn_guard.get(), flags, &txn); - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_txn_begin(nested)", err); - stack.emplace(scoped_txn_guard(txn), serial, fifo, speculum); - std::swap(txn_guard, std::get<0>(stack.top())); - log_verbose("begin level#%zu txn #%" PRIu64 ", flags 0x%x, serial %" PRIu64, - stack.size(), mdbx_txn_id(txn), flags, serial); -} - -bool testcase_nested::pop_txn(bool abort) { - assert(txn_guard && !stack.empty()); - bool should_continue = true; - MDBX_txn *txn = txn_guard.release(); - bool commited = false; - if (abort) { - log_verbose( - "abort level#%zu txn #%" PRIu64 ", undo serial %" PRIu64 " <- %" PRIu64, - stack.size(), mdbx_txn_id(txn), serial, std::get<1>(stack.top())); - int err = mdbx_txn_abort(txn); - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_txn_abort()", err); - } else { - log_verbose("commit level#%zu txn, nested serial %" PRIu64 " -> %" PRIu64, - stack.size(), serial, std::get<1>(stack.top())); - int err = mdbx_txn_commit(txn); - if (likely(err == MDBX_SUCCESS)) - commited = true; - else { - should_continue = false; - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - err = mdbx_txn_abort(txn); - if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && - err != MDBX_BAD_TXN)) - failure_perror("mdbx_txn_abort()", err); - } else - failure_perror("mdbx_txn_commit()", err); - } - } - - std::swap(txn_guard, std::get<0>(stack.top())); - if (!commited) { - serial = std::get<1>(stack.top()); - std::swap(fifo, std::get<2>(stack.top())); - std::swap(speculum, std::get<3>(stack.top())); - } - stack.pop(); - return should_continue; -} - -bool testcase_nested::stochastic_breakable_restart_with_nested( - bool force_restart) { - log_trace(">> stochastic_breakable_restart_with_nested%s", - force_restart ? ": force_restart" : ""); - - if (force_restart) - while (txn_guard) - pop_txn(true); - - bool should_continue = true; - while (!stack.empty() && - (flipcoin() || txn_underutilization_x256(txn_guard.get()) < 42)) - should_continue &= pop_txn(); - - if (should_continue) - while (stack.empty() || - (is_nested_txn_available() && flipcoin() && stack.size() < 5)) - push_txn(); - - log_trace("<< stochastic_breakable_restart_with_nested: should_continue=%s", - should_continue ? "yes" : "no"); - return should_continue; -} - -bool testcase_nested::trim_tail(unsigned window_width) { - if (window_width || flipcoin()) { - clear_stepbystep_passed += window_width == 0; - while (fifo.size() > window_width) { - uint64_t tail_serial = fifo.back().first; - const unsigned tail_count = fifo.back().second; - log_verbose("nested: pop-tail (serial %" PRIu64 ", count %u)", - tail_serial, tail_count); - fifo.pop_back(); - for (unsigned n = 0; n < tail_count; ++n) { - log_trace("nested: remove-tail %" PRIu64, tail_serial); - generate_pair(tail_serial); - int err = remove(key, data); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("nested: tail-bailout due '%s'", mdbx_strerror(err)); - return false; - } - failure_perror("mdbx_del(tail)", err); - } - if (unlikely(!keyvalue_maker.increment(tail_serial, 1))) - failure("nested: unexpected key-space overflow on the tail"); - } - report(tail_count); - } - } else if (!fifo.empty()) { - log_verbose("nested: purge state %" PRIu64 " - %" PRIu64 ", fifo-items %zu", - fifo.front().first, fifo.back().first + fifo.back().second, - fifo.size()); - db_table_clear(dbi, txn_guard.get()); - fifo.clear(); - clear_wholetable_passed += 1; - report(1); - } - return true; -} - -bool testcase_nested::grow_head(unsigned head_count) { - const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_NODUPDATA - : MDBX_NODUPDATA | MDBX_NOOVERWRITE; -retry: - fifo.push_front(std::make_pair(serial, head_count)); - for (unsigned n = 0; n < head_count; ++n) { - log_trace("nested: insert-head %" PRIu64, serial); - generate_pair(serial); - int err = insert(key, data, insert_flags); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("nested: head-insert skip due '%s'", mdbx_strerror(err)); - head_count = n; - stochastic_breakable_restart_with_nested(true); - dbfull_passed += 1; - goto retry; - } - failure_perror("mdbx_put(head)", err); - } - - if (unlikely(!keyvalue_maker.increment(serial, 1))) { - log_notice("nested: unexpected key-space overflow"); - keyspace_overflow = true; - head_count = n; - stochastic_breakable_restart_with_nested(true); - goto retry; - } - } - - return true; -} - -bool testcase_nested::run() { - uint64_t seed = - prng64_map2_white(config.params.keygen.seed) + config.actor_id; - - clear_wholetable_passed = 0; - clear_stepbystep_passed = 0; - dbfull_passed = 0; - unsigned loops = 0; - while (true) { - const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; - const unsigned window_width = - (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt); - const unsigned head_count = edge2count(salt); - log_debug("nested: step #%" PRIu64 " (serial %" PRIu64 - ", window %u, count %u) salt %" PRIu64, - nops_completed, serial, window_width, head_count, salt); - - if (!trim_tail(window_width)) - return false; - if (!stochastic_breakable_restart_with_nested()) { - log_notice("nested: bailout at commit/restart after tail-trim"); - return false; - } - if (!speculum_verify()) { - log_notice("nested: bailout after tail-trim"); - return false; - } - - if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed || - !clear_stepbystep_passed)) { - unsigned underutilization_x256 = - txn_underutilization_x256(txn_guard.get()); - if (dbfull_passed > underutilization_x256) { - log_notice("nested: skip head-grow to avoid one more dbfull (was %u, " - "underutilization %.2f%%)", - dbfull_passed, underutilization_x256 / 2.560); - continue; - } - if (!grow_head(head_count)) - return false; - if (!stochastic_breakable_restart_with_nested()) - log_notice("nested: skip commit/restart after head-grow"); - if (!speculum_verify()) { - log_notice("nested: bailout after head-grow"); - return false; - } - loops += 1; - } else if (fifo.empty()) { - log_notice("nested: done %u whole loops, %" PRIu64 " ops, %" PRIu64 - " items", - loops, nops_completed, serial); - break; - } else { - log_notice("nested: done, wait for empty, skip head-grow"); - } - } - - while (!stack.empty()) - pop_txn(false); - - return speculum_verify(); -} diff --git a/libs/libmdbx/src/test/osal-unix.cc b/libs/libmdbx/src/test/osal-unix.cc deleted file mode 100644 index a85ce32a11..0000000000 --- a/libs/libmdbx/src/test/osal-unix.cc +++ /dev/null @@ -1,537 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -#if !(defined(_WIN32) || defined(_WIN64)) - -#include <pthread.h> -#include <signal.h> -#include <sys/mman.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <unistd.h> - -#include <atomic> - -#ifndef MDBX_LOCKING -#error "Opps, MDBX_LOCKING is undefined!" -#endif - -#if defined(__APPLE__) && (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008) -#include "darwin/pthread_barrier.c" -#endif /* __APPLE__ && MDBX_LOCKING >= MDBX_LOCKING_POSIX2001 */ - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV -#include <sys/ipc.h> -#include <sys/sem.h> -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - -#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#include <semaphore.h> - -#if __cplusplus >= 201103L -#include <atomic> -static __inline __maybe_unused int atomic_decrement(std::atomic_int *p) { - return std::atomic_fetch_sub(p, 1) - 1; -} -#else -static __inline __maybe_unused int atomic_decrement(volatile int *p) { -#if defined(__GNUC__) || defined(__clang__) - return __sync_sub_and_fetch(p, 1); -#elif defined(_MSC_VER) - STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile int)); - return _InterlockedDecrement((volatile long *)p); -#elif defined(__APPLE__) - return OSAtomicDecrement32Barrier((volatile int *)p); -#else -#error FIXME: Unsupported compiler -#endif -} -#endif /* C++11 */ -#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX1988 */ - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV -static int ipc; -static pid_t ipc_overlord_pid; -static void ipc_remove(void) { - if (ipc_overlord_pid == getpid()) - semctl(ipc, 0, IPC_RMID, nullptr); -} - -#else -struct shared_t { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - pthread_barrier_t barrier; - pthread_mutex_t mutex; - size_t count; - pthread_cond_t events[1]; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - struct { -#if __cplusplus >= 201103L - std::atomic_int countdown; -#else - volatile int countdown; -#endif /* C++11 */ - sem_t sema; - } barrier; - size_t count; - sem_t events[1]; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ -}; -static shared_t *shared; -#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ - -void osal_wait4barrier(void) { -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - struct sembuf op; - op.sem_num = 0; - op.sem_op = -1; - op.sem_flg = IPC_NOWAIT; - if (semop(ipc, &op, 1)) - failure_perror("semop(dec)", errno); - op.sem_op = 0; - op.sem_flg = 0; - if (semop(ipc, &op, 1)) - failure_perror("semop(wait)", errno); -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - assert(shared != nullptr && shared != MAP_FAILED); - int err = pthread_barrier_wait(&shared->barrier); - if (err != 0 && err != PTHREAD_BARRIER_SERIAL_THREAD) - failure_perror("pthread_barrier_wait(shared)", err); -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - assert(shared != nullptr && shared != MAP_FAILED); - int err = (atomic_decrement(&shared->barrier.countdown) > 0 && - sem_wait(&shared->barrier.sema)) - ? errno - : 0; - if (err != 0) - failure_perror("sem_wait(shared)", err); - if (sem_post(&shared->barrier.sema)) - failure_perror("sem_post(shared)", errno); -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ -} - -void osal_setup(const std::vector<actor_config> &actors) { -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - if (ipc_overlord_pid) - failure("ipc already created by %ld pid", (long)ipc_overlord_pid); - ipc_overlord_pid = getpid(); -#ifndef SEM_A -#define SEM_A S_IRUSR -#endif -#ifndef SEM_R -#define SEM_R S_IWUSR -#endif - ipc = semget(IPC_PRIVATE, actors.size() + 2, IPC_CREAT | SEM_A | SEM_R); - if (ipc < 0) - failure_perror("semget(IPC_PRIVATE, shared_sems)", errno); - if (atexit(ipc_remove)) - failure_perror("atexit(ipc_remove)", errno); - if (semctl(ipc, 0, SETVAL, (int)(actors.size() + 1))) - failure_perror("semctl(SETVAL.0, shared_sems)", errno); - for (size_t i = 1; i < actors.size() + 2; ++i) - if (semctl(ipc, i, SETVAL, 1)) - failure_perror("semctl(SETVAL.N, shared_sems)", errno); -#else - assert(shared == nullptr); - shared = (shared_t *)mmap( - nullptr, sizeof(shared_t) + actors.size() * sizeof(shared->events[0]), - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS -#ifdef MAP_HASSEMAPHORE - | MAP_HASSEMAPHORE -#endif - , - -1, 0); - if (MAP_FAILED == (void *)shared) - failure_perror("mmap(shared_conds)", errno); - - shared->count = actors.size() + 1; - -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - pthread_barrierattr_t barrierattr; - int err = pthread_barrierattr_init(&barrierattr); - if (err) - failure_perror("pthread_barrierattr_init()", err); - err = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED); - if (err) - failure_perror("pthread_barrierattr_setpshared()", err); - - err = pthread_barrier_init(&shared->barrier, &barrierattr, shared->count); - if (err) - failure_perror("pthread_barrier_init(shared)", err); - pthread_barrierattr_destroy(&barrierattr); - - pthread_mutexattr_t mutexattr; - err = pthread_mutexattr_init(&mutexattr); - if (err) - failure_perror("pthread_mutexattr_init()", err); - err = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED); - if (err) - failure_perror("pthread_mutexattr_setpshared()", err); - - pthread_condattr_t condattr; - err = pthread_condattr_init(&condattr); - if (err) - failure_perror("pthread_condattr_init()", err); - err = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED); - if (err) - failure_perror("pthread_condattr_setpshared()", err); - - err = pthread_mutex_init(&shared->mutex, &mutexattr); - if (err) - failure_perror("pthread_mutex_init(shared)", err); - - for (size_t i = 0; i < shared->count; ++i) { - pthread_cond_t *event = &shared->events[i]; - err = pthread_cond_init(event, &condattr); - if (err) - failure_perror("pthread_cond_init(shared)", err); - log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i, - __Wpedantic_format_voidptr(event)); - } - pthread_condattr_destroy(&condattr); - pthread_mutexattr_destroy(&mutexattr); -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - shared->barrier.countdown = shared->count; - if (sem_init(&shared->barrier.sema, true, 1)) - failure_perror("sem_init(shared.barrier)", errno); - for (size_t i = 0; i < shared->count; ++i) { - sem_t *event = &shared->events[i]; - if (sem_init(event, true, 0)) - failure_perror("sem_init(shared.event)", errno); - log_trace("osal_setup: event(shared sem_init) %" PRIuPTR " -> %p", i, - __Wpedantic_format_voidptr(event)); - } -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ -#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ -} - -void osal_broadcast(unsigned id) { - log_trace("osal_broadcast: event %u", id); -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - if (semctl(ipc, id + 1, SETVAL, 0)) - failure_perror("semctl(SETVAL)", errno); -#else - assert(shared != nullptr && shared != MAP_FAILED); - if (id >= shared->count) - failure("osal_broadcast: id > limit"); -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - int err = pthread_cond_broadcast(shared->events + id); - if (err) - failure_perror("pthread_cond_broadcast(shared)", err); -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - if (sem_post(shared->events + id)) - failure_perror("sem_post(shared)", errno); -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ -#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ -} - -int osal_waitfor(unsigned id) { - log_trace("osal_waitfor: event %u", id); -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - struct sembuf op; - memset(&op, 0, sizeof(op)); - op.sem_num = (short)(id + 1); - int rc = semop(ipc, &op, 1) ? errno : MDBX_SUCCESS; -#else - assert(shared != nullptr && shared != MAP_FAILED); - if (id >= shared->count) - failure("osal_waitfor: id > limit"); - -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - int rc = pthread_mutex_lock(&shared->mutex); - if (rc != 0) - failure_perror("pthread_mutex_lock(shared)", rc); - - rc = pthread_cond_wait(shared->events + id, &shared->mutex); - if (rc && rc != EINTR) - failure_perror("pthread_cond_wait(shared)", rc); - - rc = pthread_mutex_unlock(&shared->mutex); - if (rc != 0) - failure_perror("pthread_mutex_unlock(shared)", rc); -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - int rc = sem_wait(shared->events + id) ? errno : 0; - if (rc == 0 && sem_post(shared->events + id)) - failure_perror("sem_post(shared)", errno); -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ -#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ - - return (rc == 0) ? true : false; -} - -//----------------------------------------------------------------------------- - -const std::string -actor_config::osal_serialize(simple_checksum &checksum) const { - (void)checksum; - /* not used in workload, but just for testing */ - return "unix.fork"; -} - -bool actor_config::osal_deserialize(const char *str, const char *end, - simple_checksum &checksum) { - (void)checksum; - /* not used in workload, but just for testing */ - return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end; -} - -//----------------------------------------------------------------------------- - -static pid_t overlord_pid; - -static std::atomic<int> sigusr1_head, sigusr2_head; -static void handler_SIGUSR(int signum) { - switch (signum) { - case SIGUSR1: - ++sigusr1_head; - return; - case SIGUSR2: - ++sigusr2_head; - return; - default: - abort(); - } -} - -bool osal_progress_push(bool active) { - if (overlord_pid) { - if (kill(overlord_pid, active ? SIGUSR1 : SIGUSR2)) - failure_perror("osal_progress_push: kill(overload)", errno); - return true; - } - - return false; -} - -//----------------------------------------------------------------------------- - -static std::unordered_map<pid_t, actor_status> childs; - -static std::atomic<int> sigalarm_head; -static void handler_SIGCHLD(int signum) { - if (signum == SIGALRM) - ++sigalarm_head; -} - -mdbx_pid_t osal_getpid(void) { return getpid(); } - -int osal_delay(unsigned seconds) { return sleep(seconds) ? errno : 0; } - -int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { - if (childs.empty()) { - struct sigaction act; - memset(&act, 0, sizeof(act)); - act.sa_handler = handler_SIGCHLD; - sigaction(SIGCHLD, &act, nullptr); - sigaction(SIGALRM, &act, nullptr); - act.sa_handler = handler_SIGUSR; - sigaction(SIGUSR1, &act, nullptr); - sigaction(SIGUSR2, &act, nullptr); - - sigset_t mask; - sigemptyset(&mask); - sigaddset(&mask, SIGCHLD); - sigaddset(&mask, SIGUSR1); - sigaddset(&mask, SIGUSR2); - sigprocmask(SIG_UNBLOCK, &mask, nullptr); - } - - pid = fork(); - - if (pid == 0) { - overlord_pid = getppid(); - const bool result = test_execute(config); - exit(result ? EXIT_SUCCESS : EXIT_FAILURE); - } - - if (pid < 0) - return errno; - - log_trace("osal_actor_start: fork pid %ld for %u", (long)pid, - config.actor_id); - childs[pid] = as_running; - return 0; -} - -actor_status osal_actor_info(const mdbx_pid_t pid) { return childs.at(pid); } - -void osal_killall_actors(void) { - for (auto &pair : childs) { - kill(pair.first, SIGKILL); - pair.second = as_killed; - } -} - -int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { - static sig_atomic_t sigalarm_tail; - alarm(0) /* cancel prev timeout */; - sigalarm_tail = sigalarm_head /* reset timeout flag */; - - int options = WNOHANG; - if (timeout) { - alarm((timeout > INT_MAX) ? INT_MAX : timeout); - options = 0; - } - -#ifdef WUNTRACED - options |= WUNTRACED; -#endif -#ifdef WCONTINUED - options |= WCONTINUED; -#endif - - while (sigalarm_tail == sigalarm_head) { - int status; - pid = waitpid(0, &status, options); - - if (pid > 0) { - if (WIFEXITED(status)) - childs[pid] = - (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed; - else if (WCOREDUMP(status)) - childs[pid] = as_coredump; - else if (WIFSIGNALED(status)) - childs[pid] = as_killed; - else if (WIFSTOPPED(status)) - childs[pid] = as_debugging; - else if (WIFCONTINUED(status)) - childs[pid] = as_running; - else { - assert(false); - } - return 0; - } - - static sig_atomic_t sigusr1_tail, sigusr2_tail; - if (sigusr1_tail != sigusr1_head) { - sigusr1_tail = sigusr1_head; - logging::progress_canary(true); - if (pid < 0 && errno == EINTR) - continue; - } - if (sigusr2_tail != sigusr2_head) { - sigusr2_tail = sigusr2_head; - logging::progress_canary(false); - if (pid < 0 && errno == EINTR) - continue; - } - - if (pid == 0) - break; - - int err = errno; - if (err != EINTR) - return err; - } - return 0 /* timeout */; -} - -void osal_yield(void) { - if (sched_yield()) - failure_perror("sched_yield()", errno); -} - -void osal_udelay(unsigned us) { - chrono::time until, now = chrono::now_motonic(); - until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint; - struct timespec ts; - - static unsigned threshold_us; - if (threshold_us == 0) { -#if defined(_POSIX_CPUTIME) && _POSIX_CPUTIME > -1 && \ - defined(CLOCK_PROCESS_CPUTIME_ID) - if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) { - int rc = errno; - log_warning("clock_getres(CLOCK_PROCESS_CPUTIME_ID), failed errno %d", - rc); - } -#endif /* CLOCK_PROCESS_CPUTIME_ID */ - if (threshold_us == 0 && clock_getres(CLOCK_MONOTONIC, &ts)) { - int rc = errno; - failure_perror("clock_getres(CLOCK_MONOTONIC)", rc); - } - chrono::time threshold = chrono::from_timespec(ts); - assert(threshold.seconds() == 0); - - threshold_us = chrono::fractional2us(threshold.fractional); - if (threshold_us < 1000) - threshold_us = 1000; - } - - ts.tv_sec = ts.tv_nsec = 0; - if (us > threshold_us) { - ts.tv_sec = us / 1000000u; - ts.tv_nsec = (us % 1000000u) * 1000u; - } - - do { - if (us > threshold_us) { - if (nanosleep(&ts, &ts)) { - int rc = errno; - /* if (rc == EINTR) { ... } ? */ - failure_perror("usleep()", rc); - } - us = ts.tv_sec * 1000000u + ts.tv_nsec / 1000u; - } - cpu_relax(); - - now = chrono::now_motonic(); - } while (until.fixedpoint > now.fixedpoint); -} - -bool osal_istty(int fd) { return isatty(fd) == 1; } - -std::string osal_tempdir(void) { - const char *tempdir = getenv("TMPDIR"); - if (!tempdir) - tempdir = getenv("TMP"); - if (!tempdir) - tempdir = getenv("TEMPDIR"); - if (!tempdir) - tempdir = getenv("TEMP"); - if (tempdir && *tempdir) { - std::string dir(tempdir); - if (dir.back() != '/') - dir.append("/"); - return dir; - } - if (access("/dev/shm/", R_OK | W_OK | X_OK) == 0) - return "/dev/shm/"; - return ""; -} - -int osal_removefile(const std::string &pathname) { - return unlink(pathname.c_str()) ? errno : MDBX_SUCCESS; -} - -#endif /* !Windows */ diff --git a/libs/libmdbx/src/test/osal-windows.cc b/libs/libmdbx/src/test/osal-windows.cc deleted file mode 100644 index 9bde047a2c..0000000000 --- a/libs/libmdbx/src/test/osal-windows.cc +++ /dev/null @@ -1,459 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -#if defined(_WIN32) || defined(_WIN64) - -static std::unordered_map<unsigned, HANDLE> events; -static HANDLE hBarrierSemaphore, hBarrierEvent; -static HANDLE hProgressActiveEvent, hProgressPassiveEvent; - -static int waitstatus2errcode(DWORD result) { - switch (result) { - case WAIT_OBJECT_0: - return MDBX_SUCCESS; - case WAIT_FAILED: - return GetLastError(); - case WAIT_ABANDONED: - return ERROR_ABANDONED_WAIT_0; - case WAIT_IO_COMPLETION: - return ERROR_USER_APC; - case WAIT_TIMEOUT: - return ERROR_TIMEOUT; - default: - return ERROR_UNHANDLED_ERROR; - } -} - -void osal_wait4barrier(void) { - DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0); - switch (rc) { - default: - failure_perror("WaitForSingleObject(BarrierSemaphore)", - waitstatus2errcode(rc)); - case WAIT_OBJECT_0: - rc = WaitForSingleObject(hBarrierEvent, INFINITE); - if (rc != WAIT_OBJECT_0) - failure_perror("WaitForSingleObject(BarrierEvent)", - waitstatus2errcode(rc)); - break; - case WAIT_TIMEOUT: - if (!SetEvent(hBarrierEvent)) - failure_perror("SetEvent(BarrierEvent)", GetLastError()); - break; - } -} - -static HANDLE make_inheritable(HANDLE hHandle) { - assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE); - if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(), - &hHandle, 0, TRUE, - DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS)) - failure_perror("DuplicateHandle()", GetLastError()); - return hHandle; -} - -void osal_setup(const std::vector<actor_config> &actors) { - assert(events.empty()); - const size_t n = actors.size() + 1; - events.reserve(n); - - for (unsigned i = 0; i < n; ++i) { - HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if (!hEvent) - failure_perror("CreateEvent()", GetLastError()); - hEvent = make_inheritable(hEvent); - log_trace("osal_setup: event %" PRIuPTR " -> %p", i, hEvent); - events[i] = hEvent; - } - - hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL); - if (!hBarrierSemaphore) - failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError()); - hBarrierSemaphore = make_inheritable(hBarrierSemaphore); - - hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if (!hBarrierEvent) - failure_perror("CreateEvent(BarrierEvent)", GetLastError()); - hBarrierEvent = make_inheritable(hBarrierEvent); - - hProgressActiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL); - if (!hProgressActiveEvent) - failure_perror("CreateEvent(ProgressActiveEvent)", GetLastError()); - hProgressActiveEvent = make_inheritable(hProgressActiveEvent); - - hProgressPassiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL); - if (!hProgressPassiveEvent) - failure_perror("CreateEvent(ProgressPassiveEvent)", GetLastError()); - hProgressPassiveEvent = make_inheritable(hProgressPassiveEvent); -} - -void osal_broadcast(unsigned id) { - log_trace("osal_broadcast: event %u", id); - if (!SetEvent(events.at(id))) - failure_perror("SetEvent()", GetLastError()); -} - -int osal_waitfor(unsigned id) { - log_trace("osal_waitfor: event %u", id); - DWORD rc = WaitForSingleObject(events.at(id), INFINITE); - return waitstatus2errcode(rc); -} - -mdbx_pid_t osal_getpid(void) { return GetCurrentProcessId(); } - -int osal_delay(unsigned seconds) { - Sleep(seconds * 1000u); - return 0; -} - -//----------------------------------------------------------------------------- - -const std::string -actor_config::osal_serialize(simple_checksum &checksum) const { - checksum.push(hBarrierSemaphore); - checksum.push(hBarrierEvent); - checksum.push(hProgressActiveEvent); - checksum.push(hProgressPassiveEvent); - - HANDLE hWait = INVALID_HANDLE_VALUE; - if (wait4id) { - hWait = events.at(wait4id); - checksum.push(hWait); - } - - HANDLE hSignal = INVALID_HANDLE_VALUE; - if (wanna_event4signalling()) { - hSignal = events.at(actor_id); - checksum.push(hSignal); - } - - return format("%p.%p.%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait, - hSignal, hProgressActiveEvent, hProgressPassiveEvent); -} - -bool actor_config::osal_deserialize(const char *str, const char *end, - simple_checksum &checksum) { - - std::string copy(str, end - str); - TRACE(">> osal_deserialize(%s)\n", copy.c_str()); - - assert(hBarrierSemaphore == 0); - assert(hBarrierEvent == 0); - assert(hProgressActiveEvent == 0); - assert(hProgressPassiveEvent == 0); - assert(events.empty()); - - HANDLE hWait, hSignal; - if (sscanf_s(copy.c_str(), "%p.%p.%p.%p.%p.%p", &hBarrierSemaphore, - &hBarrierEvent, &hWait, &hSignal, &hProgressActiveEvent, - &hProgressPassiveEvent) != 6) { - TRACE("<< osal_deserialize: failed\n"); - return false; - } - - checksum.push(hBarrierSemaphore); - checksum.push(hBarrierEvent); - checksum.push(hProgressActiveEvent); - checksum.push(hProgressPassiveEvent); - - if (wait4id) { - checksum.push(hWait); - events[wait4id] = hWait; - } - - if (wanna_event4signalling()) { - checksum.push(hSignal); - events[actor_id] = hSignal; - } - - TRACE("<< osal_deserialize: OK\n"); - return true; -} - -//----------------------------------------------------------------------------- - -typedef std::pair<HANDLE, actor_status> child; -static std::unordered_map<mdbx_pid_t, child> childs; - -bool osal_progress_push(bool active) { - if (!childs.empty()) { - if (!SetEvent(active ? hProgressActiveEvent : hProgressPassiveEvent)) - failure_perror("osal_progress_push: SetEvent(overlord.progress)", - GetLastError()); - return true; - } - - return false; -} - -static void ArgvQuote(std::string &CommandLine, const std::string &Argument, - bool Force = false) - -/*++ - -https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ - -Routine Description: - - This routine appends the given argument to a command line such - that CommandLineToArgvW will return the argument string unchanged. - Arguments in a command line should be separated by spaces; this - function does not add these spaces. - -Arguments: - - Argument - Supplies the argument to encode. - - CommandLine - Supplies the command line to which we append the encoded -argument string. - - Force - Supplies an indication of whether we should quote - the argument even if it does not contain any characters that would - ordinarily require quoting. - -Return Value: - - None. - -Environment: - - Arbitrary. - ---*/ - -{ - // - // Unless we're told otherwise, don't quote unless we actually - // need to do so --- hopefully avoid problems if programs won't - // parse quotes properly - // - - if (Force == false && Argument.empty() == false && - Argument.find_first_of(" \t\n\v\"") == Argument.npos) { - CommandLine.append(Argument); - } else { - CommandLine.push_back('"'); - - for (auto It = Argument.begin();; ++It) { - unsigned NumberBackslashes = 0; - - while (It != Argument.end() && *It == '\\') { - ++It; - ++NumberBackslashes; - } - - if (It == Argument.end()) { - // - // Escape all backslashes, but let the terminating - // double quotation mark we add below be interpreted - // as a metacharacter. - // - CommandLine.append(NumberBackslashes * 2, '\\'); - break; - } else if (*It == L'"') { - // - // Escape all backslashes and the following - // double quotation mark. - // - CommandLine.append(NumberBackslashes * 2 + 1, '\\'); - CommandLine.push_back(*It); - } else { - // - // Backslashes aren't special here. - // - CommandLine.append(NumberBackslashes, '\\'); - CommandLine.push_back(*It); - } - } - - CommandLine.push_back('"'); - } -} - -int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { - if (childs.size() == MAXIMUM_WAIT_OBJECTS) - failure("Could't manage more that %u actors on Windows\n", - MAXIMUM_WAIT_OBJECTS); - - _flushall(); - - STARTUPINFOA StartupInfo; - GetStartupInfoA(&StartupInfo); - - char exename[_MAX_PATH + 1]; - DWORD exename_size = sizeof(exename); - if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename, - &exename_size)) - failure_perror("QueryFullProcessImageName()", GetLastError()); - - if (exename[1] != ':') { - exename_size = GetModuleFileName(NULL, exename, sizeof(exename)); - if (exename_size >= sizeof(exename)) - return ERROR_BAD_LENGTH; - } - - std::string cmdline = "$ "; - ArgvQuote(cmdline, thunk_param(config)); - - if (cmdline.size() >= 32767) - return ERROR_BAD_LENGTH; - - PROCESS_INFORMATION ProcessInformation; - if (!CreateProcessA(exename, const_cast<char *>(cmdline.c_str()), - NULL, // Retuned process handle is not inheritable. - NULL, // Retuned thread handle is not inheritable. - TRUE, // Child inherits all inheritable handles. - NORMAL_PRIORITY_CLASS | INHERIT_PARENT_AFFINITY, - NULL, // Inherit the parent's environment. - NULL, // Inherit the parent's current directory. - &StartupInfo, &ProcessInformation)) - failure_perror(exename, GetLastError()); - - CloseHandle(ProcessInformation.hThread); - pid = ProcessInformation.dwProcessId; - childs[pid] = std::make_pair(ProcessInformation.hProcess, as_running); - return 0; -} - -actor_status osal_actor_info(const mdbx_pid_t pid) { - actor_status status = childs.at(pid).second; - if (status > as_running) - return status; - - DWORD ExitCode; - if (!GetExitCodeProcess(childs.at(pid).first, &ExitCode)) - failure_perror("GetExitCodeProcess()", GetLastError()); - - switch (ExitCode) { - case STILL_ACTIVE: - return as_running; - case EXIT_SUCCESS: - status = as_successful; - break; - case EXCEPTION_BREAKPOINT: - case EXCEPTION_SINGLE_STEP: - status = as_debugging; - break; - case STATUS_CONTROL_C_EXIT: - status = as_killed; - break; - case EXCEPTION_ACCESS_VIOLATION: - case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: - case EXCEPTION_DATATYPE_MISALIGNMENT: - case EXCEPTION_STACK_OVERFLOW: - case EXCEPTION_INVALID_DISPOSITION: - case EXCEPTION_ILLEGAL_INSTRUCTION: - case EXCEPTION_NONCONTINUABLE_EXCEPTION: - status = as_coredump; - break; - default: - status = as_failed; - break; - } - - childs.at(pid).second = status; - return status; -} - -void osal_killall_actors(void) { - for (auto &pair : childs) - TerminateProcess(pair.second.first, STATUS_CONTROL_C_EXIT); -} - -int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { - std::vector<HANDLE> handles; - handles.reserve(childs.size() + 2); - handles.push_back(hProgressActiveEvent); - handles.push_back(hProgressPassiveEvent); - for (const auto &pair : childs) - if (pair.second.second <= as_running) - handles.push_back(pair.second.first); - - while (true) { - DWORD rc = - MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0], - (timeout > 60) ? 60 * 1000 : timeout * 1000, - QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0); - - if (rc == WAIT_OBJECT_0) { - logging::progress_canary(true); - continue; - } - if (rc == WAIT_OBJECT_0 + 1) { - logging::progress_canary(false); - continue; - } - - if (rc >= WAIT_OBJECT_0 + 2 && rc < WAIT_OBJECT_0 + handles.size()) { - pid = 0; - for (const auto &pair : childs) - if (pair.second.first == handles[rc - WAIT_OBJECT_0]) { - pid = pair.first; - break; - } - return 0; - } - - if (rc == WAIT_TIMEOUT) { - pid = 0; - return 0; - } - - return waitstatus2errcode(rc); - } -} - -void osal_yield(void) { SwitchToThread(); } - -void osal_udelay(unsigned us) { - chrono::time until, now = chrono::now_motonic(); - until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint; - - static unsigned threshold_us; - if (threshold_us == 0) { - unsigned timeslice_ms = 1; - while (timeBeginPeriod(timeslice_ms) == TIMERR_NOCANDO) - ++timeslice_ms; - threshold_us = timeslice_ms * 1500u; - assert(threshold_us > 0); - } - - do { - if (us > threshold_us && us > 1000) { - DWORD rc = SleepEx(us / 1000, TRUE); - if (rc) - failure_perror("SleepEx()", waitstatus2errcode(rc)); - us = 0; - } - - YieldProcessor(); - now = chrono::now_motonic(); - } while (now.fixedpoint < until.fixedpoint); -} - -bool osal_istty(int fd) { return _isatty(fd) != 0; } - -std::string osal_tempdir(void) { - char buf[MAX_PATH + 1]; - DWORD len = GetTempPathA(sizeof(buf), buf); - return std::string(buf, len); -} - -int osal_removefile(const std::string &pathname) { - return DeleteFileA(pathname.c_str()) ? MDBX_SUCCESS : GetLastError(); -} - -#endif /* Windows */ diff --git a/libs/libmdbx/src/test/osal.h b/libs/libmdbx/src/test/osal.h deleted file mode 100644 index e47450b931..0000000000 --- a/libs/libmdbx/src/test/osal.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -#include "base.h" - -void osal_setup(const std::vector<actor_config> &actors); -void osal_broadcast(unsigned id); -int osal_waitfor(unsigned id); - -int osal_actor_start(const actor_config &config, mdbx_pid_t &pid); -actor_status osal_actor_info(const mdbx_pid_t pid); -void osal_killall_actors(void); -int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout); -void osal_wait4barrier(void); - -bool osal_progress_push(bool active); - -mdbx_pid_t osal_getpid(void); -int osal_delay(unsigned seconds); -void osal_udelay(unsigned us); -void osal_yield(void); -bool osal_istty(int fd); -std::string osal_tempdir(void); -int osal_removefile(const std::string &pathname); - -#ifdef _MSC_VER -#ifndef STDIN_FILENO -#define STDIN_FILENO _fileno(stdin) -#endif -#ifndef STDOUT_FILENO -#define STDOUT_FILENO _fileno(stdout) -#endif -#ifndef STDERR_FILENO -#define STDERR_FILENO _fileno(stderr) -#endif -#endif /* _MSC_VER */ diff --git a/libs/libmdbx/src/test/pcrf/README.md b/libs/libmdbx/src/test/pcrf/README.md deleted file mode 100644 index b2c9b5ce95..0000000000 --- a/libs/libmdbx/src/test/pcrf/README.md +++ /dev/null @@ -1,2 +0,0 @@ -PCRF Session DB emulation test - diff --git a/libs/libmdbx/src/test/pcrf/pcrf_test.c b/libs/libmdbx/src/test/pcrf/pcrf_test.c deleted file mode 100644 index 2db58a023a..0000000000 --- a/libs/libmdbx/src/test/pcrf/pcrf_test.c +++ /dev/null @@ -1,413 +0,0 @@ -/* - * Copyright 2016-2020 Leonid Yuriev <leo@yuriev.ru>. - * Copyright 2015 Vladimir Romanov - * <https://www.linkedin.com/in/vladimirromanov>, Yota Lab. - * - * This file is part of libmdbx. - * - * ReOpenMDBX is free software; you can redistribute it and/or modify it under - * the terms of the GNU Affero General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * ReOpenMDBX is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <sys/stat.h> -#include <sys/time.h> - -#include "mdbx.h" -#include <assert.h> -#include <inttypes.h> -#include <limits.h> -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#define IP_PRINTF_ARG_HOST(addr) \ - (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \ - (int)((addr)&0xff) - -char opt_db_path[PATH_MAX] = "./mdbx_bench2"; -static MDBX_env *env; -#define REC_COUNT 10240000 -int64_t ids[REC_COUNT * 10]; -int32_t ids_count = 0; - -int64_t mdbx_add_count = 0; -int64_t mdbx_del_count = 0; -uint64_t mdbx_add_time = 0; -uint64_t mdbx_del_time = 0; -int64_t obj_id = 0; -int64_t mdbx_data_size = 0; -int64_t mdbx_key_size = 0; - -typedef struct { - char session_id1[100]; - char session_id2[100]; - char ip[20]; - uint8_t fill[100]; -} session_data_t; - -typedef struct { - int64_t obj_id; - int8_t event_type; -} __attribute__((__packed__)) event_data_t; - -static void add_id_to_pool(int64_t id) { - ids[ids_count] = id; - ids_count++; -} - -static inline int64_t getClockUs(void) { - struct timespec val; -#ifdef CYGWIN - clock_gettime(CLOCK_REALTIME, &val); -#else - clock_gettime(CLOCK_MONOTONIC, &val); -#endif - return val.tv_sec * ((int64_t)1000000) + val.tv_nsec / 1000; -} - -static int64_t get_id_from_pool() { - if (ids_count == 0) { - return -1; - } - int32_t index = rand() % ids_count; - int64_t id = ids[index]; - ids[index] = ids[ids_count - 1]; - ids_count--; - return id; -} - -#define MDBX_CHECK(x) \ - do { \ - const int rc = (x); \ - if (rc != MDBX_SUCCESS) { \ - printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \ - __FILE__, __LINE__); \ - exit(EXIT_FAILURE); \ - } \ - } while (0) - -static void db_connect() { - MDBX_dbi dbi_session; - MDBX_dbi dbi_session_id; - MDBX_dbi dbi_event; - MDBX_dbi dbi_ip; - - MDBX_CHECK(mdbx_env_create(&env)); - MDBX_CHECK(mdbx_env_set_geometry( - env, 0, 0, REC_COUNT * sizeof(session_data_t) * 10, -1, -1, -1)); - MDBX_CHECK(mdbx_env_set_maxdbs(env, 30)); - MDBX_CHECK(mdbx_env_open(env, opt_db_path, - MDBX_CREATE | MDBX_WRITEMAP | MDBX_MAPASYNC | - MDBX_SAFE_NOSYNC | MDBX_LIFORECLAIM, - 0664)); - MDBX_txn *txn; - - // transaction init - MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - // open database in read-write mode - MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session)); - MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id)); - MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event)); - MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip)); - // transaction commit - MDBX_CHECK(mdbx_txn_commit(txn)); - printf("Connection open\n"); -} - -static void create_record(uint64_t record_id) { - MDBX_dbi dbi_session; - MDBX_dbi dbi_session_id; - MDBX_dbi dbi_event; - MDBX_dbi dbi_ip; - event_data_t event; - MDBX_txn *txn; - session_data_t data; - // transaction init - snprintf(data.session_id1, sizeof(data.session_id1), - "prefix%02u_%02u.fill.fill.fill.fill.fill.fill;%" PRIu64, - (unsigned)(record_id % 3) + 1, (unsigned)(record_id % 9) + 1, - record_id); - snprintf(data.session_id2, sizeof(data.session_id2), - "dprefix%" PRIu64 ";%" PRIu64 ".fill.fill.;suffix", record_id, - (record_id + UINT64_C(1442695040888963407)) % - UINT64_C(6364136223846793005)); - snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d", - IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); - event.obj_id = record_id; - event.event_type = 1; - - MDBX_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)}; - MDBX_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)}; - MDBX_val _ip_rec = {data.ip, strlen(data.ip)}; - MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)}; - MDBX_val _data_rec = {&data, offsetof(session_data_t, fill) + - (rand() % sizeof(data.fill))}; - MDBX_val _event_rec = {&event, sizeof(event)}; - - uint64_t start = getClockUs(); - MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session)); - MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id)); - MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event)); - MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip)); - MDBX_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec, - MDBX_NOOVERWRITE | MDBX_NODUPDATA)); - MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, - MDBX_NOOVERWRITE | MDBX_NODUPDATA)); - MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, - MDBX_NOOVERWRITE | MDBX_NODUPDATA)); - MDBX_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0)); - MDBX_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0)); - MDBX_CHECK(mdbx_txn_commit(txn)); - - mdbx_data_size += (_data_rec.iov_len + _obj_id_rec.iov_len * 4); - mdbx_key_size += - (_obj_id_rec.iov_len + _session_id1_rec.iov_len + - _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len); - - // transaction commit - mdbx_add_count++; - mdbx_add_time += (getClockUs() - start); -} - -static void delete_record(int64_t record_id) { - MDBX_dbi dbi_session; - MDBX_dbi dbi_session_id; - MDBX_dbi dbi_event; - MDBX_dbi dbi_ip; - event_data_t event; - MDBX_txn *txn; - - // transaction init - uint64_t start = getClockUs(); - MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - // open database in read-write mode - MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session)); - MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id)); - MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event)); - MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip)); - // put data - MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)}; - MDBX_val _data_rec; - // get data - MDBX_CHECK(mdbx_get(txn, dbi_session, &_obj_id_rec, &_data_rec)); - session_data_t *data = (session_data_t *)_data_rec.iov_base; - - MDBX_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)}; - MDBX_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)}; - MDBX_val _ip_rec = {data->ip, strlen(data->ip)}; - MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id1_rec, NULL)); - MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id2_rec, NULL)); - MDBX_CHECK(mdbx_del(txn, dbi_ip, &_ip_rec, NULL)); - event.obj_id = record_id; - event.event_type = 1; - MDBX_val _event_rec = {&event, sizeof(event)}; - MDBX_CHECK(mdbx_del(txn, dbi_event, &_event_rec, NULL)); - MDBX_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL)); - - mdbx_data_size -= (_data_rec.iov_len + _obj_id_rec.iov_len * 4); - mdbx_key_size -= - (_obj_id_rec.iov_len + _session_id1_rec.iov_len + - _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len); - - // transaction commit - MDBX_CHECK(mdbx_txn_commit(txn)); - mdbx_del_count++; - mdbx_del_time += (getClockUs() - start); -} - -static void db_disconnect() { - mdbx_env_close(env); - printf("Connection closed\n"); -} - -static void get_db_stat(const char *db, int64_t *ms_branch_pages, - int64_t *ms_leaf_pages) { - MDBX_txn *txn; - MDBX_stat stat; - MDBX_dbi dbi; - - MDBX_CHECK(mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn)); - MDBX_CHECK(mdbx_dbi_open(txn, db, MDBX_CREATE, &dbi)); - MDBX_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat))); - mdbx_txn_abort(txn); - printf("%15s | %15" PRIu64 " | %5u | %10" PRIu64 " | %10" PRIu64 - " | %11" PRIu64 " |\n", - db, stat.ms_branch_pages, stat.ms_depth, stat.ms_entries, - stat.ms_leaf_pages, stat.ms_overflow_pages); - (*ms_branch_pages) += stat.ms_branch_pages; - (*ms_leaf_pages) += stat.ms_leaf_pages; -} - -static void periodic_stat(void) { - int64_t ms_branch_pages = 0; - int64_t ms_leaf_pages = 0; - MDBX_stat mst; - MDBX_envinfo mei; - MDBX_CHECK(mdbx_env_stat_ex(env, NULL, &mst, sizeof(mst))); - MDBX_CHECK(mdbx_env_info_ex(env, NULL, &mei, sizeof(mei))); - printf("Environment Info\n"); - printf(" Pagesize: %u\n", mst.ms_psize); - if (mei.mi_geo.lower != mei.mi_geo.upper) { - printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 - "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 - "/-%" PRIu64 ")\n", - mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, - mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize, - mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize, - mei.mi_geo.shrink / mst.ms_psize); - printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); - } else { - printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); - } - printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", - mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize); - printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1); - printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); - printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n", - mei.mi_latter_reader_txnid, - mei.mi_latter_reader_txnid - mei.mi_recent_txnid); - printf(" Max readers: %u\n", mei.mi_maxreaders); - printf(" Number of readers used: %u\n", mei.mi_numreaders); - - printf(" Name | ms_branch_pages | depth | entries | leaf_pages " - "| overf_pages |\n"); - get_db_stat("session", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); - printf("%15s | %15" PRIu64 " | %5s | %10s | %10" PRIu64 " | %11s |\n", "", - ms_branch_pages, "", "", ms_leaf_pages, ""); - - static int64_t prev_add_count; - static int64_t prev_del_count; - static uint64_t prev_add_time; - static uint64_t prev_del_time; - static int64_t t = -1; - if (t > 0) { - int64_t delta = (getClockUs() - t); - printf("CPS: add %" PRIu64 ", delete %" PRIu64 - ", items processed - %" PRIu64 "K data=%" PRIu64 "K key=%" PRIu64 - "K\n", - (mdbx_add_count - prev_add_count) * 1000000 / delta, - (mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024, - mdbx_data_size / 1024, mdbx_key_size / 1024); - printf("usage data=%" PRIu64 "%%", - ((mdbx_data_size + mdbx_key_size) * 100) / - ((ms_leaf_pages + ms_branch_pages) * 4096)); - if (prev_add_time != mdbx_add_time) { - printf(" Add : %" PRIu64 " c/s", (mdbx_add_count - prev_add_count) * - 1000000 / - (mdbx_add_time - prev_add_time)); - } - if (prev_del_time != mdbx_del_time) { - printf(" Del : %" PRIu64 " c/s", (mdbx_del_count - prev_del_count) * - 1000000 / - (mdbx_del_time - prev_del_time)); - } - if (mdbx_add_time) { - printf(" tAdd : %" PRIu64 " c/s", - mdbx_add_count * 1000000 / mdbx_add_time); - } - if (mdbx_del_time) { - printf(" tDel : %" PRIu64 " c/s", - mdbx_del_count * 1000000 / mdbx_del_time); - } - puts(""); - } - t = getClockUs(); - prev_add_count = mdbx_add_count; - prev_del_count = mdbx_del_count; - prev_add_time = mdbx_add_time; - prev_del_time = mdbx_del_time; -} - -#if 0 /* unused */ -static void periodic_add_rec() { - for (int i = 0; i < 10240; i++) { - if (ids_count <= REC_COUNT) { - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - } - if (ids_count > REC_COUNT) { - int64_t id = get_id_from_pool(); - delete_record(id); - } - } - periodic_stat(); -} -#endif - -int main(int argc, char **argv) { - (void)argc; - (void)argv; - - char filename[PATH_MAX]; - int i; - - strcpy(filename, opt_db_path); - strcat(filename, MDBX_DATANAME); - remove(filename); - - strcpy(filename, opt_db_path); - strcat(filename, MDBX_LOCKNAME); - remove(filename); - - puts("Open DB..."); - db_connect(); - puts("Create data..."); - int64_t t = getClockUs(); - for (i = 0; i < REC_COUNT; i++) { - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - if (i % 1000 == 0) { - int64_t now = getClockUs(); - if ((now - t) > 1000000L) { - periodic_stat(); - t = now; - } - } - } - periodic_stat(); - while (1) { - int i; - for (i = 0; i < 1000; i++) { - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - id = get_id_from_pool(); - delete_record(id); - } - for (i = 0; i < 50; i++) { - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - } - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - int64_t now = getClockUs(); - if ((now - t) > 10000000L) { - periodic_stat(); - t = now; - } - } - db_disconnect(); - return 0; -} diff --git a/libs/libmdbx/src/test/test.cc b/libs/libmdbx/src/test/test.cc deleted file mode 100644 index 9af04ac9b0..0000000000 --- a/libs/libmdbx/src/test/test.cc +++ /dev/null @@ -1,755 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" - -const char *testcase2str(const actor_testcase testcase) { - switch (testcase) { - default: - assert(false); - return "?!"; - case ac_none: - return "none"; - case ac_hill: - return "hill"; - case ac_deadread: - return "deadread"; - case ac_deadwrite: - return "deadwrite"; - case ac_jitter: - return "jitter"; - case ac_try: - return "try"; - case ac_copy: - return "copy"; - case ac_append: - return "append"; - case ac_ttl: - return "ttl"; - case ac_nested: - return "nested"; - } -} - -const char *status2str(actor_status status) { - switch (status) { - default: - assert(false); - return "?!"; - case as_debugging: - return "debugging"; - case as_running: - return "running"; - case as_successful: - return "successful"; - case as_killed: - return "killed"; - case as_failed: - return "failed"; - case as_coredump: - return "coredump"; - } -} - -const char *keygencase2str(const keygen_case keycase) { - switch (keycase) { - default: - assert(false); - return "?!"; - case kc_random: - return "random"; - case kc_dashes: - return "dashes"; - case kc_custom: - return "custom"; - } -} - -//----------------------------------------------------------------------------- - -int testcase::oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, - uint64_t txn, unsigned gap, size_t space, - int retry) { - - testcase *self = (testcase *)mdbx_env_get_userctx(env); - - if (retry == 0) - log_notice("oom_callback: waitfor pid %lu, thread %" PRIuPTR - ", txn #%" PRIu64 ", gap %d, scape %zu", - (long)pid, (size_t)tid, txn, gap, space); - - if (self->should_continue(true)) { - osal_yield(); - if (retry > 0) - osal_udelay(retry * 100); - return 0 /* always retry */; - } - - return -1; -} - -void testcase::db_prepare() { - log_trace(">> db_prepare"); - assert(!db_guard); - - MDBX_env *env = nullptr; - int rc = mdbx_env_create(&env); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_env_create()", rc); - - assert(env != nullptr); - db_guard.reset(env); - - rc = mdbx_env_set_userctx(env, this); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_env_set_userctx()", rc); - - rc = mdbx_env_set_maxreaders(env, config.params.max_readers); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_env_set_maxreaders()", rc); - - rc = mdbx_env_set_maxdbs(env, config.params.max_tables); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_env_set_maxdbs()", rc); - - rc = mdbx_env_set_oomfunc(env, testcase::oom_callback); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_env_set_oomfunc()", rc); - - rc = mdbx_env_set_geometry( - env, config.params.size_lower, config.params.size_now, - config.params.size_upper, config.params.growth_step, - config.params.shrink_threshold, config.params.pagesize); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_env_set_mapsize()", rc); - - log_trace("<< db_prepare"); -} - -void testcase::db_open() { - log_trace(">> db_open"); - - if (!db_guard) - db_prepare(); - - jitter_delay(true); - - unsigned mode = (unsigned)config.params.mode_flags; - if (config.params.random_writemap && flipcoin()) - mode ^= MDBX_WRITEMAP; - - actual_db_mode = mode; - int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(), - mode, 0640); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_env_open()", rc); - - log_trace("<< db_open"); -} - -void testcase::db_close() { - log_trace(">> db_close"); - cursor_guard.reset(); - txn_guard.reset(); - db_guard.reset(); - log_trace("<< db_close"); -} - -void testcase::txn_begin(bool readonly, unsigned flags) { - assert((flags & MDBX_RDONLY) == 0); - log_trace(">> txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", - flags); - assert(!txn_guard); - - MDBX_txn *txn = nullptr; - int rc = mdbx_txn_begin(db_guard.get(), nullptr, - readonly ? flags | MDBX_RDONLY : flags, &txn); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_txn_begin()", rc); - txn_guard.reset(txn); - need_speculum_assign = config.params.speculum && !readonly; - - log_trace("<< txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", - flags); -} - -int testcase::breakable_commit() { - log_trace(">> txn_commit"); - assert(txn_guard); - - MDBX_txn *txn = txn_guard.release(); - txn_inject_writefault(txn); - int rc = mdbx_txn_commit(txn); - if (unlikely(rc != MDBX_SUCCESS) && - (rc != MDBX_MAP_FULL || !config.params.ignore_dbfull)) - failure_perror("mdbx_txn_commit()", rc); - - if (need_speculum_assign) { - need_speculum_assign = false; - if (unlikely(rc != MDBX_SUCCESS)) - speculum = speculum_commited; - else - speculum_commited = speculum; - } - - log_trace("<< txn_commit: %s", rc ? "failed" : "Ok"); - return rc; -} - -unsigned testcase::txn_underutilization_x256(MDBX_txn *txn) const { - if (txn) { - MDBX_txn_info info; - int err = mdbx_txn_info(txn, &info, false); - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_txn_info()", err); - const size_t left = size_t(info.txn_space_leftover); - const size_t total = - size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty); - return (unsigned)(left / (total >> 8)); - } - return 0; -} - -void testcase::txn_end(bool abort) { - log_trace(">> txn_end(%s)", abort ? "abort" : "commit"); - assert(txn_guard); - - MDBX_txn *txn = txn_guard.release(); - if (abort) { - int err = mdbx_txn_abort(txn); - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_txn_abort()", err); - if (need_speculum_assign) - speculum = speculum_commited; - } else { - txn_inject_writefault(txn); - int err = mdbx_txn_commit(txn); - if (unlikely(err != MDBX_SUCCESS)) - failure_perror("mdbx_txn_commit()", err); - if (need_speculum_assign) - speculum_commited = speculum; - } - - log_trace("<< txn_end(%s)", abort ? "abort" : "commit"); -} - -void testcase::cursor_open(MDBX_dbi handle) { - log_trace(">> cursor_open(%u)", handle); - assert(!cursor_guard); - assert(txn_guard); - - MDBX_cursor *cursor = nullptr; - int rc = mdbx_cursor_open(txn_guard.get(), handle, &cursor); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_cursor_open()", rc); - cursor_guard.reset(cursor); - - log_trace("<< cursor_open(%u)", handle); -} - -void testcase::cursor_close() { - log_trace(">> cursor_close()"); - assert(cursor_guard); - MDBX_cursor *cursor = cursor_guard.release(); - mdbx_cursor_close(cursor); - log_trace("<< cursor_close()"); -} - -int testcase::breakable_restart() { - int rc = MDBX_SUCCESS; - if (txn_guard) - rc = breakable_commit(); - if (cursor_guard) - cursor_close(); - txn_begin(false, 0); - return rc; -} - -void testcase::txn_restart(bool abort, bool readonly, unsigned flags) { - if (txn_guard) - txn_end(abort); - if (cursor_guard) - cursor_close(); - txn_begin(readonly, flags); -} - -void testcase::txn_inject_writefault(void) { - if (txn_guard) - txn_inject_writefault(txn_guard.get()); -} - -void testcase::txn_inject_writefault(MDBX_txn *txn) { - if (config.params.inject_writefaultn && txn) { - if (config.params.inject_writefaultn <= nops_completed && - (mdbx_txn_flags(txn) & MDBX_RDONLY) == 0) { - log_verbose( - "== txn_inject_writefault(): got %u nops or more, inject FAULT", - config.params.inject_writefaultn); - log_flush(); -#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) - TerminateProcess(GetCurrentProcess(), 42); -#else - raise(SIGKILL); -#endif - } - } -} - -bool testcase::wait4start() { - if (config.wait4id) { - log_trace(">> wait4start(%u)", config.wait4id); - assert(!global::singlemode); - int rc = osal_waitfor(config.wait4id); - if (rc) { - log_trace("<< wait4start(%u), failed %s", config.wait4id, - test_strerror(rc)); - return false; - } - } else { - log_trace("== skip wait4start: not needed"); - } - - if (config.params.delaystart) { - int rc = osal_delay(config.params.delaystart); - if (rc) { - log_trace("<< delay(%u), failed %s", config.params.delaystart, - test_strerror(rc)); - return false; - } - } else { - log_trace("== skip delay: not needed"); - } - - return true; -} - -void testcase::kick_progress(bool active) const { - if (!global::config::progress_indicator) - return; - logging::progress_canary(active); -} - -void testcase::report(size_t nops_done) { - assert(nops_done > 0); - if (!nops_done) - return; - - nops_completed += nops_done; - log_debug("== complete +%" PRIuPTR " iteration, total %" PRIu64 " done", - nops_done, nops_completed); - - kick_progress(true); - - if (config.signal_nops && !signalled && - config.signal_nops <= nops_completed) { - log_trace(">> signal(n-ops %" PRIu64 ")", nops_completed); - if (!global::singlemode) - osal_broadcast(config.actor_id); - signalled = true; - log_trace("<< signal(n-ops %" PRIu64 ")", nops_completed); - } -} - -void testcase::signal() { - if (!signalled) { - log_trace(">> signal(forced)"); - if (!global::singlemode) - osal_broadcast(config.actor_id); - signalled = true; - log_trace("<< signal(forced)"); - } -} - -bool testcase::setup() { - db_prepare(); - if (!wait4start()) - return false; - - start_timestamp = chrono::now_motonic(); - nops_completed = 0; - return true; -} - -bool testcase::teardown() { - log_trace(">> testcase::teardown"); - signal(); - db_close(); - log_trace("<< testcase::teardown"); - return true; -} - -bool testcase::should_continue(bool check_timeout_only) const { - bool result = true; - - if (config.params.test_duration) { - chrono::time since; - since.fixedpoint = - chrono::now_motonic().fixedpoint - start_timestamp.fixedpoint; - if (since.seconds() >= config.params.test_duration) - result = false; - } - - if (!check_timeout_only && config.params.test_nops && - nops_completed >= config.params.test_nops) - result = false; - - if (result) - kick_progress(false); - - return result; -} - -void testcase::fetch_canary() { - mdbx_canary canary_now; - log_trace(">> fetch_canary"); - - int rc = mdbx_canary_get(txn_guard.get(), &canary_now); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_canary_get()", rc); - - if (canary_now.v < last.canary.v) - failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64 - "(canary-last.v)", - canary_now.v, last.canary.v); - if (canary_now.y < last.canary.y) - failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64 - "(canary-last.y)", - canary_now.y, last.canary.y); - - last.canary = canary_now; - log_trace("<< fetch_canary: db-sequence %" PRIu64 - ", db-sequence.txnid %" PRIu64, - last.canary.y, last.canary.v); -} - -void testcase::update_canary(uint64_t increment) { - mdbx_canary canary_now = last.canary; - - log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y, - increment); - canary_now.y += increment; - - int rc = mdbx_canary_put(txn_guard.get(), &canary_now); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_canary_put()", rc); - - log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y); -} - -int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &handle) { - db_open(); - - int err, retry_left = 42; - for (;;) { - txn_begin(false); - handle = db_table_open(true); - db_table_clear(handle); - err = breakable_commit(); - if (likely(err == MDBX_SUCCESS)) { - txn_begin(false); - return MDBX_SUCCESS; - } - if (--retry_left == 0) - break; - jitter_delay(true); - } - log_notice("db_begin_table_create_open_clean: bailout due '%s'", - mdbx_strerror(err)); - return err; -} - -MDBX_dbi testcase::db_table_open(bool create) { - log_trace(">> testcase::db_table_create"); - - char tablename_buf[16]; - const char *tablename = nullptr; - if (config.space_id) { - int rc = snprintf(tablename_buf, sizeof(tablename_buf), "TBL%04u", - config.space_id); - if (rc < 4 || rc >= (int)sizeof(tablename_buf) - 1) - failure("snprintf(tablename): %d", rc); - tablename = tablename_buf; - } - log_debug("use %s table", tablename ? tablename : "MAINDB"); - - MDBX_dbi handle = 0; - int rc = mdbx_dbi_open(txn_guard.get(), tablename, - (create ? MDBX_CREATE : 0) | config.params.table_flags, - &handle); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_dbi_open()", rc); - - log_trace("<< testcase::db_table_create, handle %u", handle); - return handle; -} - -void testcase::db_table_drop(MDBX_dbi handle) { - log_trace(">> testcase::db_table_drop, handle %u", handle); - - if (config.params.drop_table) { - int rc = mdbx_drop(txn_guard.get(), handle, true); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_drop(delete=true)", rc); - speculum.clear(); - log_trace("<< testcase::db_table_drop"); - } else { - log_trace("<< testcase::db_table_drop: not needed"); - } -} - -void testcase::db_table_clear(MDBX_dbi handle, MDBX_txn *txn) { - log_trace(">> testcase::db_table_clear, handle %u", handle); - int rc = mdbx_drop(txn ? txn : txn_guard.get(), handle, false); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_drop(delete=false)", rc); - speculum.clear(); - log_trace("<< testcase::db_table_clear"); -} - -void testcase::db_table_close(MDBX_dbi handle) { - log_trace(">> testcase::db_table_close, handle %u", handle); - assert(!txn_guard); - int rc = mdbx_dbi_close(db_guard.get(), handle); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_dbi_close()", rc); - log_trace("<< testcase::db_table_close"); -} - -void testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, - MDBX_val expected_valued) { - MDBX_val actual_value = expected_valued; - int rc = mdbx_get_nearest(txn_guard.get(), handle, &key2check, &actual_value); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror(step, rc); - if (!is_samedata(&actual_value, &expected_valued)) - failure("%s data mismatch", step); -} - -//----------------------------------------------------------------------------- - -bool test_execute(const actor_config &config_const) { - const mdbx_pid_t pid = osal_getpid(); - actor_config config = config_const; - - if (global::singlemode) { - logging::setup(format("single_%s", testcase2str(config.testcase))); - } else { - logging::setup((logging::loglevel)config.params.loglevel, - format("child_%u.%u", config.actor_id, config.space_id)); - log_trace(">> wait4barrier"); - osal_wait4barrier(); - log_trace("<< wait4barrier"); - } - - try { - std::unique_ptr<testcase> test; - switch (config.testcase) { - case ac_hill: - test.reset(new testcase_hill(config, pid)); - break; - case ac_deadread: - test.reset(new testcase_deadread(config, pid)); - break; - case ac_deadwrite: - test.reset(new testcase_deadwrite(config, pid)); - break; - case ac_jitter: - test.reset(new testcase_jitter(config, pid)); - break; - case ac_try: - test.reset(new testcase_try(config, pid)); - break; - case ac_copy: - test.reset(new testcase_copy(config, pid)); - break; - case ac_append: - test.reset(new testcase_append(config, pid)); - break; - case ac_ttl: - test.reset(new testcase_ttl(config, pid)); - break; - case ac_nested: - test.reset(new testcase_nested(config, pid)); - break; - default: - test.reset(new testcase(config, pid)); - break; - } - - size_t iter = 0; - do { - iter++; - if (!test->setup()) { - log_notice("test setup failed"); - return false; - } - if (!test->run()) { - log_notice("test failed"); - return false; - } - if (!test->teardown()) { - log_notice("test teardown failed"); - return false; - } - - if (config.params.nrepeat == 1) - log_verbose("test successed"); - else { - if (config.params.nrepeat) - log_verbose("test successed (iteration %zi of %zi)", iter, - size_t(config.params.nrepeat)); - else - log_verbose("test successed (iteration %zi)", iter); - config.params.keygen.seed += INT32_C(0xA4F4D37B); - } - - } while (config.params.nrepeat == 0 || iter < config.params.nrepeat); - return true; - } catch (const std::exception &pipets) { - failure("***** Exception: %s *****", pipets.what()); - return false; - } -} - -//----------------------------------------------------------------------------- - -int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, - unsigned flags) { - int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags); - if (err == MDBX_SUCCESS && config.params.speculum) { - const auto S_key = S(akey); - const auto S_data = S(adata); - const bool inserted = speculum.emplace(S_key, S_data).second; - assert(inserted); - (void)inserted; - } - return err; -} - -int testcase::replace(const keygen::buffer &akey, - const keygen::buffer &new_data, - const keygen::buffer &old_data, unsigned flags) { - if (config.params.speculum) { - const auto S_key = S(akey); - const auto S_old = S(old_data); - const auto S_new = S(new_data); - const auto removed = speculum.erase(SET::key_type(S_key, S_old)); - assert(removed == 1); - (void)removed; - const bool inserted = speculum.emplace(S_key, S_new).second; - assert(inserted); - (void)inserted; - } - return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value, - &old_data->value, flags); -} - -int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) { - if (config.params.speculum) { - const auto S_key = S(akey); - const auto S_data = S(adata); - const auto removed = speculum.erase(SET::key_type(S_key, S_data)); - assert(removed == 1); - (void)removed; - } - return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value); -} - -bool testcase::speculum_verify() { - if (!config.params.speculum) - return true; - - if (!txn_guard) - txn_begin(true); - - char dump_key[128], dump_value[128]; - char dump_mkey[128], dump_mvalue[128]; - - MDBX_cursor *cursor; - int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); - if (err != MDBX_SUCCESS) - failure_perror("mdbx_cursor_open()", err); - - bool rc = true; - MDBX_val akey, avalue; - MDBX_val mkey, mvalue; - err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST); - - unsigned extra = 0, lost = 0, n = 0; - assert(std::is_sorted(speculum.cbegin(), speculum.cend(), ItemCompare(this))); - auto it = speculum.cbegin(); - while (true) { - if (err != MDBX_SUCCESS) { - akey.iov_len = avalue.iov_len = 0; - akey.iov_base = avalue.iov_base = nullptr; - } - const auto S_key = S(akey); - const auto S_data = S(avalue); - if (it != speculum.cend()) { - mkey.iov_base = (void *)it->first.c_str(); - mkey.iov_len = it->first.size(); - mvalue.iov_base = (void *)it->second.c_str(); - mvalue.iov_len = it->second.size(); - } - if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first && - S_data == it->second) { - ++it; - err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); - } else if (err == MDBX_SUCCESS && - (it == speculum.cend() || S_key < it->first || - (S_key == it->first && S_data < it->second))) { - extra += 1; - if (it != speculum.cend()) { - log_error("extra pair %u/%u: db{%s, %s} < mi{%s, %s}", n, extra, - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), - mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), - mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); - } else { - log_error("extra pair %u/%u: db{%s, %s} < mi.END", n, extra, - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), - mdbx_dump_val(&avalue, dump_value, sizeof(dump_value))); - } - err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); - rc = false; - } else if (it != speculum.cend() && - (err == MDBX_NOTFOUND || S_key > it->first || - (S_key == it->first && S_data > it->second))) { - lost += 1; - if (err == MDBX_NOTFOUND) { - log_error("lost pair %u/%u: db.END > mi{%s, %s}", n, lost, - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), - mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); - } else { - log_error("lost pair %u/%u: db{%s, %s} > mi{%s, %s}", n, lost, - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), - mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), - mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); - } - ++it; - rc = false; - } else if (err == MDBX_NOTFOUND && it == speculum.cend()) { - break; - } else if (err != MDBX_SUCCESS) { - failure_perror("mdbx_cursor_get()", err); - } else { - assert(!"WTF?"); - } - n += 1; - } - - mdbx_cursor_close(cursor); - return rc; -} diff --git a/libs/libmdbx/src/test/test.h b/libs/libmdbx/src/test/test.h deleted file mode 100644 index 43c6c03848..0000000000 --- a/libs/libmdbx/src/test/test.h +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once - -#include "base.h" -#include "chrono.h" -#include "config.h" -#include "keygen.h" -#include "log.h" -#include "osal.h" -#include "utils.h" - -#include <deque> -#include <set> -#include <stack> -#include <tuple> - -#ifndef HAVE_cxx17_std_string_view -#if __cplusplus >= 201703L && __has_include(<string_view>) -#include <string_view> -#define HAVE_cxx17_std_string_view 1 -#else -#define HAVE_cxx17_std_string_view 0 -#endif -#endif /* HAVE_cxx17_std_string_view */ - -#if HAVE_cxx17_std_string_view -#include <string_view> -#endif - -bool test_execute(const actor_config &config); -std::string thunk_param(const actor_config &config); -void testcase_setup(const char *casename, actor_params ¶ms, - unsigned &last_space_id); -void configure_actor(unsigned &last_space_id, const actor_testcase testcase, - const char *space_id_cstr, const actor_params ¶ms); -void keycase_setup(const char *casename, actor_params ¶ms); - -namespace global { - -extern const char thunk_param_prefix[]; -extern std::vector<actor_config> actors; -extern std::unordered_map<unsigned, actor_config *> events; -extern std::unordered_map<mdbx_pid_t, actor_config *> pid2actor; -extern std::set<std::string> databases; -extern unsigned nactors; -extern chrono::time start_motonic; -extern chrono::time deadline_motonic; -extern bool singlemode; - -namespace config { -extern unsigned timeout_duration_seconds; -extern bool dump_config; -extern bool cleanup_before; -extern bool cleanup_after; -extern bool failfast; -extern bool progress_indicator; -extern bool console_mode; -} /* namespace config */ - -} /* namespace global */ - -//----------------------------------------------------------------------------- - -struct db_deleter /* : public std::unary_function<void, MDBX_env *> */ { - void operator()(MDBX_env *env) const { mdbx_env_close(env); } -}; - -struct txn_deleter /* : public std::unary_function<void, MDBX_txn *> */ { - void operator()(MDBX_txn *txn) const { - int rc = mdbx_txn_abort(txn); - if (rc) - log_trouble(__func__, "mdbx_txn_abort()", rc); - } -}; - -struct cursor_deleter /* : public std::unary_function<void, MDBX_cursor *> */ { - void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); } -}; - -typedef std::unique_ptr<MDBX_env, db_deleter> scoped_db_guard; -typedef std::unique_ptr<MDBX_txn, txn_deleter> scoped_txn_guard; -typedef std::unique_ptr<MDBX_cursor, cursor_deleter> scoped_cursor_guard; - -//----------------------------------------------------------------------------- - -class testcase { -protected: -#if HAVE_cxx17_std_string_view - using data_view = std::string_view; -#else - using data_view = std::string; -#endif - static inline data_view S(const MDBX_val &v) { - return (v.iov_base && v.iov_len) - ? data_view(static_cast<const char *>(v.iov_base), v.iov_len) - : data_view(); - } - static inline data_view S(const keygen::buffer &b) { return S(b->value); } - - using Item = std::pair<std::string, std::string>; - struct ItemCompare { - const testcase *context; - ItemCompare(const testcase *owner) : context(owner) {} - - bool operator()(const Item &a, const Item &b) const { - MDBX_val va, vb; - va.iov_base = (void *)a.first.data(); - va.iov_len = a.first.size(); - vb.iov_base = (void *)b.first.data(); - vb.iov_len = b.first.size(); - int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb); - if (cmp == 0 && - (context->config.params.table_flags & MDBX_DUPSORT) != 0) { - va.iov_base = (void *)a.second.data(); - va.iov_len = a.second.size(); - vb.iov_base = (void *)b.second.data(); - vb.iov_len = b.second.size(); - cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb); - } - return cmp < 0; - } - }; - - // for simplify the set<pair<key,value>> - // is used instead of multimap<key,value> - using SET = std::set<Item, ItemCompare>; - - const actor_config &config; - const mdbx_pid_t pid; - - MDBX_dbi dbi{0}; - scoped_db_guard db_guard; - scoped_txn_guard txn_guard; - scoped_cursor_guard cursor_guard; - bool signalled{false}; - bool need_speculum_assign{false}; - - uint64_t nops_completed{0}; - chrono::time start_timestamp; - keygen::buffer key; - keygen::buffer data; - keygen::maker keyvalue_maker; - - struct { - mdbx_canary canary; - } last; - - SET speculum{ItemCompare(this)}, speculum_commited{ItemCompare(this)}; - bool speculum_verify(); - int insert(const keygen::buffer &akey, const keygen::buffer &adata, - unsigned flags); - int replace(const keygen::buffer &akey, const keygen::buffer &new_value, - const keygen::buffer &old_value, unsigned flags); - int remove(const keygen::buffer &akey, const keygen::buffer &adata); - - static int oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, - uint64_t txn, unsigned gap, size_t space, int retry); - - unsigned actual_db_mode{0}; - bool is_nested_txn_available() const { - return (actual_db_mode & MDBX_WRITEMAP) == 0; - } - void kick_progress(bool active) const; - void db_prepare(); - void db_open(); - void db_close(); - void txn_begin(bool readonly, unsigned flags = 0); - int breakable_commit(); - void txn_end(bool abort); - int breakable_restart(); - void txn_restart(bool abort, bool readonly, unsigned flags = 0); - void cursor_open(MDBX_dbi handle); - void cursor_close(); - void txn_inject_writefault(void); - void txn_inject_writefault(MDBX_txn *txn); - void fetch_canary(); - void update_canary(uint64_t increment); - void checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, - MDBX_val expected_valued); - unsigned txn_underutilization_x256(MDBX_txn *txn) const; - - MDBX_dbi db_table_open(bool create); - void db_table_drop(MDBX_dbi handle); - void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr); - void db_table_close(MDBX_dbi handle); - int db_open__begin__table_create_open_clean(MDBX_dbi &handle); - - bool wait4start(); - void report(size_t nops_done); - void signal(); - bool should_continue(bool check_timeout_only = false) const; - - void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key, - keygen::buffer &out_value, keygen::serial_t data_age) { - keyvalue_maker.pair(serial, out_key, out_value, data_age, false); - } - - void generate_pair(const keygen::serial_t serial) { - keyvalue_maker.pair(serial, key, data, 0, true); - } - - bool mode_readonly() const { - return (config.params.mode_flags & MDBX_RDONLY) ? true : false; - } - -public: - testcase(const actor_config &config, const mdbx_pid_t pid) - : config(config), pid(pid) { - start_timestamp.reset(); - memset(&last, 0, sizeof(last)); - } - - virtual bool setup(); - virtual bool run() { return true; } - virtual bool teardown(); - virtual ~testcase() {} -}; - -class testcase_hill : public testcase { -public: - testcase_hill(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} - bool run() override; -}; - -class testcase_append : public testcase { -public: - testcase_append(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} - bool run() override; -}; - -class testcase_deadread : public testcase { -public: - testcase_deadread(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} - bool run() override; -}; - -class testcase_deadwrite : public testcase { -public: - testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} - bool run() override; -}; - -class testcase_jitter : public testcase { -public: - testcase_jitter(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} - bool run() override; -}; - -class testcase_try : public testcase { -public: - testcase_try(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} - bool run() override; -}; - -class testcase_copy : public testcase { - const std::string copy_pathname; - void copy_db(const bool with_compaction); - -public: - testcase_copy(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid), - copy_pathname(config.params.pathname_db + "-copy") {} - bool run() override; -}; - -class testcase_ttl : public testcase { - using inherited = testcase; - -protected: - struct { - unsigned max_window_size{0}; - unsigned max_step_size{0}; - } sliding; - unsigned edge2window(uint64_t edge); - unsigned edge2count(uint64_t edge); - -public: - testcase_ttl(const actor_config &config, const mdbx_pid_t pid) - : inherited(config, pid) {} - bool setup() override; - bool run() override; -}; - -class testcase_nested : public testcase_ttl { - using inherited = testcase_ttl; - using FIFO = std::deque<std::pair<uint64_t, unsigned>>; - - uint64_t serial{0}; - unsigned clear_wholetable_passed{0}; - unsigned clear_stepbystep_passed{0}; - unsigned dbfull_passed{0}; - bool keyspace_overflow{false}; - FIFO fifo; - std::stack<std::tuple<scoped_txn_guard, uint64_t, FIFO, SET>> stack; - - bool trim_tail(unsigned window_width); - bool grow_head(unsigned head_count); - bool pop_txn(bool abort); - bool pop_txn() { - return pop_txn(inherited::is_nested_txn_available() ? flipcoin_x3() - : flipcoin_x2()); - } - void push_txn(); - bool stochastic_breakable_restart_with_nested(bool force_restart = false); - -public: - testcase_nested(const actor_config &config, const mdbx_pid_t pid) - : inherited(config, pid) {} - bool setup() override; - bool run() override; - bool teardown() override; -}; diff --git a/libs/libmdbx/src/test/try.cc b/libs/libmdbx/src/test/try.cc deleted file mode 100644 index adb0113096..0000000000 --- a/libs/libmdbx/src/test/try.cc +++ /dev/null @@ -1,20 +0,0 @@ -#include "test.h" - -bool testcase_try::run() { - db_open(); - assert(!txn_guard); - - MDBX_txn *txn = nullptr; - MDBX_txn *txn2 = nullptr; - int rc = mdbx_txn_begin(db_guard.get(), nullptr, 0, &txn); - if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc); - else { - rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TRYTXN, &txn2); - if (unlikely(rc != MDBX_BUSY)) - failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc); - } - - txn_guard.reset(txn); - return true; -} diff --git a/libs/libmdbx/src/test/ttl.cc b/libs/libmdbx/src/test/ttl.cc deleted file mode 100644 index e3927d9cd4..0000000000 --- a/libs/libmdbx/src/test/ttl.cc +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" -#include <cmath> -#include <deque> - -/* LY: тест "эмуляцией time-to-live": - * - организуется "скользящее окно", которое двигается вперед вдоль - * числовой оси каждую транзакцию. - * - по переднему краю "скользящего окна" записи добавляются в таблицу, - * а по заднему удаляются. - * - количество добавляемых/удаляемых записей псевдослучайно зависит - * от номера транзакции, но с экспоненциальным распределением. - * - размер "скользящего окна" также псевдослучайно зависит от номера - * транзакции с "отрицательным" экспоненциальным распределением - * MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний - * край и удаляются записи позади него. - * - * Таким образом имитируется поведение таблицы с TTL: записи стохастически - * добавляются и удаляются, но изредка происходит массивное удаление. - */ - -unsigned testcase_ttl::edge2count(uint64_t edge) { - const double rnd = u64_to_double1(prng64_map1_white(edge)); - const unsigned count = std::lrint(std::pow(sliding.max_step_size, rnd)); - // average value: (X - 1) / ln(X), where X = sliding.max_step_size - return count; -} - -unsigned testcase_ttl::edge2window(uint64_t edge) { - const double rnd = u64_to_double1(bleach64(edge)); - const unsigned size = sliding.max_window_size - - std::lrint(std::pow(sliding.max_window_size, rnd)); - // average value: Y - (Y - 1) / ln(Y), where Y = sliding.max_window_size - return size; -} - -static inline double estimate(const double x, const double y) { - /* среднее кол-во операций N = X' * Y', где X' и Y' средние значения - * размера окна и кол-ва добавляемых за один шаг записей: - * X' = (X - 1) / ln(X), где X = sliding.max_step_size - * Y' = Y - (Y - 1) / ln(Y), где Y = sliding.max_window_size */ - return (x - 1) / std::log(x) * (y - (y - 1) / std::log(y)); -} - -bool testcase_ttl::setup() { - const unsigned window_top_lower = - 7 /* нижний предел для верхней границы диапазона, в котором будет - стохастически колебаться размер окна */ - ; - const unsigned count_top_lower = - 7 /* нижний предел для верхней границы диапазона, в котором будет - стохастически колебаться кол-во записей добавляемых на одном шаге */ - ; - - /* для параметризации используем подходящие параметры, - * которые не имеют здесь смысла в первоначальном значении. */ - const double ratio = - double(config.params.batch_read ? config.params.batch_read : 1) / - double(config.params.batch_write ? config.params.batch_write : 1); - - /* проще найти двоичным поиском (вариация метода Ньютона) */ - double hi = config.params.test_nops, lo = 1; - double x = std::sqrt(hi + lo) / ratio; - while (hi > lo) { - const double n = estimate(x, x * ratio); - if (n > config.params.test_nops) - hi = x - 1; - else - lo = x + 1; - x = (hi + lo) / 2; - } - - sliding.max_step_size = std::lrint(x); - if (sliding.max_step_size < count_top_lower) - sliding.max_step_size = count_top_lower; - sliding.max_window_size = std::lrint(x * ratio); - if (sliding.max_window_size < window_top_lower) - sliding.max_window_size = window_top_lower; - - while (estimate(sliding.max_step_size, sliding.max_window_size) > - config.params.test_nops * 2.0) { - if (ratio * sliding.max_step_size > sliding.max_window_size) { - if (sliding.max_step_size < count_top_lower) - break; - sliding.max_step_size = sliding.max_step_size * 7 / 8; - } else { - if (sliding.max_window_size < window_top_lower) - break; - sliding.max_window_size = sliding.max_window_size * 7 / 8; - } - } - - log_verbose("come up window_max %u from `batch_read`", - sliding.max_window_size); - log_verbose("come up step_max %u from `batch_write`", sliding.max_step_size); - return inherited::setup(); -} - -bool testcase_ttl::run() { - int err = db_open__begin__table_create_open_clean(dbi); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err)); - return false; - } - - uint64_t seed = - prng64_map2_white(config.params.keygen.seed) + config.actor_id; - keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); - key = keygen::alloc(config.params.keylen_max); - data = keygen::alloc(config.params.datalen_max); - const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_NODUPDATA - : MDBX_NODUPDATA | MDBX_NOOVERWRITE; - - std::deque<std::pair<uint64_t, unsigned>> fifo; - uint64_t serial = 0; - bool rc = false; - unsigned clear_wholetable_passed = 0; - unsigned clear_stepbystep_passed = 0; - unsigned dbfull_passed = 0; - unsigned loops = 0; - bool keyspace_overflow = false; - while (true) { - const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; - - const unsigned window_width = - (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt); - unsigned head_count = edge2count(salt); - log_debug("ttl: step #%" PRIu64 " (serial %" PRIu64 - ", window %u, count %u) salt %" PRIu64, - nops_completed, serial, window_width, head_count, salt); - - if (window_width || flipcoin()) { - clear_stepbystep_passed += window_width == 0; - while (fifo.size() > window_width) { - uint64_t tail_serial = fifo.back().first; - const unsigned tail_count = fifo.back().second; - log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, - tail_count); - fifo.pop_back(); - for (unsigned n = 0; n < tail_count; ++n) { - log_trace("ttl: remove-tail %" PRIu64, tail_serial); - generate_pair(tail_serial); - err = remove(key, data); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("ttl: tail-bailout due '%s'", mdbx_strerror(err)); - goto bailout; - } - failure_perror("mdbx_del(tail)", err); - } - if (unlikely(!keyvalue_maker.increment(tail_serial, 1))) - failure("ttl: unexpected key-space overflow on the tail"); - } - report(tail_count); - } - } else { - log_trace("ttl: purge state"); - db_table_clear(dbi); - fifo.clear(); - clear_wholetable_passed += 1; - report(1); - } - - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("ttl: bailout at commit due '%s'", mdbx_strerror(err)); - break; - } - if (!speculum_verify()) { - log_notice("ttl: bailout after tail-trim"); - return false; - } - - if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed || - !clear_stepbystep_passed)) { - unsigned underutilization_x256 = - txn_underutilization_x256(txn_guard.get()); - if (dbfull_passed > underutilization_x256) { - log_notice("ttl: skip head-grow to avoid one more dbfull (was %u, " - "underutilization %.2f%%)", - dbfull_passed, underutilization_x256 / 2.560); - continue; - } - fifo.push_front(std::make_pair(serial, head_count)); - retry: - for (unsigned n = 0; n < head_count; ++n) { - log_trace("ttl: insert-head %" PRIu64, serial); - generate_pair(serial); - err = insert(key, data, insert_flags); - if (unlikely(err != MDBX_SUCCESS)) { - if ((err == MDBX_TXN_FULL || err == MDBX_MAP_FULL) && - config.params.ignore_dbfull) { - log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err)); - txn_restart(true, false); - serial = fifo.front().first; - fifo.front().second = head_count = n; - dbfull_passed += 1; - goto retry; - } - failure_perror("mdbx_put(head)", err); - } - - if (unlikely(!keyvalue_maker.increment(serial, 1))) { - log_notice("ttl: unexpected key-space overflow"); - keyspace_overflow = true; - txn_restart(true, false); - serial = fifo.front().first; - fifo.front().second = head_count = n; - goto retry; - } - } - err = breakable_restart(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err)); - serial = fifo.front().first; - fifo.pop_front(); - } - if (!speculum_verify()) { - log_notice("ttl: bailout after head-grow"); - return false; - } - loops += 1; - } else if (fifo.empty()) { - log_notice("ttl: done %u whole loops, %" PRIu64 " ops, %" PRIu64 " items", - loops, nops_completed, serial); - rc = true; - break; - } else { - log_notice("ttl: done, wait for empty, skip head-grow"); - } - } - -bailout: - txn_end(true); - if (dbi) { - if (config.params.drop_table && !mode_readonly()) { - txn_begin(false); - db_table_drop(dbi); - err = breakable_commit(); - if (unlikely(err != MDBX_SUCCESS)) { - log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err)); - return false; - } - } else - db_table_close(dbi); - } - return rc; -} diff --git a/libs/libmdbx/src/test/utils.cc b/libs/libmdbx/src/test/utils.cc deleted file mode 100644 index 051671ff87..0000000000 --- a/libs/libmdbx/src/test/utils.cc +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#include "test.h" -#include <float.h> -#if defined(HAVE_IEEE754_H) || __has_include(<ieee754.h>) -#include <ieee754.h> -#endif -#if defined(__APPLE__) || defined(__MACH__) -#include <mach/mach_time.h> -#endif /* defined(__APPLE__) || defined(__MACH__) */ - -std::string format(const char *fmt, ...) { - va_list ap, ones; - va_start(ap, fmt); - va_copy(ones, ap); -#ifdef _MSC_VER - int needed = _vscprintf(fmt, ap); -#else - int needed = vsnprintf(nullptr, 0, fmt, ap); -#endif - assert(needed >= 0); - va_end(ap); - std::string result; - result.reserve((size_t)needed + 1); - result.resize((size_t)needed, '\0'); - int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones); - assert(actual == needed); - (void)actual; - va_end(ones); - return result; -} - -std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) { - std::string result; - if (bytes > 0) { - const uint8_t *data = (const uint8_t *)ptr; - checksum.push(data, bytes); - result.reserve(bytes * 2); - const uint8_t *const end = data + bytes; - do { - char h = *data >> 4; - char l = *data & 15; - result.push_back((l < 10) ? l + '0' : l - 10 + 'a'); - result.push_back((h < 10) ? h + '0' : h - 10 + 'a'); - } while (++data < end); - } - assert(result.size() == bytes * 2); - return result; -} - -bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, - size_t bytes, simple_checksum &checksum) { - if (bytes * 2 != (size_t)(hex_end - hex_begin)) - return false; - - uint8_t *data = (uint8_t *)ptr; - for (const char *hex = hex_begin; hex != hex_end; hex += 2, ++data) { - unsigned l = hex[0], h = hex[1]; - - if (l >= '0' && l <= '9') - l = l - '0'; - else if (l >= 'A' && l <= 'F') - l = l - 'A' + 10; - else if (l >= 'a' && l <= 'f') - l = l - 'a' + 10; - else - return false; - - if (h >= '0' && h <= '9') - h = h - '0'; - else if (h >= 'A' && h <= 'F') - h = h - 'A' + 10; - else if (h >= 'a' && h <= 'f') - h = h - 'a' + 10; - else - return false; - - uint32_t c = l + (h << 4); - checksum.push(c); - *data = (uint8_t)c; - } - return true; -} - -bool is_samedata(const MDBX_val *a, const MDBX_val *b) { - return a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0; -} - -//----------------------------------------------------------------------------- - -/* TODO: replace my 'libmera' from t1ha. */ -uint64_t entropy_ticks(void) { -#if defined(EMSCRIPTEN) - return (uint64_t)emscripten_get_now(); -#endif /* EMSCRIPTEN */ - -#if defined(__APPLE__) || defined(__MACH__) - return mach_absolute_time(); -#endif /* defined(__APPLE__) || defined(__MACH__) */ - -#if defined(__sun__) || defined(__sun) - return gethrtime(); -#endif /* __sun__ */ - -#if defined(__GNUC__) || defined(__clang__) - -#if defined(__ia64__) - uint64_t ticks; - __asm __volatile("mov %0=ar.itc" : "=r"(ticks)); - return ticks; -#elif defined(__hppa__) - uint64_t ticks; - __asm __volatile("mfctl 16, %0" : "=r"(ticks)); - return ticks; -#elif defined(__s390__) - uint64_t ticks; - __asm __volatile("stck 0(%0)" : : "a"(&(ticks)) : "memory", "cc"); - return ticks; -#elif defined(__alpha__) - uint64_t ticks; - __asm __volatile("rpcc %0" : "=r"(ticks)); - return ticks; -#elif defined(__sparc__) || defined(__sparc) || defined(__sparc64__) || \ - defined(__sparc64) || defined(__sparc_v8plus__) || \ - defined(__sparc_v8plus) || defined(__sparc_v8plusa__) || \ - defined(__sparc_v8plusa) || defined(__sparc_v9__) || defined(__sparc_v9) - - union { - uint64_t u64; - struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - uint32_t h, l; -#else - uint32_t l, h; -#endif - } u32; - } cycles; - -#if defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || \ - defined(__sparc_v9__) || defined(__sparc_v8plus) || \ - defined(__sparc_v8plusa) || defined(__sparc_v9) - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || \ - defined(__sparc64__) || defined(__sparc64) - __asm __volatile("rd %%tick, %0" : "=r"(cycles.u64)); -#else - __asm __volatile("rd %%tick, %1; srlx %1, 32, %0" - : "=r"(cycles.u32.h), "=r"(cycles.u32.l)); -#endif /* __sparc64__ */ - -#else - __asm __volatile(".byte 0x83, 0x41, 0x00, 0x00; mov %%g1, %0" - : "=r"(cycles.u64) - : - : "%g1"); -#endif /* __sparc8plus__ || __sparc_v9__ */ - return cycles.u64; - -#elif (defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) || \ - defined(__powerpc64)) - uint64_t ticks; - __asm __volatile("mfspr %0, 268" : "=r"(ticks)); - return ticks; -#elif (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) || \ - defined(__ppc)) -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul - uint64_t ticks; - __asm __volatile("mftb %0" : "=r"(ticks)); - *now = ticks; -#else - uint64_t ticks; - uint32_t low, high_before, high_after; - __asm __volatile("mftbu %0; mftb %1; mftbu %2" - : "=r"(high_before), "=r"(low), "=r"(high_after)); - ticks = (uint64_t)high_after << 32; - ticks |= low & /* zeroes if high part has changed */ - ~(high_before - high_after); -#endif -#elif (defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 7)) && \ - !defined(MDBX_SAFE4QEMU) - uint64_t virtual_timer; - __asm __volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer)); - return virtual_timer; -#elif (defined(__ARM_ARCH) && __ARM_ARCH > 5 && __ARM_ARCH < 8) || \ - defined(_M_ARM) - static uint32_t pmcntenset = 0x00425B00; - if (unlikely(pmcntenset == 0x00425B00)) { - uint32_t pmuseren; -#ifdef _M_ARM - pmuseren = _MoveFromCoprocessor(15, 0, 9, 14, 0); -#else - __asm("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); -#endif - if (1 & pmuseren /* Is it allowed for user mode code? */) { -#ifdef _M_ARM - pmcntenset = _MoveFromCoprocessor(15, 0, 9, 12, 1); -#else - __asm("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); -#endif - } else - pmcntenset = 0; - } - if (pmcntenset & 0x80000000ul /* Is it counting? */) { -#ifdef _M_ARM - return __rdpmccntr64(); -#else - uint32_t pmccntr; - __asm __volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); - return pmccntr; -#endif - } -#elif defined(__mips__) || defined(__mips) || defined(_R4000) - unsigned count; - __asm __volatile("rdhwr %0, $2" : "=r"(count)); - return count; -#endif /* arch selector */ -#endif /* __GNUC__ || __clang__ */ - -#if defined(__e2k__) || defined(__ia32__) - return __rdtsc(); -#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) - LARGE_INTEGER PerformanceCount; - if (QueryPerformanceCounter(&PerformanceCount)) - return PerformanceCount.QuadPart; - return GetTickCount64(); -#else - struct timespec ts; -#if defined(CLOCK_MONOTONIC_COARSE) - clockid_t clk_id = CLOCK_MONOTONIC_COARSE; -#elif defined(CLOCK_MONOTONIC_RAW) - clockid_t clk_id = CLOCK_MONOTONIC_RAW; -#else - clockid_t clk_id = CLOCK_MONOTONIC; -#endif - int rc = clock_gettime(clk_id, &ts); - if (unlikely(rc)) - failure_perror("clock_gettime()", rc); - - return (((uint64_t)ts.tv_sec) << 32) + ts.tv_nsec; -#endif -} - -//----------------------------------------------------------------------------- - -uint64_t prng64_white(uint64_t &state) { - state = prng64_map2_careless(state); - return bleach64(state); -} - -uint32_t prng32(uint64_t &state) { - return (uint32_t)(prng64_careless(state) >> 32); -} - -void prng_fill(uint64_t &state, void *ptr, size_t bytes) { - uint32_t u32 = prng32(state); - - while (bytes >= 4) { - memcpy(ptr, &u32, 4); - ptr = (uint32_t *)ptr + 1; - bytes -= 4; - u32 = prng32(state); - } - - switch (bytes & 3) { - case 3: - memcpy(ptr, &u32, 3); - break; - case 2: - memcpy(ptr, &u32, 2); - break; - case 1: - memcpy(ptr, &u32, 1); - break; - case 0: - break; - } -} - -static __thread uint64_t prng_state; - -void prng_seed(uint64_t seed) { prng_state = bleach64(seed); } - -uint32_t prng32(void) { return prng32(prng_state); } - -uint64_t prng64(void) { return prng64_white(prng_state); } - -void prng_fill(void *ptr, size_t bytes) { prng_fill(prng_state, ptr, bytes); } - -uint64_t entropy_white() { return bleach64(entropy_ticks()); } - -double double_from_lower(uint64_t salt) { -#ifdef IEEE754_DOUBLE_BIAS - ieee754_double r; - r.ieee.negative = 0; - r.ieee.exponent = IEEE754_DOUBLE_BIAS; - r.ieee.mantissa0 = (unsigned)(salt >> 32); - r.ieee.mantissa1 = (unsigned)salt; - return r.d; -#else - const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1; - const double scale = 1.0 / (double)top; - return (salt & top) * scale; -#endif -} - -double double_from_upper(uint64_t salt) { -#ifdef IEEE754_DOUBLE_BIAS - ieee754_double r; - r.ieee.negative = 0; - r.ieee.exponent = IEEE754_DOUBLE_BIAS; - salt >>= 64 - DBL_MANT_DIG; - r.ieee.mantissa0 = (unsigned)(salt >> 32); - r.ieee.mantissa1 = (unsigned)salt; - return r.d; -#else - const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1; - const double scale = 1.0 / (double)top; - return (salt >> (64 - DBL_MANT_DIG)) * scale; -#endif -} - -bool flipcoin() { return bleach32((uint32_t)entropy_ticks()) & 1; } -bool flipcoin_x2() { return (bleach32((uint32_t)entropy_ticks()) & 3) == 0; } -bool flipcoin_x3() { return (bleach32((uint32_t)entropy_ticks()) & 7) == 0; } -bool flipcoin_x4() { return (bleach32((uint32_t)entropy_ticks()) & 15) == 0; } - -bool jitter(unsigned probability_percent) { - const uint32_t top = UINT32_MAX - UINT32_MAX % 100; - uint32_t dice, edge = (top) / 100 * probability_percent; - do - dice = bleach32((uint32_t)entropy_ticks()); - while (dice >= top); - return dice < edge; -} - -void jitter_delay(bool extra) { - unsigned dice = entropy_white() & 3; - if (dice == 0) { - log_trace("== jitter.no-delay"); - } else { - log_trace(">> jitter.delay: dice %u", dice); - do { - cpu_relax(); - memory_barrier(); - cpu_relax(); - if (dice > 1) { - osal_yield(); - cpu_relax(); - if (dice > 2) { - unsigned us = entropy_white() & - (extra ? 0xffff /* 656 ms */ : 0x3ff /* 1 ms */); - log_trace("== jitter.delay: %0.6f", us / 1000000.0); - osal_udelay(us); - } - } - } while (flipcoin()); - log_trace("<< jitter.delay: dice %u", dice); - } -} diff --git a/libs/libmdbx/src/test/utils.h b/libs/libmdbx/src/test/utils.h deleted file mode 100644 index 9e6d4627aa..0000000000 --- a/libs/libmdbx/src/test/utils.h +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Copyright 2017-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. - */ - -#pragma once -#include "base.h" - -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) -#error __BYTE_ORDER__ should be defined. -#endif - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ - __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ -#error Unsupported byte order. -#endif - -#if __GNUC_PREREQ(4, 4) || defined(__clang__) -#ifndef bswap64 -#define bswap64(v) __builtin_bswap64(v) -#endif -#ifndef bswap32 -#define bswap32(v) __builtin_bswap32(v) -#endif -#if (__GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)) && \ - !defined(bswap16) -#define bswap16(v) __builtin_bswap16(v) -#endif - -#elif defined(_MSC_VER) - -#if _MSC_FULL_VER < 190024215 -#pragma message( \ - "It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.") -#endif - -#define bswap64(v) _byteswap_uint64(v) -#define bswap32(v) _byteswap_ulong(v) -#define bswap16(v) _byteswap_ushort(v) -#define rot64(v, s) _rotr64(v, s) -#define rot32(v, s) _rotr(v, s) - -#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) -#pragma intrinsic(_umul128) -#define mul_64x64_128(a, b, ph) _umul128(a, b, ph) -#pragma intrinsic(__umulh) -#define mul_64x64_high(a, b) __umulh(a, b) -#endif - -#if defined(_M_IX86) -#pragma intrinsic(__emulu) -#define mul_32x32_64(a, b) __emulu(a, b) -#elif defined(_M_ARM) -#define mul_32x32_64(a, b) _arm_umull(a, b) -#endif - -#endif /* compiler */ - -#ifndef bswap64 -#ifdef __bswap_64 -#define bswap64(v) __bswap_64(v) -#else -static __inline uint64_t bswap64(uint64_t v) { - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff0000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); -} -#endif -#endif /* bswap64 */ - -#ifndef bswap32 -#ifdef __bswap_32 -#define bswap32(v) __bswap_32(v) -#else -static __inline uint32_t bswap32(uint32_t v) { - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); -} -#endif -#endif /* bswap32 */ - -#ifndef bswap16 -#ifdef __bswap_16 -#define bswap16(v) __bswap_16(v) -#else -static __inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; } -#endif -#endif /* bswap16 */ - -#define is_byteorder_le() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -#define is_byteorder_be() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - -#ifndef htole16 -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define htobe16(v) bswap16(v) -#define htole16(v) (v) -#define be16toh(v) bswap16(v) -#define le16toh(v) (v) -#else -#define htobe16(v) (v) -#define htole16(v) bswap16(v) -#define be16toh(v) (v) -#define le16toh(v) bswap16(v) -#endif -#endif /* htole16 */ - -#ifndef htole32 -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define htobe32(v) bswap32(v) -#define htole32(v) (v) -#define be32toh(v) bswap32(v) -#define le32toh(v) (v) -#else -#define htobe32(v) (v) -#define htole32(v) bswap32(v) -#define be32toh(v) (v) -#define le32toh(v) bswap32(v) -#endif -#endif /* htole32 */ - -#ifndef htole64 -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define htobe64(v) bswap64(v) -#define htole64(v) (v) -#define be64toh(v) bswap64(v) -#define le64toh(v) (v) -#else -#define htobe64(v) (v) -#define htole64(v) bswap_64(v) -#define be64toh(v) (v) -#define le64toh(v) bswap_64(v) -#endif -#endif /* htole64 */ - -namespace unaligned { - -template <typename T> static __inline T load(const void *ptr) { -#if defined(_MSC_VER) && \ - (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) - return *(const T __unaligned *)ptr; -#elif MDBX_UNALIGNED_OK - return *(const T *)ptr; -#else - T local; -#if defined(__GNUC__) || defined(__clang__) - __builtin_memcpy(&local, (const T *)ptr, sizeof(T)); -#else - memcpy(&local, (const T *)ptr, sizeof(T)); -#endif /* __GNUC__ || __clang__ */ - return local; -#endif /* MDBX_UNALIGNED_OK */ -} - -template <typename T> static __inline void store(void *ptr, const T &value) { -#if defined(_MSC_VER) && \ - (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) - *((T __unaligned *)ptr) = value; -#elif MDBX_UNALIGNED_OK - *(volatile T *)ptr = value; -#else -#if defined(__GNUC__) || defined(__clang__) - __builtin_memcpy(ptr, &value, sizeof(T)); -#else - memcpy(ptr, &value, sizeof(T)); -#endif /* __GNUC__ || __clang__ */ -#endif /* MDBX_UNALIGNED_OK */ -} - -} /* namespace unaligned */ - -//----------------------------------------------------------------------------- - -#ifndef rot64 -static __inline uint64_t rot64(uint64_t v, unsigned s) { - return (v >> s) | (v << (64 - s)); -} -#endif /* rot64 */ - -static __inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; } - -#undef roundup2 -static __inline size_t roundup2(size_t value, size_t granularity) { - assert(is_power2(granularity)); - return (value + granularity - 1) & ~(granularity - 1); -} - -//----------------------------------------------------------------------------- - -static __inline void memory_barrier(void) { -#if __has_extension(c_atomic) || __has_extension(cxx_atomic) - __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); -#elif defined(__ATOMIC_SEQ_CST) - __atomic_thread_fence(__ATOMIC_SEQ_CST); -#elif defined(__clang__) || defined(__GNUC__) - __sync_synchronize(); -#elif defined(_MSC_VER) - MemoryBarrier(); -#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ -#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) - __mf(); -#elif defined(__ia32__) - _mm_mfence(); -#else -#error "Unknown target for Intel Compiler, please report to us." -#endif -#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) - __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) - _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) - __lwsync(); -#else -#error "Could not guess the kind of compiler, please report to us." -#endif -} - -static __inline void cpu_relax() { -#if defined(__ia32__) - _mm_pause(); -#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \ - defined(YieldProcessor) - YieldProcessor(); -#else -/* nope */ -#endif -} - -//----------------------------------------------------------------------------- - -struct simple_checksum { - uint64_t value{0}; - - simple_checksum() = default; - - void push(const uint32_t &data) { - value += data * UINT64_C(9386433910765580089) + 1; - value ^= value >> 41; - value *= UINT64_C(0xBD9CACC22C6E9571); - } - - void push(const uint64_t &data) { - push((uint32_t)data); - push((uint32_t)(data >> 32)); - } - - void push(const bool data) { - push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E)); - } - - void push(const void *ptr, size_t bytes) { - const uint8_t *data = (const uint8_t *)ptr; - for (size_t i = 0; i < bytes; ++i) - push((uint32_t)data[i]); - } - - void push(const double &data) { push(&data, sizeof(double)); } - void push(const char *cstr) { push(cstr, strlen(cstr)); } - void push(const std::string &str) { push(str.data(), str.size()); } - - void push(unsigned salt, const MDBX_val &val) { - push(unsigned(val.iov_len)); - push(salt); - push(val.iov_base, val.iov_len); - } - -#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) - void push(const HANDLE &handle) { push(&handle, sizeof(handle)); } -#endif /* _WINDOWS */ -}; - -std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum); -bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, - size_t bytes, simple_checksum &checksum); -bool is_samedata(const MDBX_val *a, const MDBX_val *b); -std::string format(const char *fmt, ...); - -uint64_t entropy_ticks(void); -uint64_t entropy_white(void); -static inline uint64_t bleach64(uint64_t v) { - // Tommy Ettinger, https://www.blogger.com/profile/04953541827437796598 - // http://mostlymangling.blogspot.com/2019/01/better-stronger-mixer-and-test-procedure.html - v ^= rot64(v, 25) ^ rot64(v, 50); - v *= UINT64_C(0xA24BAED4963EE407); - v ^= rot64(v, 24) ^ rot64(v, 49); - v *= UINT64_C(0x9FB21C651E98DF25); - return v ^ v >> 28; -} - -static inline uint32_t bleach32(uint32_t x) { - // https://github.com/skeeto/hash-prospector - // exact bias: 0.17353355999581582 - x ^= x >> 16; - x *= UINT32_C(0x7feb352d); - x ^= 0x3027C563 ^ (x >> 15); - x *= UINT32_C(0x846ca68b); - x ^= x >> 16; - return x; -} - -static inline uint64_t prng64_map1_careless(uint64_t state) { - return state * UINT64_C(6364136223846793005) + 1; -} - -static inline uint64_t prng64_map2_careless(uint64_t state) { - return (state + UINT64_C(1442695040888963407)) * - UINT64_C(6364136223846793005); -} - -static inline uint64_t prng64_map1_white(uint64_t state) { - return bleach64(prng64_map1_careless(state)); -} - -static inline uint64_t prng64_map2_white(uint64_t state) { - return bleach64(prng64_map2_careless(state)); -} - -static inline uint64_t prng64_careless(uint64_t &state) { - state = prng64_map1_careless(state); - return state; -} - -static inline double u64_to_double1(uint64_t v) { - union { - uint64_t u64; - double d; - } casting; - - casting.u64 = UINT64_C(0x3ff) << 52 | (v >> 12); - assert(casting.d >= 1.0 && casting.d < 2.0); - return casting.d - 1.0; -} - -uint64_t prng64_white(uint64_t &state); -uint32_t prng32(uint64_t &state); -void prng_fill(uint64_t &state, void *ptr, size_t bytes); - -void prng_seed(uint64_t seed); -uint32_t prng32(void); -uint64_t prng64(void); -void prng_fill(void *ptr, size_t bytes); - -bool flipcoin(); -bool flipcoin_x2(); -bool flipcoin_x3(); -bool flipcoin_x4(); -bool jitter(unsigned probability_percent); -void jitter_delay(bool extra = false); diff --git a/libs/libmdbx/src/test/valgrind_suppress.txt b/libs/libmdbx/src/test/valgrind_suppress.txt deleted file mode 100644 index e1e152051d..0000000000 --- a/libs/libmdbx/src/test/valgrind_suppress.txt +++ /dev/null @@ -1,540 +0,0 @@ -{ - msync-whole-mmap-1 - Memcheck:Param - msync(start) - fun:msync - ... - fun:mdbx_sync_locked -} -{ - msync-whole-mmap-2 - Memcheck:Param - msync(start) - fun:msync - ... - fun:mdbx_env_sync_internal -} - -# modern Valgrind don't support the `vector[...]` pattern -# for((i=0;i<64;++i)); do echo -e "{\n pwrite-page-flush-$i\n Memcheck:Param\n pwritev(vector[$i])\n fun:pwritev\n ...\n fun:mdbx_page_flush\n}"; done >> valgrind_suppress.txt - -{ - pwrite-page-flush - Memcheck:Param - pwritev(vector[...]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-0 - Memcheck:Param - pwritev(vector[0]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-1 - Memcheck:Param - pwritev(vector[1]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-2 - Memcheck:Param - pwritev(vector[2]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-3 - Memcheck:Param - pwritev(vector[3]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-4 - Memcheck:Param - pwritev(vector[4]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-5 - Memcheck:Param - pwritev(vector[5]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-6 - Memcheck:Param - pwritev(vector[6]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-7 - Memcheck:Param - pwritev(vector[7]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-8 - Memcheck:Param - pwritev(vector[8]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-9 - Memcheck:Param - pwritev(vector[9]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-10 - Memcheck:Param - pwritev(vector[10]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-11 - Memcheck:Param - pwritev(vector[11]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-12 - Memcheck:Param - pwritev(vector[12]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-13 - Memcheck:Param - pwritev(vector[13]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-14 - Memcheck:Param - pwritev(vector[14]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-15 - Memcheck:Param - pwritev(vector[15]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-16 - Memcheck:Param - pwritev(vector[16]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-17 - Memcheck:Param - pwritev(vector[17]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-18 - Memcheck:Param - pwritev(vector[18]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-19 - Memcheck:Param - pwritev(vector[19]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-20 - Memcheck:Param - pwritev(vector[20]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-21 - Memcheck:Param - pwritev(vector[21]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-22 - Memcheck:Param - pwritev(vector[22]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-23 - Memcheck:Param - pwritev(vector[23]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-24 - Memcheck:Param - pwritev(vector[24]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-25 - Memcheck:Param - pwritev(vector[25]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-26 - Memcheck:Param - pwritev(vector[26]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-27 - Memcheck:Param - pwritev(vector[27]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-28 - Memcheck:Param - pwritev(vector[28]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-29 - Memcheck:Param - pwritev(vector[29]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-30 - Memcheck:Param - pwritev(vector[30]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-31 - Memcheck:Param - pwritev(vector[31]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-32 - Memcheck:Param - pwritev(vector[32]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-33 - Memcheck:Param - pwritev(vector[33]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-34 - Memcheck:Param - pwritev(vector[34]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-35 - Memcheck:Param - pwritev(vector[35]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-36 - Memcheck:Param - pwritev(vector[36]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-37 - Memcheck:Param - pwritev(vector[37]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-38 - Memcheck:Param - pwritev(vector[38]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-39 - Memcheck:Param - pwritev(vector[39]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-40 - Memcheck:Param - pwritev(vector[40]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-41 - Memcheck:Param - pwritev(vector[41]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-42 - Memcheck:Param - pwritev(vector[42]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-43 - Memcheck:Param - pwritev(vector[43]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-44 - Memcheck:Param - pwritev(vector[44]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-45 - Memcheck:Param - pwritev(vector[45]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-46 - Memcheck:Param - pwritev(vector[46]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-47 - Memcheck:Param - pwritev(vector[47]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-48 - Memcheck:Param - pwritev(vector[48]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-49 - Memcheck:Param - pwritev(vector[49]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-50 - Memcheck:Param - pwritev(vector[50]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-51 - Memcheck:Param - pwritev(vector[51]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-52 - Memcheck:Param - pwritev(vector[52]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-53 - Memcheck:Param - pwritev(vector[53]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-54 - Memcheck:Param - pwritev(vector[54]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-55 - Memcheck:Param - pwritev(vector[55]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-56 - Memcheck:Param - pwritev(vector[56]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-57 - Memcheck:Param - pwritev(vector[57]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-58 - Memcheck:Param - pwritev(vector[58]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-59 - Memcheck:Param - pwritev(vector[59]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-60 - Memcheck:Param - pwritev(vector[60]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-61 - Memcheck:Param - pwritev(vector[61]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-62 - Memcheck:Param - pwritev(vector[62]) - fun:pwritev - ... - fun:mdbx_page_flush -} -{ - pwrite-page-flush-63 - Memcheck:Param - pwritev(vector[63]) - fun:pwritev - ... - fun:mdbx_page_flush -} diff --git a/libs/libmdbx/src/version.c b/libs/libmdbx/src/version.c deleted file mode 100644 index f091d689ba..0000000000 --- a/libs/libmdbx/src/version.c +++ /dev/null @@ -1,46 +0,0 @@ -/* This is CMake-template for libmdbx's version.c - ******************************************************************************/ - -#include "src/internals.h" - -#if MDBX_VERSION_MAJOR != 0 || \ - MDBX_VERSION_MINOR != 8 -#error "API version mismatch! Had `git fetch --tags` done?" -#endif - -static const char sourcery[] = STRINGIFY(MDBX_BUILD_SOURCERY); - -__dll_export -#ifdef __attribute_used__ - __attribute_used__ -#elif defined(__GNUC__) || __has_attribute(__used__) - __attribute__((__used__)) -#endif -#ifdef __attribute_externally_visible__ - __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) - __attribute__((__externally_visible__)) -#endif - const mdbx_version_info mdbx_version = { - 0, - 8, - 2, - 2, - {"2020-07-08T16:46:29+03:00", "91ba90ae5964c49852b2fe8ec5c6733e10d0414c", "4fffd033615ec45fd2669edcf26abd4134f76b94", - "v0.8.2-2-g4fffd03"}, - sourcery}; - -__dll_export -#ifdef __attribute_used__ - __attribute_used__ -#elif defined(__GNUC__) || __has_attribute(__used__) - __attribute__((__used__)) -#endif -#ifdef __attribute_externally_visible__ - __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) - __attribute__((__externally_visible__)) -#endif - const char *const mdbx_sourcery_anchor = sourcery; |