diff options
| author | George Hazan <ghazan@miranda.im> | 2020-01-07 22:56:32 +0300 |
|---|---|---|
| committer | George Hazan <ghazan@miranda.im> | 2020-01-07 22:56:39 +0300 |
| commit | 7524acda0a8778572262429f27b359d7ee91fc90 (patch) | |
| tree | 6443228a439b83c5bc6d15923a9ccd91fa4718e3 /libs/libmdbx/src/test | |
| parent | 62c842df372abe0349f025ac4cbbc8426526ac8b (diff) | |
libmdbx: update to 0.5.0
Diffstat (limited to 'libs/libmdbx/src/test')
30 files changed, 1572 insertions, 755 deletions
diff --git a/libs/libmdbx/src/test/CMakeLists.txt b/libs/libmdbx/src/test/CMakeLists.txt index bb4abd5d6d..0cc22182c9 100644 --- a/libs/libmdbx/src/test/CMakeLists.txt +++ b/libs/libmdbx/src/test/CMakeLists.txt @@ -1,10 +1,10 @@ -set(TARGET mdbx_test) -project(${TARGET}) +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set(TEST_OSAL windows) +else() + set(TEST_OSAL unix) +endif() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-declarations") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-cast-qual") - -add_executable(${TARGET} +add_executable(mdbx_test base.h cases.cc chrono.cc @@ -21,7 +21,7 @@ add_executable(${TARGET} log.h main.cc osal.h - osal-unix.cc + osal-${TEST_OSAL}.cc test.cc test.h try.cc @@ -29,9 +29,25 @@ add_executable(${TARGET} utils.h append.cc ttl.cc + nested.cc ) -target_link_libraries(${TARGET} - mdbx - ) +set_target_properties(mdbx_test PROPERTIES + INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}> + CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + +if(CC_HAS_FASTMATH) + target_compile_options(mdbx_test PRIVATE "-ffast-math") +endif() +if(CC_HAS_VISIBILITY AND (LTO_ENABLED OR INTERPROCEDURAL_OPTIMIZATION)) + set_target_properties(mdbx_test PROPERTIES LINK_FLAGS "-fvisibility=hidden") +endif() + +target_link_libraries(mdbx_test mdbx ${LIB_MATH} ${CMAKE_THREAD_LIBS_INIT}) +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + target_link_libraries(mdbx_test winmm.lib) +endif() +if(UNIX AND NOT SUBPROJECT) + add_subdirectory(pcrf) +endif() diff --git a/libs/libmdbx/src/test/append.cc b/libs/libmdbx/src/test/append.cc index 273f68b810..a30351dcde 100644 --- a/libs/libmdbx/src/test/append.cc +++ b/libs/libmdbx/src/test/append.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -15,7 +15,6 @@ #include "test.h" bool testcase_append::run() { - MDBX_dbi dbi; int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { log_notice("append: bailout-prepare due '%s'", mdbx_strerror(err)); @@ -54,7 +53,7 @@ bool testcase_append::run() { } log_trace("append: append-a %" PRIu64, serial); - generate_pair(serial, key, data); + generate_pair(serial); int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value, &last_key->value) : 1; diff --git a/libs/libmdbx/src/test/base.h b/libs/libmdbx/src/test/base.h index 5ca134a81d..05dfd1c4e3 100644 --- a/libs/libmdbx/src/test/base.h +++ b/libs/libmdbx/src/test/base.h @@ -80,14 +80,19 @@ #include <unordered_set> #include <vector> +#define MDBX_INTERNAL_FUNC +#define MDBX_INTERNAL_VAR extern +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ #include "../mdbx.h" -#include "../src/defs.h" -#include "../src/osal.h" +#include "../src/elements/defs.h" +#include "../src/elements/osal.h" #if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) #define __thread __declspec(thread) #endif /* __thread */ +#include "../src/elements/options.h" + #ifdef _MSC_VER #pragma warning(pop) #pragma warning(disable : 4201) /* nonstandard extension used : \ diff --git a/libs/libmdbx/src/test/cases.cc b/libs/libmdbx/src/test/cases.cc index a98834a457..b01169004b 100644 --- a/libs/libmdbx/src/test/cases.cc +++ b/libs/libmdbx/src/test/cases.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -17,7 +17,6 @@ void configure_actor(unsigned &last_space_id, const actor_testcase testcase, const char *space_id_cstr, const actor_params ¶ms) { unsigned wait4id = 0; - if (params.waitfor_nops) { for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) { if (i->is_waitable(params.waitfor_nops)) { @@ -61,15 +60,15 @@ void testcase_setup(const char *casename, actor_params ¶ms, unsigned &last_space_id) { if (strcmp(casename, "basic") == 0) { log_notice(">>> testcase_setup(%s)", casename); - configure_actor(last_space_id, ac_jitter, nullptr, params); - configure_actor(last_space_id, ac_hill, nullptr, params); - configure_actor(last_space_id, ac_ttl, nullptr, params); - configure_actor(last_space_id, ac_jitter, nullptr, params); + configure_actor(last_space_id, ac_nested, nullptr, params); configure_actor(last_space_id, ac_hill, nullptr, params); configure_actor(last_space_id, ac_ttl, nullptr, params); - configure_actor(last_space_id, ac_try, nullptr, params); configure_actor(last_space_id, ac_copy, nullptr, params); configure_actor(last_space_id, ac_append, nullptr, params); + configure_actor(last_space_id, ac_jitter, nullptr, params); + configure_actor(last_space_id, ac_try, nullptr, params); + configure_actor(last_space_id, ac_jitter, nullptr, params); + configure_actor(last_space_id, ac_try, nullptr, params); log_notice("<<< testcase_setup(%s): done", casename); } else { failure("unknown testcase `%s`", casename); diff --git a/libs/libmdbx/src/test/chrono.cc b/libs/libmdbx/src/test/chrono.cc index 38cb321a81..315d379088 100644 --- a/libs/libmdbx/src/test/chrono.cc +++ b/libs/libmdbx/src/test/chrono.cc @@ -16,7 +16,10 @@ namespace chrono { +#ifndef NSEC_PER_SEC #define NSEC_PER_SEC 1000000000u +#endif /* NSEC_PER_SEC */ + uint32_t ns2fractional(uint32_t ns) { assert(ns < NSEC_PER_SEC); /* LY: здесь и далее используется "длинное деление", которое @@ -30,7 +33,9 @@ uint32_t fractional2ns(uint32_t fractional) { return (fractional * (uint64_t)NSEC_PER_SEC) >> 32; } +#ifndef USEC_PER_SEC #define USEC_PER_SEC 1000000u +#endif /* USEC_PER_SEC */ uint32_t us2fractional(uint32_t us) { assert(us < USEC_PER_SEC); return ((uint64_t)us << 32) / USEC_PER_SEC; @@ -40,7 +45,9 @@ uint32_t fractional2us(uint32_t fractional) { return (fractional * (uint64_t)USEC_PER_SEC) >> 32; } +#ifndef MSEC_PER_SEC #define MSEC_PER_SEC 1000u +#endif /* MSEC_PER_SEC */ uint32_t ms2fractional(uint32_t ms) { assert(ms < MSEC_PER_SEC); return ((uint64_t)ms << 32) / MSEC_PER_SEC; diff --git a/libs/libmdbx/src/test/chrono.h b/libs/libmdbx/src/test/chrono.h index 11675195ac..07cdef66cf 100644 --- a/libs/libmdbx/src/test/chrono.h +++ b/libs/libmdbx/src/test/chrono.h @@ -15,7 +15,6 @@ #pragma once #include "base.h" -#include "log.h" #include "utils.h" namespace chrono { @@ -24,7 +23,7 @@ namespace chrono { typedef union time { uint64_t fixedpoint; - struct { + __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ uint32_t fractional; union { diff --git a/libs/libmdbx/src/test/config.cc b/libs/libmdbx/src/test/config.cc index dd150e9a91..eedfd7b2d4 100644 --- a/libs/libmdbx/src/test/config.cc +++ b/libs/libmdbx/src/test/config.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -294,14 +294,21 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, //----------------------------------------------------------------------------- -const struct option_verb mode_bits[] = { - {"rdonly", MDBX_RDONLY}, {"mapasync", MDBX_MAPASYNC}, - {"utterly", MDBX_UTTERLY_NOSYNC}, {"nosubdir", MDBX_NOSUBDIR}, - {"nosync", MDBX_NOSYNC}, {"nometasync", MDBX_NOMETASYNC}, - {"writemap", MDBX_WRITEMAP}, {"notls", MDBX_NOTLS}, - {"nordahead", MDBX_NORDAHEAD}, {"nomeminit", MDBX_NOMEMINIT}, - {"coalesce", MDBX_COALESCE}, {"lifo", MDBX_LIFORECLAIM}, - {"perturb", MDBX_PAGEPERTURB}, {nullptr, 0}}; +const struct option_verb mode_bits[] = {{"rdonly", MDBX_RDONLY}, + {"mapasync", MDBX_MAPASYNC}, + {"nosync-utterly", MDBX_UTTERLY_NOSYNC}, + {"nosubdir", MDBX_NOSUBDIR}, + {"nosync-safe", MDBX_SAFE_NOSYNC}, + {"nometasync", MDBX_NOMETASYNC}, + {"writemap", MDBX_WRITEMAP}, + {"notls", MDBX_NOTLS}, + {"nordahead", MDBX_NORDAHEAD}, + {"nomeminit", MDBX_NOMEMINIT}, + {"coalesce", MDBX_COALESCE}, + {"lifo", MDBX_LIFORECLAIM}, + {"perturb", MDBX_PAGEPERTURB}, + {"accede", MDBX_ACCEDE}, + {nullptr, 0}}; const struct option_verb table_bits[] = { {"key.reverse", MDBX_REVERSEKEY}, @@ -314,7 +321,7 @@ const struct option_verb table_bits[] = { static void dump_verbs(const char *caption, size_t bits, const struct option_verb *verbs) { - log_info("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits); + log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits); const char *comma = ""; while (verbs->mask && bits) { @@ -330,7 +337,7 @@ static void dump_verbs(const char *caption, size_t bits, } static void dump_duration(const char *caption, unsigned duration) { - log_info("%s: ", caption); + log_verbose("%s: ", caption); if (duration) { if (duration > 24 * 3600) logging::feed("%u_", duration / (24 * 3600)); @@ -347,84 +354,92 @@ void dump(const char *title) { logging::local_suffix indent(title); for (auto i = global::actors.begin(); i != global::actors.end(); ++i) { - log_info("#%u, testcase %s, space_id/table %u\n", i->actor_id, - testcase2str(i->testcase), i->space_id); + log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id, + testcase2str(i->testcase), i->space_id); indent.push(); if (i->params.loglevel) { - log_info("log: level %u, %s\n", i->params.loglevel, - i->params.pathname_log.empty() ? "console" - : i->params.pathname_log.c_str()); + log_verbose("log: level %u, %s\n", i->params.loglevel, + i->params.pathname_log.empty() + ? "console" + : i->params.pathname_log.c_str()); } - log_info("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR - ", %i %i, %i]\n", - i->params.pathname_db.c_str(), i->params.size_now, - i->params.size_lower, i->params.size_upper, - i->params.shrink_threshold, i->params.growth_step, - i->params.pagesize); + log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR + ", %i %i, %i]\n", + i->params.pathname_db.c_str(), i->params.size_now, + i->params.size_lower, i->params.size_upper, + i->params.shrink_threshold, i->params.growth_step, + i->params.pagesize); dump_verbs("mode", i->params.mode_flags, mode_bits); dump_verbs("table", i->params.table_flags, table_bits); if (i->params.test_nops) - log_info("iterations/records %u\n", i->params.test_nops); + log_verbose("iterations/records %u\n", i->params.test_nops); else dump_duration("duration", i->params.test_duration); if (i->params.nrepeat) - log_info("repeat %u\n", i->params.nrepeat); + log_verbose("repeat %u\n", i->params.nrepeat); else - log_info("repeat ETERNALLY\n"); + log_verbose("repeat ETERNALLY\n"); - log_info("threads %u\n", i->params.nthreads); + log_verbose("threads %u\n", i->params.nthreads); - log_info( + log_verbose( "keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64 ", split %u/%u\n", keygencase2str(i->params.keygen.keycase), i->params.keygen.width, i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset, i->params.keygen.split, i->params.keygen.width - i->params.keygen.split); - log_info("keygen.seed: %u\n", i->params.keygen.seed); - log_info("key: minlen %u, maxlen %u\n", i->params.keylen_min, - i->params.keylen_max); - log_info("data: minlen %u, maxlen %u\n", i->params.datalen_min, - i->params.datalen_max); + log_verbose("keygen.seed: %u\n", i->params.keygen.seed); + log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min, + i->params.keylen_max); + log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min, + i->params.datalen_max); - log_info("batch: read %u, write %u\n", i->params.batch_read, - i->params.batch_write); + log_verbose("batch: read %u, write %u\n", i->params.batch_read, + i->params.batch_write); if (i->params.waitfor_nops) - log_info("wait: actor %u for %u ops\n", i->wait4id, - i->params.waitfor_nops); + log_verbose("wait: actor %u for %u ops\n", i->wait4id, + i->params.waitfor_nops); else if (i->params.delaystart) dump_duration("delay", i->params.delaystart); else - log_info("no-delay\n"); + log_verbose("no-delay\n"); if (i->params.inject_writefaultn) - log_info("inject-writefault on %u ops\n", i->params.inject_writefaultn); + log_verbose("inject-writefault on %u ops\n", + i->params.inject_writefaultn); else - log_info("no-inject-writefault\n"); + log_verbose("no-inject-writefault\n"); - log_info("limits: readers %u, tables %u\n", i->params.max_readers, - i->params.max_tables); + log_verbose("limits: readers %u, tables %u, txn-bytes %zu\n", + i->params.max_readers, i->params.max_tables, + mdbx_limits_txnsize_max(i->params.pagesize)); + + log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No"); + log_verbose("ignore MDBX_MAP_FULL error: %s\n", + i->params.ignore_dbfull ? "Yes" : "No"); + log_verbose("verifying by speculum: %s\n", + i->params.speculum ? "Yes" : "No"); - log_info("drop table: %s\n", i->params.drop_table ? "Yes" : "No"); - log_info("ignore MDBX_MAP_FULL error: %s\n", - i->params.ignore_dbfull ? "Yes" : "No"); indent.pop(); } dump_duration("timeout", global::config::timeout_duration_seconds); - log_info("cleanup: before %s, after %s\n", - global::config::cleanup_before ? "Yes" : "No", - global::config::cleanup_after ? "Yes" : "No"); - - log_info("failfast: %s\n", global::config::failfast ? "Yes" : "No"); - log_info("progress indicator: %s\n", - global::config::progress_indicator ? "Yes" : "No"); + log_verbose("cleanup: before %s, after %s\n", + global::config::cleanup_before ? "Yes" : "No", + global::config::cleanup_after ? "Yes" : "No"); + + log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No"); + log_verbose("progress indicator: %s\n", + global::config::progress_indicator ? "Yes" : "No"); + log_verbose("console mode: %s\n", + global::config::console_mode ? "Yes" : "No"); } } /* namespace config */ @@ -452,26 +467,31 @@ const std::string actor_config::serialize(const char *prefix) const { checksum.push(params.pathname_db); result.append(params.pathname_db); - result.append("|"); + result.push_back('|'); checksum.push(params.pathname_log); result.append(params.pathname_log); - result.append("|"); + result.push_back('|'); static_assert(std::is_pod<actor_params_pod>::value, "actor_params_pod should by POD"); result.append(data2hex(static_cast<const actor_params_pod *>(¶ms), sizeof(actor_params_pod), checksum)); - result.append("|"); + result.push_back('|'); static_assert(std::is_pod<actor_config_pod>::value, "actor_config_pod should by POD"); result.append(data2hex(static_cast<const actor_config_pod *>(this), sizeof(actor_config_pod), checksum)); - result.append("|"); + result.push_back('|'); + result.push_back(global::config::progress_indicator ? 'Y' : 'N'); + checksum.push(global::config::progress_indicator); + result.push_back(global::config::console_mode ? 'Y' : 'N'); + checksum.push(global::config::console_mode); + result.push_back('|'); result.append(osal_serialize(checksum)); - result.append("|"); + result.push_back('|'); result.append(std::to_string(checksum.value)); return result; @@ -535,6 +555,20 @@ bool actor_config::deserialize(const char *str, actor_config &config) { TRACE("<< actor_config::deserialize: slash-5\n"); return false; } + if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N')) { + global::config::progress_indicator = str[0] == 'Y'; + checksum.push(global::config::progress_indicator); + global::config::console_mode = str[1] == 'Y'; + checksum.push(global::config::console_mode); + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-6\n"); + return false; + } + } + if (!config.osal_deserialize(str, slash, checksum)) { TRACE("<< actor_config::deserialize: osal\n"); return false; @@ -556,10 +590,7 @@ unsigned actor_params::mdbx_keylen_min() const { } unsigned actor_params::mdbx_keylen_max() const { - return (table_flags & MDBX_INTEGERKEY) - ? 8 - : std::min((unsigned)mdbx_limits_keysize_max(pagesize), - (unsigned)UINT16_MAX); + return (unsigned)mdbx_limits_keysize_max(pagesize, table_flags); } unsigned actor_params::mdbx_datalen_min() const { @@ -567,10 +598,6 @@ unsigned actor_params::mdbx_datalen_min() const { } unsigned actor_params::mdbx_datalen_max() const { - return (table_flags & MDBX_INTEGERDUP) - ? 8 - : std::min((table_flags & MDBX_DUPSORT) - ? (unsigned)mdbx_limits_keysize_max(pagesize) - : (unsigned)MDBX_MAXDATASIZE, - (unsigned)UINT16_MAX); + return std::min((unsigned)UINT16_MAX, + (unsigned)mdbx_limits_valsize_max(pagesize, table_flags)); } diff --git a/libs/libmdbx/src/test/config.h b/libs/libmdbx/src/test/config.h index 89889d8eb6..9e083f3f22 100644 --- a/libs/libmdbx/src/test/config.h +++ b/libs/libmdbx/src/test/config.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -29,12 +29,13 @@ enum actor_testcase { ac_try, ac_copy, ac_append, - ac_ttl + ac_ttl, + ac_nested }; enum actor_status { as_unknown, - as_debuging, + as_debugging, as_running, as_successful, as_killed, @@ -123,8 +124,6 @@ inline bool parse_option_intptr(int argc, char *const argv[], int &narg, #pragma pack(push, 1) struct keygen_params_pod { - keygen_case keycase; - /* Параметры генератора пар key-value. * * Ключи и значения генерируются по задаваемым параметрам на основе "плоской" @@ -231,11 +230,10 @@ struct keygen_params_pod { uint8_t split; uint32_t seed; uint64_t offset; + keygen_case keycase; }; struct actor_params_pod { - unsigned loglevel; - unsigned mode_flags; unsigned table_flags; intptr_t size_lower; @@ -264,8 +262,10 @@ struct actor_params_pod { unsigned max_tables; keygen_params_pod keygen; + uint8_t loglevel; bool drop_table; bool ignore_dbfull; + bool speculum; }; struct actor_config_pod { diff --git a/libs/libmdbx/src/test/copy.cc b/libs/libmdbx/src/test/copy.cc index e239d41e34..ff53153e1a 100644 --- a/libs/libmdbx/src/test/copy.cc +++ b/libs/libmdbx/src/test/copy.cc @@ -1,4 +1,4 @@ -#include "test.h" +#include "test.h" void testcase_copy::copy_db(const bool with_compaction) { int err = osal_removefile(copy_pathname); diff --git a/libs/libmdbx/src/test/dead.cc b/libs/libmdbx/src/test/dead.cc index a1a8b5f9de..8f83bbeb2e 100644 --- a/libs/libmdbx/src/test/dead.cc +++ b/libs/libmdbx/src/test/dead.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. diff --git a/libs/libmdbx/src/test/hill.cc b/libs/libmdbx/src/test/hill.cc index 1b03ddf0fc..9d9890951a 100644 --- a/libs/libmdbx/src/test/hill.cc +++ b/libs/libmdbx/src/test/hill.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -15,12 +15,13 @@ #include "test.h" bool testcase_hill::run() { - MDBX_dbi dbi; int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err)); - return true; + return false; } + speculum.clear(); + speculum_commited.clear(); /* LY: тест "холмиком": * - сначала наполняем таблицу циклическими CRUD-манипуляциями, @@ -63,6 +64,7 @@ bool testcase_hill::run() { uint64_t commited_serial = serial_count; unsigned txn_nops = 0; + bool rc = false; while (should_continue()) { const keygen::serial_t a_serial = serial_count; if (unlikely(!keyvalue_maker.increment(serial_count, 1))) { @@ -78,53 +80,74 @@ bool testcase_hill::run() { log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift, a_serial); generate_pair(a_serial, a_key, a_data_1, age_shift); - err = mdbx_put(txn_guard.get(), dbi, &a_key->value, &a_data_1->value, - insert_flags); + + err = insert(a_key, a_data_1, insert_flags); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; + speculum = speculum_commited; break; } failure_perror("mdbx_put(insert-a.1)", err); } + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-a, before commit"); + goto bailout; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; + speculum = speculum_commited; break; } + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-a, after commit"); + goto bailout; + } } // создаем вторую запись из пары log_trace("uphill: insert-b %" PRIu64, b_serial); generate_pair(b_serial, b_key, b_data, 0); - err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value, - insert_flags); + err = insert(b_key, b_data, insert_flags); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; + speculum = speculum_commited; break; } failure_perror("mdbx_put(insert-b)", err); } + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-b, before commit"); + goto bailout; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; + speculum = speculum_commited; break; } + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-b, after commit"); + goto bailout; + } } // обновляем данные в первой записи @@ -132,52 +155,73 @@ bool testcase_hill::run() { a_serial); generate_pair(a_serial, a_key, a_data_0, 0); checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value); - err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value, - &a_data_1->value, update_flags); + err = replace(a_key, a_data_0, a_data_1, update_flags); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; + speculum = speculum_commited; break; } failure_perror("mdbx_replace(update-a: 1->0)", err); } + if (!speculum_verify()) { + log_notice("uphill: bailout after update-a, before commit"); + goto bailout; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; + speculum = speculum_commited; break; } + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after update-a, after commit"); + goto bailout; + } } // удаляем вторую запись log_trace("uphill: delete-b %" PRIu64, b_serial); checkdata("uphill: delete-b", dbi, b_key->value, b_data->value); - err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value); + err = remove(b_key, b_data); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; + speculum = speculum_commited; break; } failure_perror("mdbx_del(b)", err); } + if (!speculum_verify()) { + log_notice("uphill: bailout after delete-b, before commit"); + goto bailout; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; + speculum = speculum_commited; break; } + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after delete-b, after commit"); + goto bailout; + } } report(1); @@ -204,101 +248,145 @@ bool testcase_hill::run() { generate_pair(a_serial, a_key, a_data_0, 0); generate_pair(a_serial, a_key, a_data_1, age_shift); checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value); - err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value, - &a_data_0->value, update_flags); + err = replace(a_key, a_data_1, a_data_0, update_flags); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("downhill: bailout at update-a due '%s'", mdbx_strerror(err)); txn_end(true); + speculum = speculum_commited; break; } failure_perror("mdbx_put(update-a: 0->1)", err); } + if (!speculum_verify()) { + log_notice("downhill: bailout after update-a, before commit"); + break; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; break; } + speculum_commited = speculum; txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after update-a, after commit"); + break; + } } // создаем вторую запись из пары log_trace("downhill: insert-b %" PRIu64, b_serial); generate_pair(b_serial, b_key, b_data, 0); - err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value, - insert_flags); + err = insert(b_key, b_data, insert_flags); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("downhill: bailout at insert-a due '%s'", mdbx_strerror(err)); txn_end(true); + speculum = speculum_commited; break; } failure_perror("mdbx_put(insert-b)", err); } + if (!speculum_verify()) { + log_notice("downhill: bailout after insert-b, before commit"); + break; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; break; } + speculum_commited = speculum; txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after insert-b, after commit"); + break; + } } // удаляем первую запись log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift, a_serial); checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value); - err = mdbx_del(txn_guard.get(), dbi, &a_key->value, &a_data_1->value); + err = remove(a_key, a_data_1); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("downhill: bailout at delete-a due '%s'", mdbx_strerror(err)); txn_end(true); + speculum = speculum_commited; break; } failure_perror("mdbx_del(a)", err); } + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-a, before commit"); + break; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; break; } + speculum_commited = speculum; txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-a, after commit"); + break; + } } // удаляем вторую запись log_trace("downhill: delete-b %" PRIu64, b_serial); checkdata("downhill: delete-b", dbi, b_key->value, b_data->value); - err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value); + err = remove(b_key, b_data); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { log_notice("downhill: bailout at delete-b due '%s'", mdbx_strerror(err)); txn_end(true); + speculum = speculum_commited; break; } failure_perror("mdbx_del(b)", err); } + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-b, before commit"); + break; + } if (++txn_nops >= config.params.batch_write) { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; break; } + speculum_commited = speculum; txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-b, after commit"); + goto bailout; + } } report(1); } + rc = speculum_verify(); +bailout: if (txn_guard) { err = breakable_commit(); if (unlikely(err != MDBX_SUCCESS)) @@ -312,10 +400,10 @@ bool testcase_hill::run() { err = breakable_commit(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err)); - return true; + return rc; } } else db_table_close(dbi); } - return true; + return rc; } diff --git a/libs/libmdbx/src/test/jitter.cc b/libs/libmdbx/src/test/jitter.cc index 82d1d764ff..c7b8bccb51 100644 --- a/libs/libmdbx/src/test/jitter.cc +++ b/libs/libmdbx/src/test/jitter.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -15,10 +15,26 @@ #include "test.h" bool testcase_jitter::run() { + int err; + size_t upper_limit = config.params.size_upper; + if (upper_limit < 1) + upper_limit = config.params.size_now * 2; + while (should_continue()) { jitter_delay(); db_open(); + if (upper_limit < 1) { + MDBX_envinfo info; + err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &info, + sizeof(info)); + if (err) + failure_perror("mdbx_env_info_ex()", err); + upper_limit = (info.mi_geo.upper < INTPTR_MAX) + ? (intptr_t)info.mi_geo.upper + : INTPTR_MAX; + } + if (flipcoin()) { jitter_delay(); txn_begin(true); @@ -27,19 +43,29 @@ bool testcase_jitter::run() { txn_end(flipcoin()); } + const bool coin4size = flipcoin(); jitter_delay(); txn_begin(mode_readonly()); jitter_delay(); if (!mode_readonly()) { fetch_canary(); update_canary(1); - /* TODO: - * - db_setsize() - * ... - */ + err = mdbx_env_set_geometry( + db_guard.get(), -1, -1, + coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1); + if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE && + err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE) + failure_perror("mdbx_env_set_geometry-1", err); } txn_end(flipcoin()); + err = mdbx_env_set_geometry( + db_guard.get(), -1, -1, + !coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1); + if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE && + err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE) + failure_perror("mdbx_env_set_geometry-2", err); + if (flipcoin()) { jitter_delay(); txn_begin(true); @@ -48,6 +74,12 @@ bool testcase_jitter::run() { } jitter_delay(); + err = + mdbx_env_set_geometry(db_guard.get(), -1, -1, upper_limit, -1, -1, -1); + if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE && + err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE) + failure_perror("mdbx_env_set_geometry-3", err); + db_close(); /* just 'align' nops with other tests with batching */ diff --git a/libs/libmdbx/src/test/keygen.cc b/libs/libmdbx/src/test/keygen.cc index 0110b049bf..c2098e6ef9 100644 --- a/libs/libmdbx/src/test/keygen.cc +++ b/libs/libmdbx/src/test/keygen.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -72,7 +72,7 @@ serial_t injective(const serial_t serial, } void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, - serial_t value_age) { + serial_t value_age, const bool keylen_changeable) { assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith); assert(mapping.split <= mapping.width); assert(mapping.mesh <= mapping.width); @@ -131,14 +131,65 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial, value_serial); - mk(key_serial, key_essentials, *key); - mk(value_serial, value_essentials, *value); + mk_begin(key_serial, key_essentials, *key); + mk_begin(value_serial, value_essentials, *value); + +#if 0 /* unused for now */ + if (key->value.iov_len + value->value.iov_len > pair_maxlen) { + unsigned extra = key->value.iov_len + value->value.iov_len - pair_maxlen; + if (keylen_changeable && + key->value.iov_len > std::max(8u, (unsigned)key_essentials.minlen)) { +#if defined(__GNUC__) || defined(__clang__) + const bool coin = __builtin_parityll(serial) != 0; +#else + const bool coin = INT64_C(0xF2CEECA9989BD96A) * int64_t(serial) < 0; +#endif + if (coin) { + const unsigned gap = + key->value.iov_len - std::max(8u, (unsigned)key_essentials.minlen); + const unsigned chop = std::min(gap, extra); + log_trace("keygen-pair: chop %u key-len %u -> %u", chop, + (unsigned)key->value.iov_len, + (unsigned)key->value.iov_len - chop); + key->value.iov_len -= chop; + extra -= chop; + } + } + if (extra && value->value.iov_len > + std::max(8u, (unsigned)value_essentials.minlen)) { + const unsigned gap = value->value.iov_len - + std::max(8u, (unsigned)value_essentials.minlen); + const unsigned chop = std::min(gap, extra); + log_trace("keygen-pair: chop %u value-len %u -> %u", chop, + (unsigned)value->value.iov_len, + (unsigned)value->value.iov_len - chop); + value->value.iov_len -= chop; + extra -= chop; + } + if (keylen_changeable && extra && + key->value.iov_len > std::max(8u, (unsigned)key_essentials.minlen)) { + const unsigned gap = + key->value.iov_len - std::max(8u, (unsigned)key_essentials.minlen); + const unsigned chop = std::min(gap, extra); + log_trace("keygen-pair: chop %u key-len %u -> %u", chop, + (unsigned)key->value.iov_len, + (unsigned)key->value.iov_len - chop); + key->value.iov_len -= chop; + extra -= chop; + } + } +#else + (void)keylen_changeable; +#endif /* unused for now */ + + mk_continue(key_serial, key_essentials, *key); + mk_continue(value_serial, value_essentials, *value); if (log_enabled(logging::trace)) { - char dump_key[128], dump_value[128]; + char dump_key[4096], dump_value[4096]; log_trace("keygen-pair: key %s, value %s", - mdbx_dkey(&key->value, dump_key, sizeof(dump_key)), - mdbx_dkey(&value->value, dump_value, sizeof(dump_value))); + mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)), + mdbx_dump_val(&value->value, dump_value, sizeof(dump_value))); } } @@ -146,19 +197,22 @@ void maker::setup(const config::actor_params_pod &actor, unsigned actor_id, unsigned thread_number) { key_essentials.flags = actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT); - assert(actor.keylen_min <= UINT8_MAX); - key_essentials.minlen = (uint8_t)actor.keylen_min; - assert(actor.keylen_max <= UINT16_MAX); - key_essentials.maxlen = (uint16_t)actor.keylen_max; + assert(actor.keylen_min <= UINT16_MAX); + key_essentials.minlen = (uint16_t)actor.keylen_min; + assert(actor.keylen_max <= UINT32_MAX); + key_essentials.maxlen = std::min( + (uint32_t)actor.keylen_max, + (uint32_t)mdbx_limits_keysize_max(actor.pagesize, key_essentials.flags)); value_essentials.flags = actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP); - assert(actor.datalen_min <= UINT8_MAX); - value_essentials.minlen = (uint8_t)actor.datalen_min; - assert(actor.datalen_max <= UINT16_MAX); - value_essentials.maxlen = (uint16_t)actor.datalen_max; + assert(actor.datalen_min <= UINT16_MAX); + value_essentials.minlen = (uint16_t)actor.datalen_min; + assert(actor.datalen_max <= UINT32_MAX); + value_essentials.maxlen = std::min( + (uint32_t)actor.datalen_max, + (uint32_t)mdbx_limits_valsize_max(actor.pagesize, key_essentials.flags)); - assert(thread_number < 2); (void)thread_number; mapping = actor.keygen; salt = (actor.keygen.seed + actor_id) * UINT64_C(14653293970879851569); @@ -226,18 +280,25 @@ buffer alloc(size_t limit) { return buffer(ptr); } -void __hot maker::mk(const serial_t serial, const essentials ¶ms, - result &out) { +void __hot maker::mk_begin(const serial_t serial, const essentials ¶ms, + result &out) { assert(out.limit >= params.maxlen); assert(params.maxlen >= params.minlen); assert(params.maxlen >= length(serial)); - out.value.iov_base = out.bytes; out.value.iov_len = (params.maxlen > params.minlen) ? params.minlen + serial % (params.maxlen - params.minlen) : params.minlen; + if ((params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) == 0 && + out.value.iov_len < 8) + out.value.iov_len = std::max(length(serial), out.value.iov_len); +} + +void __hot maker::mk_continue(const serial_t serial, const essentials ¶ms, + result &out) { + out.value.iov_base = out.bytes; if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) { assert(params.maxlen == params.minlen); assert(params.minlen == 4 || params.minlen == 8); @@ -251,17 +312,13 @@ void __hot maker::mk(const serial_t serial, const essentials ¶ms, unaligned::store(out.bytes + out.value.iov_len - 8, htobe64(serial)); } else { out.u64 = htobe64(serial); - if (out.value.iov_len < 8) { - out.value.iov_len = std::max(length(serial), out.value.iov_len); + if (out.value.iov_len < 8) out.value.iov_base = out.bytes + 8 - out.value.iov_len; - } } } else { out.u64 = htole64(serial); if (out.value.iov_len > 8) memset(out.bytes + 8, '\0', out.value.iov_len - 8); - else - out.value.iov_len = std::max(length(serial), out.value.iov_len); } assert(out.value.iov_len >= params.minlen); diff --git a/libs/libmdbx/src/test/keygen.h b/libs/libmdbx/src/test/keygen.h index d0299e1e7d..d25b88349f 100644 --- a/libs/libmdbx/src/test/keygen.h +++ b/libs/libmdbx/src/test/keygen.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -93,7 +93,7 @@ struct result { //----------------------------------------------------------------------------- -struct buffer_deleter : public std::unary_function<void, result *> { +struct buffer_deleter /* : public std::unary_function<void, result *> */ { void operator()(result *buffer) const { free(buffer); } }; @@ -107,18 +107,25 @@ class maker { serial_t salt; struct essentials { - uint8_t minlen; - uint8_t flags; - uint16_t maxlen; + uint16_t minlen; + uint16_t flags; + uint32_t maxlen; } key_essentials, value_essentials; - static void mk(const serial_t serial, const essentials ¶ms, result &out); + static void mk_begin(const serial_t serial, const essentials ¶ms, + result &out); + static void mk_continue(const serial_t serial, const essentials ¶ms, + result &out); + static void mk(const serial_t serial, const essentials ¶ms, result &out) { + mk_begin(serial, params, out); + mk_continue(serial, params, out); + } public: maker() { memset(this, 0, sizeof(*this)); } void pair(serial_t serial, const buffer &key, buffer &value, - serial_t value_age); + serial_t value_age, const bool keylen_changeable); void setup(const config::actor_params_pod &actor, unsigned actor_id, unsigned thread_number); void make_ordered(); diff --git a/libs/libmdbx/src/test/log.cc b/libs/libmdbx/src/test/log.cc index 79544e11bb..2b22e28785 100644 --- a/libs/libmdbx/src/test/log.cc +++ b/libs/libmdbx/src/test/log.cc @@ -20,7 +20,7 @@ void failure(const char *fmt, ...) { va_list ap; va_start(ap, fmt); fflushall(); - logging::output(logging::failure, fmt, ap); + logging::output_nocheckloglevel_ap(logging::failure, fmt, ap); va_end(ap); fflushall(); exit(EXIT_FAILURE); @@ -37,29 +37,18 @@ void __noreturn failure_perror(const char *what, int errnum) { //----------------------------------------------------------------------------- -static void mdbx_logger(int type, const char *function, int line, +static void mdbx_logger(int priority, const char *function, int line, const char *msg, va_list args) { - logging::loglevel level = logging::info; - if (type & MDBX_DBG_EXTRA) - level = logging::extra; - if (type & MDBX_DBG_TRACE) - level = logging::trace; - if (type & MDBX_DBG_PRINT) - level = logging::verbose; - if (!function) function = "unknown"; - if (type & MDBX_DBG_ASSERT) { - log_error("mdbx: assertion failure: %s, %d", function, line); - level = logging::failure; - } - if (logging::output( - level, - strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx: %s: ", function)) - logging::feed_ap(msg, args); - if (type & MDBX_DBG_ASSERT) - abort(); + if (priority == MDBX_LOG_FATAL) + log_error("mdbx: fatal failure: %s, %d", function, line); + + logging::output_nocheckloglevel( + logging::loglevel(priority), + strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx %s: ", function); + logging::feed_ap(msg, args); } namespace logging { @@ -69,19 +58,16 @@ static std::string suffix; static loglevel level; static FILE *last; -void setlevel(loglevel _level) { - level = (_level > error) ? failure : _level; - int mdbx_dbg_opts = MDBX_DBG_ASSERT | MDBX_DBG_JITTER | MDBX_DBG_DUMP; - if (level <= trace) - mdbx_dbg_opts |= MDBX_DBG_TRACE; - if (level <= verbose) - mdbx_dbg_opts |= MDBX_DBG_PRINT; - int rc = mdbx_setup_debug(mdbx_dbg_opts, mdbx_logger); +void setlevel(loglevel priority) { + level = priority; + int rc = mdbx_setup_debug(priority, + MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER, + mdbx_logger); log_trace("set mdbx debug-opts: 0x%02x", rc); } -void setup(loglevel _level, const std::string &_prefix) { - setlevel(_level); +void setup(loglevel priority, const std::string &_prefix) { + setlevel(priority); prefix = _prefix; } @@ -95,10 +81,10 @@ const char *level2str(const loglevel alevel) { return "extra"; case trace: return "trace"; + case debug: + return "debug"; case verbose: return "verbose"; - case info: - return "info"; case notice: return "notice"; case warning: @@ -111,26 +97,28 @@ const char *level2str(const loglevel alevel) { } bool output(const loglevel priority, const char *format, ...) { - if (priority < level) + if (lower(priority, level)) return false; va_list ap; va_start(ap, format); - output(priority, format, ap); + output_nocheckloglevel_ap(priority, format, ap); va_end(ap); return true; } -bool output(const logging::loglevel priority, const char *format, va_list ap) { +void output_nocheckloglevel_ap(const logging::loglevel priority, + const char *format, va_list ap) { if (last) { putc('\n', last); fflush(last); + if (last == stderr) { + putc('\n', stdout); + fflush(stdout); + } last = nullptr; } - if (priority < level) - return false; - chrono::time now = chrono::now_realtime(); struct tm tm; #ifdef _MSC_VER @@ -144,14 +132,14 @@ bool output(const logging::loglevel priority, const char *format, va_list ap) { last = stdout; fprintf(last, - "[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05u %-10s %.4s ] %s" /* TODO */, + "[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05lu %-10s %.4s ] %s" /* TODO */, tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, - tm.tm_sec, chrono::fractional2us(now.fractional), osal_getpid(), + tm.tm_sec, chrono::fractional2us(now.fractional), (long)osal_getpid(), prefix.c_str(), level2str(priority), suffix.c_str()); va_list ones; memset(&ones, 0, sizeof(ones)) /* zap MSVC and other stupid compilers */; - if (priority >= error) + if (same_or_higher(priority, error)) va_copy(ones, ap); vfprintf(last, format, ap); @@ -178,29 +166,36 @@ bool output(const logging::loglevel priority, const char *format, va_list ap) { break; } - if (priority >= error) { + if (same_or_higher(priority, error)) { if (last != stderr) { - fprintf(stderr, "[ %05u %-10s %.4s ] %s", osal_getpid(), prefix.c_str(), - level2str(priority), suffix.c_str()); + fprintf(stderr, "[ %05lu %-10s %.4s ] %s", (long)osal_getpid(), + prefix.c_str(), level2str(priority), suffix.c_str()); vfprintf(stderr, format, ones); - if (end != '\n') - putc('\n', stderr); - fflush(stderr); + if (end == '\n') + fflush(stderr); + else + last = stderr; } va_end(ones); } - - return true; } bool feed_ap(const char *format, va_list ap) { if (!last) return false; + if (last == stderr) { + va_list ones; + va_copy(ones, ap); + vfprintf(stdout, format, ones); + va_end(ones); + } vfprintf(last, format, ap); size_t len = strlen(format); if (len && format[len - 1] == '\n') { fflush(last); + if (last == stderr) + fflush(stdout); last = nullptr; } return true; @@ -242,73 +237,123 @@ void local_suffix::pop() { local_suffix::~local_suffix() { suffix.erase(trim_pos); } +void progress_canary(bool active) { + static chrono::time progress_timestamp; + chrono::time now = chrono::now_motonic(); + + if (now.fixedpoint - progress_timestamp.fixedpoint < + chrono::from_ms(42).fixedpoint) + return; + + if (osal_progress_push(active)) { + progress_timestamp = now; + return; + } + + if (progress_timestamp.fixedpoint == 0) { + putc('>', stderr); + progress_timestamp = now; + } else if (global::config::console_mode) { + if (active) { + static int last_point = -1; + int point = (now.fixedpoint >> 29) & 3; + if (point != last_point) { + progress_timestamp = now; + fprintf(stderr, "%c\b", "-\\|/"[last_point = point]); + } + } else if (now.fixedpoint - progress_timestamp.fixedpoint > + chrono::from_seconds(2).fixedpoint) { + progress_timestamp = now; + fprintf(stderr, "%c\b", "@*"[now.utc & 1]); + } + } else { + static int count; + if (active && now.fixedpoint - progress_timestamp.fixedpoint > + chrono::from_seconds(1).fixedpoint) { + putc('.', stderr); + progress_timestamp = now; + ++count; + } else if (now.fixedpoint - progress_timestamp.fixedpoint > + chrono::from_seconds(5).fixedpoint) { + putc("@*"[now.utc & 1], stderr); + progress_timestamp = now; + ++count; + } + if (count == 60) { + count = 0; + putc('\n', stderr); + } + } + fflush(stderr); +} + } // namespace logging void log_extra(const char *msg, ...) { - if (logging::extra >= logging::level) { + if (logging::same_or_higher(logging::extra, logging::level)) { va_list ap; va_start(ap, msg); - logging::output(logging::extra, msg, ap); + logging::output_nocheckloglevel_ap(logging::extra, msg, ap); va_end(ap); } else logging::last = nullptr; } void log_trace(const char *msg, ...) { - if (logging::trace >= logging::level) { + if (logging::same_or_higher(logging::trace, logging::level)) { va_list ap; va_start(ap, msg); - logging::output(logging::trace, msg, ap); + logging::output_nocheckloglevel_ap(logging::trace, msg, ap); va_end(ap); } else logging::last = nullptr; } -void log_verbose(const char *msg, ...) { - if (logging::verbose >= logging::level) { +void log_debug(const char *msg, ...) { + if (logging::same_or_higher(logging::debug, logging::level)) { va_list ap; va_start(ap, msg); - logging::output(logging::verbose, msg, ap); + logging::output_nocheckloglevel_ap(logging::debug, msg, ap); va_end(ap); } else logging::last = nullptr; } -void log_info(const char *msg, ...) { - if (logging::info >= logging::level) { +void log_verbose(const char *msg, ...) { + if (logging::same_or_higher(logging::verbose, logging::level)) { va_list ap; va_start(ap, msg); - logging::output(logging::info, msg, ap); + logging::output_nocheckloglevel_ap(logging::verbose, msg, ap); va_end(ap); } else logging::last = nullptr; } void log_notice(const char *msg, ...) { - if (logging::notice >= logging::level) { + if (logging::same_or_higher(logging::notice, logging::level)) { va_list ap; va_start(ap, msg); - logging::output(logging::notice, msg, ap); + logging::output_nocheckloglevel_ap(logging::notice, msg, ap); va_end(ap); } else logging::last = nullptr; } void log_warning(const char *msg, ...) { - if (logging::warning >= logging::level) { + if (logging::same_or_higher(logging::warning, logging::level)) { va_list ap; va_start(ap, msg); - logging::output(logging::warning, msg, ap); + logging::output_nocheckloglevel_ap(logging::warning, msg, ap); va_end(ap); } else logging::last = nullptr; } void log_error(const char *msg, ...) { - if (logging::error >= logging::level) { + if (logging::same_or_higher(logging::error, logging::level)) { va_list ap; va_start(ap, msg); - logging::output(logging::error, msg, ap); + logging::output_nocheckloglevel_ap(logging::error, msg, ap); va_end(ap); } else logging::last = nullptr; @@ -319,7 +364,7 @@ void log_trouble(const char *where, const char *what, int errnum) { } bool log_enabled(const logging::loglevel priority) { - return (priority >= logging::level); + return logging::same_or_higher(priority, logging::level); } void log_flush(void) { fflushall(); } diff --git a/libs/libmdbx/src/test/log.h b/libs/libmdbx/src/test/log.h index 7d6b4012f1..bb8f997b36 100644 --- a/libs/libmdbx/src/test/log.h +++ b/libs/libmdbx/src/test/log.h @@ -15,6 +15,7 @@ #pragma once #include "base.h" +#include "chrono.h" void __noreturn usage(void); void __noreturn __printf_args(1, 2) failure(const char *fmt, ...); @@ -24,27 +25,47 @@ const char *test_strerror(int errnum); namespace logging { enum loglevel { - extra, - trace, - verbose, - info, - notice, - warning, - error, - failure, + extra = MDBX_LOG_EXTRA, + trace = MDBX_LOG_TRACE, + debug = MDBX_LOG_DEBUG, + verbose = MDBX_LOG_VERBOSE, + notice = MDBX_LOG_NOTICE, + warning = MDBX_LOG_WARN, + error = MDBX_LOG_ERROR, + failure = MDBX_LOG_FATAL }; +inline bool lower(loglevel left, loglevel right) { + static_assert(MDBX_LOG_EXTRA > MDBX_LOG_FATAL, "WTF?"); + return left > right; +} + +inline bool same_or_higher(loglevel left, loglevel right) { + return left <= right; +} + const char *level2str(const loglevel level); -void setup(loglevel level, const std::string &prefix); +void setup(loglevel priority, const std::string &prefix); void setup(const std::string &prefix); -void setlevel(loglevel level); +void setlevel(loglevel priority); -bool output(const loglevel priority, const char *format, va_list ap); +void output_nocheckloglevel_ap(const loglevel priority, const char *format, + va_list ap); bool __printf_args(2, 3) output(const loglevel priority, const char *format, ...); bool feed_ap(const char *format, va_list ap); bool __printf_args(1, 2) feed(const char *format, ...); +void inline __printf_args(2, 3) + output_nocheckloglevel(const loglevel priority, const char *format, ...) { + va_list ap; + va_start(ap, format); + output_nocheckloglevel_ap(priority, format, ap); + va_end(ap); +} + +void progress_canary(bool active); + class local_suffix { protected: size_t trim_pos; @@ -66,8 +87,8 @@ public: void __printf_args(1, 2) log_extra(const char *msg, ...); void __printf_args(1, 2) log_trace(const char *msg, ...); +void __printf_args(1, 2) log_debug(const char *msg, ...); void __printf_args(1, 2) log_verbose(const char *msg, ...); -void __printf_args(1, 2) log_info(const char *msg, ...); void __printf_args(1, 2) log_notice(const char *msg, ...); void __printf_args(1, 2) log_warning(const char *msg, ...); void __printf_args(1, 2) log_error(const char *msg, ...); diff --git a/libs/libmdbx/src/test/long_stochastic.sh b/libs/libmdbx/src/test/long_stochastic.sh index 954d18268a..58ec6d5c6e 100644 --- a/libs/libmdbx/src/test/long_stochastic.sh +++ b/libs/libmdbx/src/test/long_stochastic.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if ! which make cc c++ tee lz4 >/dev/null; then echo "Please install the following prerequisites: make cc c++ tee lz4" >&2 exit 1 @@ -7,19 +7,42 @@ fi set -euo pipefail UNAME="$(uname -s 2>/dev/null || echo Unknown)" + +## NOTE: Valgrind could produce some false-positive warnings +## in multi-process environment with shared memory. +## For instance, when the process "A" explicitly marks a memory +## region as "undefined", the process "B" fill it, +## and after this process "A" read such region, etc. +#VALGRIND="valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt" + +############################################################################### +# 1. clean data from prev runs and examine available RAM + +if [[ -v VALGRIND && ! -z "$VALGRIND" ]]; then + rm -f valgrind-*.log +else + VALGRIND=time +fi + +WANNA_MOUNT=0 case ${UNAME} in Linux) MAKE=make if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then + for old_test_dir in $(ls -d /dev/shm/mdbx-test.[0-9]*); do + rm -rf $old_test_dir + done TESTDB_DIR="/dev/shm/mdbx-test.$$" fi mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* + if LC_ALL=C free | grep -q -i available; then ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 7) / 1024)) else ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 4) / 1024)) fi ;; + FreeBSD) MAKE=gmake if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then @@ -27,18 +50,46 @@ case ${UNAME} in umount $old_test_dir && rm -r $old_test_dir done TESTDB_DIR="/tmp/mdbx-test.$$" - rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR && mount -t tmpfs tmpfs $TESTDB_DIR + rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR + WANNA_MOUNT=1 else mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* fi + ram_avail_mb=$(($(LC_ALL=C vmstat -s | grep -ie '[0-9] pages free$' | cut -d p -f 1) * ($(LC_ALL=C vmstat -s | grep -ie '[0-9] bytes per page$' | cut -d b -f 1) / 1024) / 1024)) ;; + + Darwin) + MAKE=make + if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then + for vol in $(ls -d /Volumes/mdx[0-9]*[0-9]tst); do + disk=$(mount | grep $vol | cut -d ' ' -f 1) + echo "umount: volume $vol disk $disk" + hdiutil unmount $vol -force + hdiutil detach $disk + done + TESTDB_DIR="/Volumes/mdx$$tst" + WANNA_MOUNT=1 + else + mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* + fi + + pagesize=$(($(LC_ALL=C vm_stat | grep -o 'page size of [0-9]\+ bytes' | cut -d' ' -f 4) / 1024)) + freepages=$(LC_ALL=C vm_stat | grep '^Pages free:' | grep -o '[0-9]\+\.$' | cut -d'.' -f 1) + ram_avail_mb=$((pagesize * freepages / 1024)) + echo "pagesize ${pagesize}K, freepages ${freepages}, ram_avail_mb ${ram_avail_mb}" + + ;; + *) echo "FIXME: ${UNAME} not supported by this script" exit 2 ;; esac +############################################################################### +# 2. estimate reasonable RAM space for test-db + echo "=== ${ram_avail_mb}M RAM available" ram_reserve4logs_mb=1234 if [ $ram_avail_mb -lt $ram_reserve4logs_mb ]; then @@ -70,16 +121,47 @@ if [ $db_size_mb -gt 3072 ]; then fi echo "=== use ${db_size_mb}M for DB" -${MAKE} TESTDB=${TESTDB_DIR}/smoke.db TESTLOG=${TESTDB_DIR}/smoke.log check +############################################################################### +# 3. Create test-directory in ramfs/tmpfs, i.e. create/format/mount if required +case ${UNAME} in + Linux) + ;; + + FreeBSD) + if [[ WANNA_MOUNT ]]; then + mount -t tmpfs tmpfs $TESTDB_DIR + fi + ;; + + Darwin) + if [[ WANNA_MOUNT ]]; then + ramdisk_size_mb=$((42 + db_size_mb * 2 + ram_reserve4logs_mb)) + number_of_sectors=$((ramdisk_size_mb * 2048)) + ramdev=$(hdiutil attach -nomount ram://${number_of_sectors}) + diskutil erasevolume ExFAT "mdx$$tst" ${ramdev} + fi + ;; + + *) + echo "FIXME: ${UNAME} not supported by this script" + exit 2 + ;; +esac + +############################################################################### +# 4. Run basic test, i.e. `make check` + +${MAKE} TEST_DB=${TESTDB_DIR}/smoke.db TEST_LOG=${TESTDB_DIR}/smoke.log check rm -f ${TESTDB_DIR}/* ############################################################################### +# 5. run stochastic iterations function rep9 { printf "%*s" $1 '' | tr ' ' '9'; } function join { local IFS="$1"; shift; echo "$*"; } function bit2option { local -n arr=$1; (( ($2&(1<<$3)) != 0 )) && echo -n '+' || echo -n '-'; echo "${arr[$3]}"; } -options=(writemap coalesce lifo) +options=(writemap coalesce lifo notls) function bits2list { local -n arr=$1 @@ -95,13 +177,13 @@ function probe { echo "=============================================== $(date)" echo "${caption}: $*" rm -f ${TESTDB_DIR}/* \ - && ./mdbx_test --ignore-dbfull --repeat=42 --pathname=${TESTDB_DIR}/long.db "$@" | lz4 > ${TESTDB_DIR}/long.log.lz4 \ - && ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \ - && ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \ + && ${VALGRIND} ./mdbx_test --ignore-dbfull --repeat=42 --pathname=${TESTDB_DIR}/long.db "$@" | lz4 > ${TESTDB_DIR}/long.log.lz4 \ + && ${VALGRIND} ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \ + && ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ${VALGRIND} ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \ || (echo "FAILED"; exit 1) } -############################################################################### +#------------------------------------------------------------------------------ count=0 for nops in $(seq 2 6); do diff --git a/libs/libmdbx/src/test/main.cc b/libs/libmdbx/src/test/main.cc index 959359a515..c00837fa72 100644 --- a/libs/libmdbx/src/test/main.cc +++ b/libs/libmdbx/src/test/main.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -14,9 +14,100 @@ #include "test.h" +#if !(defined(_WIN32) || defined(_WIN64)) +#include <sys/resource.h> +#include <sys/time.h> +#endif /* !Windows */ + void __noreturn usage(void) { - printf("usage:\n" - "\tFIXME\n"); + puts( + "usage:\n" + " --help or -h Show this text\n" + "Common parameters:\n" + " --pathname=... Path and/or name of database files\n" + " --repeat=N Set repeat counter\n" + " --threads=N Number of thread (unsunpported for now)\n" + " --timeout=N[s|m|h|d] Set timeout in seconds/minutes/hours/days\n" + " --failfast[=YES/no] Lill all actors on first failure/error\n" + " --max-readers=N See mdbx_env_set_maxreaders() description\n" + " --max-tables=N Se mdbx_env_set_maxdbs() description\n" + " --dump-config[=YES/no] Dump entire test config before run\n" + " --progress[=YES/no] Enable/disable progress `canary`\n" + " --console[=yes/no] Enable/disable console-like output\n" + " --cleanup-before[=YES/no] Cleanup/remove and re-create database\n" + " --cleanup-after[=YES/no] Cleanup/remove database after completion\n" + "Database size control:\n" + " --pagesize=... Database page size: min, max, 256..65536\n" + " --size-lower=N[K|M|G|T] Lower-bound of size in Kb/Mb/Gb/Tb\n" + " --size-upper Upper-bound of size in Kb/Mb/Gb/Tb\n" + " --size Initial size in Kb/Mb/Gb/Tb\n" + " --shrink-threshold Shrink threshold in Kb/Mb/Gb/Tb\n" + " --growth-step Grow step in Kb/Mb/Gb/Tb\n" + "Predefined complext scenarios/cases:\n" + " --case=... Only `basic` scenario implemented for now\n" + " basic == Simultaneous multi-process execution\n" + " of test-actors: nested,hill,ttl,copy,append,jitter,try\n" + "Test actors:\n" + " --hill Fill-up and empty-down\n" + " by CRUD-operation quads\n" + " --ttl Stochastic time-to-live simulation\n" + " --nested Nested transactionы\n" + " with stochastic-size bellows\n" + " --jitter Jitter/delays simulation\n" + " --try Try write-transaction, no more\n" + " --copy Online copy/backup\n" + " --append Append-mode insertions\n" + " --dead.reader Dead-reader simulator\n" + " --dead.writer Dead-writer simulator\n" + "Actor options:\n" + " --batch.read=N Read-operations batch size\n" + " --batch.write=N Write-operations batch size\n" + " --delay=N | --no-delay (no)Delay test-actor before start\n" + " --wait4ops=N | --no-wait4ops (no)Wait for previous test-actor\n" + " completes # ops before start\n" + " --duration=N[s|m|h|d] Define running duration\n" + " --nops=N[K|M|G|T] Define number of operations/steps\n" + " --inject-writefault[=yes|NO] TBD (see the source code)\n" + " --drop[=yes|NO] Drop key-value space/table on " + "completion\n" + " --ignore-dbfull[=yes|NO] Ignore MDBX_MAP_FULL error\n" + " --speculum[=yes|NO] Use internal `speculum` to check " + "dataset\n" + "Keys and Value:\n" + " --keylen.min=N Minimal keys length\n" + " --keylen.max=N Miximal keys length\n" + " --datalen.min=N Minimal data length\n" + " --datalen.max=N Miximal data length\n" + " --keygen.width=N TBD (see the source code)\n" + " --keygen.mesh=N TBD (see the source code)\n" + " --keygen.seed=N TBD (see the source code)\n" + " --keygen.split=N TBD (see the source code)\n" + " --keygen.rotate=N TBD (see the source code)\n" + " --keygen.offset=N TBD (see the source code)\n" + " --keygen.case=random Generator case (only `random` for now)\n" + "Database operation mode:\n" + " --mode={[+-]FLAG}[,[+-]FLAG]...\n" + " nosubdir == MDBX_NOSUBDIR\n" + " rdonly == MDBX_RDONLY\n" + " nometasync == MDBX_NOMETASYNC\n" + " lifo == MDBX_LIFORECLAIM\n" + " coalesce == MDBX_COALESCE\n" + " nosync-safe == MDBX_SAFE_NOSYNC\n" + " writemap == MDBX_WRITEMAP\n" + " mapasync == MDBX_MAPASYNC\n" + " nosync-utterly == MDBX_UTTERLY_NOSYNC\n" + " perturb == MDBX_PAGEPERTURB\n" + " notls == MDBX_NOTLS\n" + " nordahead == MDBX_NORDAHEAD\n" + " nomeminit == MDBX_NOMEMINIT\n" + "Key-value space/table options:\n" + " --table={[+-]FLAG}[,[+-]FLAG]...\n" + " key.reverse == MDBX_REVERSEKEY\n" + " key.integer == MDBX_INTEGERKEY\n" + " data.dups == MDBX_DUPSORT\n" + " data.integer == MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT\n" + " data.fixed == MDBX_DUPFIXED | MDBX_DUPSORT\n" + " data.reverse == MDBX_REVERSEDUP | MDBX_DUPSORT\n"); exit(EXIT_FAILURE); } @@ -25,17 +116,15 @@ void __noreturn usage(void) { void actor_params::set_defaults(const std::string &tmpdir) { pathname_log = ""; loglevel = -#ifdef NDEBUG - logging::info; -#elif defined(_WIN32) || defined(_WIN64) +#if defined(NDEBUG) || defined(_WIN32) || defined(_WIN64) logging::verbose; #else logging::trace; #endif pathname_db = tmpdir + "mdbx-test.db"; - mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NORDAHEAD | - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_LIFORECLAIM; + mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NOMEMINIT | + MDBX_COALESCE | MDBX_LIFORECLAIM | MDBX_ACCEDE; table_flags = MDBX_DUPSORT; size_lower = -1; @@ -73,6 +162,7 @@ void actor_params::set_defaults(const std::string &tmpdir) { drop_table = false; ignore_dbfull = false; + speculum = false; max_readers = 42; max_tables = 42; @@ -82,7 +172,8 @@ void actor_params::set_defaults(const std::string &tmpdir) { global::config::cleanup_before = true; global::config::cleanup_after = true; global::config::failfast = true; - global::config::progress_indicator = osal_istty(STDERR_FILENO); + global::config::progress_indicator = true; + global::config::console_mode = osal_istty(STDERR_FILENO); } namespace global { @@ -103,6 +194,7 @@ bool cleanup_before; bool cleanup_after; bool failfast; bool progress_indicator; +bool console_mode; } /* namespace config */ } /* namespace global */ @@ -130,7 +222,7 @@ int main(int argc, char *const argv[]) { #endif /* _DEBUG */ if (argc < 2) - failure("No parameters given\n"); + failure("No parameters given. Try --help\n"); if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix, strlen(global::thunk_param_prefix)) == 0) @@ -139,6 +231,10 @@ int main(int argc, char *const argv[]) { ? EXIT_SUCCESS : EXIT_FAILURE; + if (argc == 2 && + (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0)) + usage(); + actor_params params; params.set_defaults(osal_tempdir()); global::config::dump_config = true; @@ -168,8 +264,8 @@ int main(int argc, char *const argv[]) { } if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize, - mdbx_limits_pgsize_min(), - mdbx_limits_pgsize_max())) { + int(mdbx_limits_pgsize_min()), + int(mdbx_limits_pgsize_max()))) { const unsigned keylen_max = params.mdbx_keylen_max(); if (params.keylen_min > keylen_max) params.keylen_min = keylen_max; @@ -182,6 +278,17 @@ int main(int argc, char *const argv[]) { params.datalen_max = datalen_max; continue; } + if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat, + config::no_scale)) + continue; + if (config::parse_option(argc, argv, narg, "threads", params.nthreads, + config::no_scale, 1, 64)) + continue; + if (config::parse_option(argc, argv, narg, "timeout", + global::config::timeout_duration_seconds, + config::duration, 1)) + continue; + if (config::parse_option_intptr(argc, argv, narg, "size-lower", params.size_lower, mdbx_limits_dbsize_min(params.pagesize), @@ -231,17 +338,6 @@ int main(int argc, char *const argv[]) { keycase_setup(value, params); continue; } - - if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat, - config::no_scale)) - continue; - if (config::parse_option(argc, argv, narg, "threads", params.nthreads, - config::no_scale, 1, 64)) - continue; - if (config::parse_option(argc, argv, narg, "timeout", - global::config::timeout_duration_seconds, - config::duration, 1)) - continue; if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min, config::no_scale, params.mdbx_keylen_min(), params.mdbx_keylen_max())) { @@ -296,6 +392,8 @@ int main(int argc, char *const argv[]) { if (config::parse_option(argc, argv, narg, "ignore-dbfull", params.ignore_dbfull)) continue; + if (config::parse_option(argc, argv, narg, "speculum", params.speculum)) + continue; if (config::parse_option(argc, argv, narg, "dump-config", global::config::dump_config)) continue; @@ -316,7 +414,7 @@ int main(int argc, char *const argv[]) { params.delaystart = 0; continue; } - if (config::parse_option(argc, argv, narg, "no-wait", nullptr)) { + if (config::parse_option(argc, argv, narg, "no-wait4ops", nullptr)) { params.waitfor_nops = 0; continue; } @@ -346,6 +444,10 @@ int main(int argc, char *const argv[]) { configure_actor(last_space_id, ac_deadwrite, value, params); continue; } + if (config::parse_option(argc, argv, narg, "try", nullptr)) { + configure_actor(last_space_id, ac_try, value, params); + continue; + } if (config::parse_option(argc, argv, narg, "copy", nullptr)) { configure_actor(last_space_id, ac_copy, value, params); continue; @@ -358,17 +460,24 @@ int main(int argc, char *const argv[]) { configure_actor(last_space_id, ac_ttl, value, params); continue; } + if (config::parse_option(argc, argv, narg, "nested", nullptr)) { + configure_actor(last_space_id, ac_nested, value, params); + continue; + } if (config::parse_option(argc, argv, narg, "failfast", global::config::failfast)) continue; if (config::parse_option(argc, argv, narg, "progress", global::config::progress_indicator)) continue; + if (config::parse_option(argc, argv, narg, "console", + global::config::console_mode)) + continue; if (*argv[narg] != '-') testcase_setup(argv[narg], params, last_space_id); else - failure("Unknown option '%s'\n", argv[narg]); + failure("Unknown option '%s'. Try --help\n", argv[narg]); } if (global::config::dump_config) @@ -456,8 +565,8 @@ int main(int argc, char *const argv[]) { if (!actor) continue; - log_info("actor #%u, id %d, pid %u: %s\n", actor->actor_id, - actor->space_id, pid, status2str(status)); + log_verbose("actor #%u, id %d, pid %ld: %s\n", actor->actor_id, + actor->space_id, (long)pid, status2str(status)); if (status > as_running) { left -= 1; if (status != as_successful) { @@ -480,9 +589,37 @@ int main(int argc, char *const argv[]) { log_notice("RESULT: %s\n", failed ? "Failed" : "Successful"); if (global::config::cleanup_before) { if (failed) - log_info("skip cleanup"); + log_verbose("skip cleanup"); else cleanup(); } + +#if !(defined(_WIN32) || defined(_WIN64)) + struct rusage spent; + if (!getrusage(global::singlemode ? RUSAGE_SELF : RUSAGE_CHILDREN, &spent)) { + log_notice("%6s: user %f, system %f", "CPU", + spent.ru_utime.tv_sec + spent.ru_utime.tv_usec * 1e-6, + spent.ru_stime.tv_sec + spent.ru_stime.tv_usec * 1e-6); +#if defined(__linux__) || defined(__gnu_linux__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || \ + defined(__bsdi__) || defined(__DragonFly__) || defined(__APPLE__) || \ + defined(__MACH__) || defined(__sun) + log_notice("%6s: read %ld, write %ld", "IOPs", spent.ru_inblock, + spent.ru_oublock); + if (spent.ru_maxrss > 0) + log_notice("%6s: %ld Kb", "RAM", + spent.ru_maxrss +#if defined(__sun) + * getpagesize() / 1024u +#elif defined(__APPLE__) + / 1024u +#endif + ); + log_notice("%6s: reclaims %ld, faults %ld, swaps %ld", "Paging", + spent.ru_minflt, spent.ru_majflt, spent.ru_nswap); +#endif /* Linux */ + } +#endif /* !Windows */ + return failed ? EXIT_FAILURE : EXIT_SUCCESS; } diff --git a/libs/libmdbx/src/test/osal-unix.cc b/libs/libmdbx/src/test/osal-unix.cc index 0157bace23..4b8694c708 100644 --- a/libs/libmdbx/src/test/osal-unix.cc +++ b/libs/libmdbx/src/test/osal-unix.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -21,112 +21,266 @@ #include <sys/wait.h> #include <unistd.h> -#ifdef __APPLE__ +#ifndef MDBX_LOCKING +#error "Opps, MDBX_LOCKING is undefined!" +#endif + +#if defined(__APPLE__) && (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008) #include "darwin/pthread_barrier.c" +#endif /* __APPLE__ && MDBX_LOCKING >= MDBX_LOCKING_POSIX2001 */ + +#if MDBX_LOCKING == MDBX_LOCKING_SYSV +#include <sys/ipc.h> +#include <sys/sem.h> +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 +#include <semaphore.h> + +#if __cplusplus >= 201103L +#include <atomic> +static __inline __maybe_unused int atomic_decrement(std::atomic_int *p) { + return std::atomic_fetch_sub(p, 1) - 1; +} +#else +static __inline __maybe_unused int atomic_decrement(volatile int *p) { +#if defined(__GNUC__) || defined(__clang__) + return __sync_sub_and_fetch(p, 1); +#elif defined(_MSC_VER) + STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile int)); + return _InterlockedDecrement((volatile long *)p); +#elif defined(__APPLE__) + return OSAtomicDecrement32Barrier((volatile int *)p); +#else +#error FIXME: Unsupported compiler #endif +} +#endif /* C++11 */ +#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX1988 */ + +#if MDBX_LOCKING == MDBX_LOCKING_SYSV +static int ipc; +static pid_t ipc_overlord_pid; +static void ipc_remove(void) { + if (ipc_overlord_pid == getpid()) + semctl(ipc, 0, IPC_RMID, nullptr); +} +#else struct shared_t { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 pthread_barrier_t barrier; pthread_mutex_t mutex; - size_t conds_size; - pthread_cond_t conds[1]; + size_t count; + pthread_cond_t events[1]; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + struct { +#if __cplusplus >= 201103L + std::atomic_int countdown; +#else + volatile int countdown; +#endif /* C++11 */ + sem_t sema; + } barrier; + size_t count; + sem_t events[1]; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ }; - static shared_t *shared; +#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ void osal_wait4barrier(void) { +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + struct sembuf op; + op.sem_num = 0; + op.sem_op = -1; + op.sem_flg = IPC_NOWAIT; + if (semop(ipc, &op, 1)) + failure_perror("semop(dec)", errno); + op.sem_op = 0; + op.sem_flg = 0; + if (semop(ipc, &op, 1)) + failure_perror("semop(wait)", errno); +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 assert(shared != nullptr && shared != MAP_FAILED); - int rc = pthread_barrier_wait(&shared->barrier); - if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { - failure_perror("pthread_barrier_wait(shared)", rc); - } + int err = pthread_barrier_wait(&shared->barrier); + if (err != 0 && err != PTHREAD_BARRIER_SERIAL_THREAD) + failure_perror("pthread_barrier_wait(shared)", err); +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + assert(shared != nullptr && shared != MAP_FAILED); + int err = (atomic_decrement(&shared->barrier.countdown) > 0 && + sem_wait(&shared->barrier.sema)) + ? errno + : 0; + if (err != 0) + failure_perror("sem_wait(shared)", err); + if (sem_post(&shared->barrier.sema)) + failure_perror("sem_post(shared)", errno); +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ } void osal_setup(const std::vector<actor_config> &actors) { +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + if (ipc_overlord_pid) + failure("ipc already created by %ld pid", (long)ipc_overlord_pid); + ipc_overlord_pid = getpid(); +#ifndef SEM_A +#define SEM_A S_IRUSR +#endif +#ifndef SEM_R +#define SEM_R S_IWUSR +#endif + ipc = semget(IPC_PRIVATE, actors.size() + 2, IPC_CREAT | SEM_A | SEM_R); + if (ipc < 0) + failure_perror("semget(IPC_PRIVATE, shared_sems)", errno); + if (atexit(ipc_remove)) + failure_perror("atexit(ipc_remove)", errno); + if (semctl(ipc, 0, SETVAL, (int)(actors.size() + 1))) + failure_perror("semctl(SETVAL.0, shared_sems)", errno); + for (size_t i = 1; i < actors.size() + 2; ++i) + if (semctl(ipc, i, SETVAL, 1)) + failure_perror("semctl(SETVAL.N, shared_sems)", errno); +#else assert(shared == nullptr); - - pthread_mutexattr_t mutexattr; - int rc = pthread_mutexattr_init(&mutexattr); - if (rc) - failure_perror("pthread_mutexattr_init()", rc); - rc = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED); - if (rc) - failure_perror("pthread_mutexattr_setpshared()", rc); - - pthread_barrierattr_t barrierattr; - rc = pthread_barrierattr_init(&barrierattr); - if (rc) - failure_perror("pthread_barrierattr_init()", rc); - rc = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED); - if (rc) - failure_perror("pthread_barrierattr_setpshared()", rc); - - pthread_condattr_t condattr; - rc = pthread_condattr_init(&condattr); - if (rc) - failure_perror("pthread_condattr_init()", rc); - rc = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED); - if (rc) - failure_perror("pthread_condattr_setpshared()", rc); - shared = (shared_t *)mmap( - nullptr, sizeof(shared_t) + actors.size() * sizeof(pthread_cond_t), - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + nullptr, sizeof(shared_t) + actors.size() * sizeof(shared->events[0]), + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS +#ifdef MAP_HASSEMAPHORE + | MAP_HASSEMAPHORE +#endif + , + -1, 0); if (MAP_FAILED == (void *)shared) failure_perror("mmap(shared_conds)", errno); - rc = pthread_mutex_init(&shared->mutex, &mutexattr); - if (rc) - failure_perror("pthread_mutex_init(shared)", rc); + shared->count = actors.size() + 1; - rc = pthread_barrier_init(&shared->barrier, &barrierattr, actors.size() + 1); - if (rc) - failure_perror("pthread_barrier_init(shared)", rc); +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + pthread_barrierattr_t barrierattr; + int err = pthread_barrierattr_init(&barrierattr); + if (err) + failure_perror("pthread_barrierattr_init()", err); + err = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED); + if (err) + failure_perror("pthread_barrierattr_setpshared()", err); + + err = pthread_barrier_init(&shared->barrier, &barrierattr, shared->count); + if (err) + failure_perror("pthread_barrier_init(shared)", err); + pthread_barrierattr_destroy(&barrierattr); - const size_t n = actors.size() + 1; - for (size_t i = 0; i < n; ++i) { - pthread_cond_t *event = &shared->conds[i]; - rc = pthread_cond_init(event, &condattr); - if (rc) - failure_perror("pthread_cond_init(shared)", rc); + pthread_mutexattr_t mutexattr; + err = pthread_mutexattr_init(&mutexattr); + if (err) + failure_perror("pthread_mutexattr_init()", err); + err = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED); + if (err) + failure_perror("pthread_mutexattr_setpshared()", err); + + pthread_condattr_t condattr; + err = pthread_condattr_init(&condattr); + if (err) + failure_perror("pthread_condattr_init()", err); + err = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED); + if (err) + failure_perror("pthread_condattr_setpshared()", err); + + err = pthread_mutex_init(&shared->mutex, &mutexattr); + if (err) + failure_perror("pthread_mutex_init(shared)", err); + + for (size_t i = 0; i < shared->count; ++i) { + pthread_cond_t *event = &shared->events[i]; + err = pthread_cond_init(event, &condattr); + if (err) + failure_perror("pthread_cond_init(shared)", err); log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i, - event); + __Wpedantic_format_voidptr(event)); } - shared->conds_size = actors.size() + 1; - - pthread_barrierattr_destroy(&barrierattr); pthread_condattr_destroy(&condattr); pthread_mutexattr_destroy(&mutexattr); +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + shared->barrier.countdown = shared->count; + if (sem_init(&shared->barrier.sema, true, 1)) + failure_perror("sem_init(shared.barrier)", errno); + for (size_t i = 0; i < shared->count; ++i) { + sem_t *event = &shared->events[i]; + if (sem_init(event, true, 0)) + failure_perror("sem_init(shared.event)", errno); + log_trace("osal_setup: event(shared sem_init) %" PRIuPTR " -> %p", i, + __Wpedantic_format_voidptr(event)); + } +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ +#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ } void osal_broadcast(unsigned id) { - assert(shared != nullptr && shared != MAP_FAILED); log_trace("osal_broadcast: event %u", id); - if (id >= shared->conds_size) +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + if (semctl(ipc, id + 1, SETVAL, 0)) + failure_perror("semctl(SETVAL)", errno); +#else + assert(shared != nullptr && shared != MAP_FAILED); + if (id >= shared->count) failure("osal_broadcast: id > limit"); - int rc = pthread_cond_broadcast(shared->conds + id); - if (rc) - failure_perror("sem_post(shared)", rc); +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + int err = pthread_cond_broadcast(shared->events + id); + if (err) + failure_perror("pthread_cond_broadcast(shared)", err); +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + if (sem_post(shared->events + id)) + failure_perror("sem_post(shared)", errno); +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ +#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ } int osal_waitfor(unsigned id) { - assert(shared != nullptr && shared != MAP_FAILED); - log_trace("osal_waitfor: event %u", id); - if (id >= shared->conds_size) +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + struct sembuf op; + memset(&op, 0, sizeof(op)); + op.sem_num = (short)(id + 1); + int rc = semop(ipc, &op, 1) ? errno : MDBX_SUCCESS; +#else + assert(shared != nullptr && shared != MAP_FAILED); + if (id >= shared->count) failure("osal_waitfor: id > limit"); +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ + MDBX_LOCKING == MDBX_LOCKING_POSIX2008 int rc = pthread_mutex_lock(&shared->mutex); if (rc != 0) failure_perror("pthread_mutex_lock(shared)", rc); - rc = pthread_cond_wait(shared->conds + id, &shared->mutex); + rc = pthread_cond_wait(shared->events + id, &shared->mutex); if (rc && rc != EINTR) failure_perror("pthread_cond_wait(shared)", rc); rc = pthread_mutex_unlock(&shared->mutex); if (rc != 0) failure_perror("pthread_mutex_unlock(shared)", rc); +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + int rc = sem_wait(shared->events + id) ? errno : 0; + if (rc == 0 && sem_post(shared->events + id)) + failure_perror("sem_post(shared)", errno); +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ +#endif /* MDBX_LOCKING != MDBX_LOCKING_SYSV */ return (rc == 0) ? true : false; } @@ -149,21 +303,69 @@ bool actor_config::osal_deserialize(const char *str, const char *end, //----------------------------------------------------------------------------- +static pid_t overlord_pid; + +static volatile sig_atomic_t sigusr1_head, sigusr2_head; +static void handler_SIGUSR(int signum) { + switch (signum) { + case SIGUSR1: + sigusr1_head += 1; + return; + case SIGUSR2: + sigusr2_head += 1; + return; + default: + abort(); + } +} + +bool osal_progress_push(bool active) { + if (overlord_pid) { + if (kill(overlord_pid, active ? SIGUSR1 : SIGUSR2)) + failure_perror("osal_progress_push: kill(overload)", errno); + return true; + } + + return false; +} + +//----------------------------------------------------------------------------- + static std::unordered_map<pid_t, actor_status> childs; -static void handler_SIGCHLD(int unused) { (void)unused; } +static volatile sig_atomic_t sigalarm_head; +static void handler_SIGCHLD(int signum) { + if (signum == SIGALRM) + sigalarm_head += 1; +} mdbx_pid_t osal_getpid(void) { return getpid(); } int osal_delay(unsigned seconds) { return sleep(seconds) ? errno : 0; } int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { - if (childs.empty()) - signal(SIGCHLD, handler_SIGCHLD); + if (childs.empty()) { + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = handler_SIGCHLD; + sigaction(SIGCHLD, &act, nullptr); + sigaction(SIGALRM, &act, nullptr); + act.sa_handler = handler_SIGUSR; + sigaction(SIGUSR1, &act, nullptr); + sigaction(SIGUSR2, &act, nullptr); + + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGCHLD); + sigaddset(&mask, SIGUSR1); + sigaddset(&mask, SIGUSR2); + sigprocmask(SIG_UNBLOCK, &mask, nullptr); + } pid = fork(); if (pid == 0) { + overlord_pid = getppid(); const bool result = test_execute(config); exit(result ? EXIT_SUCCESS : EXIT_FAILURE); } @@ -171,7 +373,8 @@ int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { if (pid < 0) return errno; - log_trace("osal_actor_start: fork pid %i for %u", pid, config.actor_id); + log_trace("osal_actor_start: fork pid %ld for %u", (long)pid, + config.actor_id); childs[pid] = as_running; return 0; } @@ -186,60 +389,67 @@ void osal_killall_actors(void) { } int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { - struct timespec ts; - ts.tv_nsec = 0; - ts.tv_sec = (timeout > INT_MAX) ? INT_MAX : timeout; -retry: - int status, options = WNOHANG; + static sig_atomic_t sigalarm_tail; + alarm(0) /* cancel prev timeout */; + sigalarm_tail = sigalarm_head /* reset timeout flag */; + + int options = WNOHANG; + if (timeout) { + alarm((timeout > INT_MAX) ? INT_MAX : timeout); + options = 0; + } + #ifdef WUNTRACED options |= WUNTRACED; #endif #ifdef WCONTINUED options |= WCONTINUED; #endif - pid = waitpid(0, &status, options); - - if (pid > 0) { - if (WIFEXITED(status)) - childs[pid] = - (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed; - else if (WCOREDUMP(status)) - childs[pid] = as_coredump; - else if (WIFSIGNALED(status)) - childs[pid] = as_killed; - else if (WIFSTOPPED(status)) - childs[pid] = as_debuging; - else if (WIFCONTINUED(status)) - childs[pid] = as_running; - else { - assert(false); - } - return 0; - } - if (pid == 0) { - /* child still running */ - if (ts.tv_sec == 0 && ts.tv_nsec == 0) - ts.tv_nsec = 1; - if (nanosleep(&ts, &ts) == 0) { - /* timeout and no signal from child */ - pid = 0; + while (sigalarm_tail == sigalarm_head) { + int status; + pid = waitpid(0, &status, options); + + if (pid > 0) { + if (WIFEXITED(status)) + childs[pid] = + (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed; + else if (WCOREDUMP(status)) + childs[pid] = as_coredump; + else if (WIFSIGNALED(status)) + childs[pid] = as_killed; + else if (WIFSTOPPED(status)) + childs[pid] = as_debugging; + else if (WIFCONTINUED(status)) + childs[pid] = as_running; + else { + assert(false); + } return 0; } - if (errno == EINTR) - goto retry; - } - switch (errno) { - case EINTR: - pid = 0; - return 0; + static sig_atomic_t sigusr1_tail, sigusr2_tail; + if (sigusr1_tail != sigusr1_head) { + sigusr1_tail = sigusr1_head; + logging::progress_canary(true); + if (pid < 0 && errno == EINTR) + continue; + } + if (sigusr2_tail != sigusr2_head) { + sigusr2_tail = sigusr2_head; + logging::progress_canary(false); + if (pid < 0 && errno == EINTR) + continue; + } - case ECHILD: - default: - pid = 0; - return errno; + if (pid == 0) + break; + + int err = errno; + if (err != EINTR) + return err; } + return 0 /* timeout */; } void osal_yield(void) { @@ -254,9 +464,17 @@ void osal_udelay(unsigned us) { static unsigned threshold_us; if (threshold_us == 0) { +#if defined(_POSIX_CPUTIME) && _POSIX_CPUTIME > -1 && \ + defined(CLOCK_PROCESS_CPUTIME_ID) if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) { int rc = errno; - failure_perror("clock_getres(CLOCK_PROCESS_CPUTIME_ID)", rc); + log_warning("clock_getres(CLOCK_PROCESS_CPUTIME_ID), failed errno %d", + rc); + } +#endif /* CLOCK_PROCESS_CPUTIME_ID */ + if (threshold_us == 0 && clock_getres(CLOCK_MONOTONIC, &ts)) { + int rc = errno; + failure_perror("clock_getres(CLOCK_MONOTONIC)", rc); } chrono::time threshold = chrono::from_timespec(ts); assert(threshold.seconds() == 0); diff --git a/libs/libmdbx/src/test/osal-windows.cc b/libs/libmdbx/src/test/osal-windows.cc index 975d8268f5..6de674651b 100644 --- a/libs/libmdbx/src/test/osal-windows.cc +++ b/libs/libmdbx/src/test/osal-windows.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -16,6 +16,7 @@ static std::unordered_map<unsigned, HANDLE> events; static HANDLE hBarrierSemaphore, hBarrierEvent; +static HANDLE hProgressActiveEvent, hProgressPassiveEvent; static int waitstatus2errcode(DWORD result) { switch (result) { @@ -85,6 +86,16 @@ void osal_setup(const std::vector<actor_config> &actors) { if (!hBarrierEvent) failure_perror("CreateEvent(BarrierEvent)", GetLastError()); hBarrierEvent = make_inheritable(hBarrierEvent); + + hProgressActiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!hProgressActiveEvent) + failure_perror("CreateEvent(ProgressActiveEvent)", GetLastError()); + hProgressActiveEvent = make_inheritable(hProgressActiveEvent); + + hProgressPassiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!hProgressPassiveEvent) + failure_perror("CreateEvent(ProgressPassiveEvent)", GetLastError()); + hProgressPassiveEvent = make_inheritable(hProgressPassiveEvent); } void osal_broadcast(unsigned id) { @@ -112,6 +123,8 @@ const std::string actor_config::osal_serialize(simple_checksum &checksum) const { checksum.push(hBarrierSemaphore); checksum.push(hBarrierEvent); + checksum.push(hProgressActiveEvent); + checksum.push(hProgressPassiveEvent); HANDLE hWait = INVALID_HANDLE_VALUE; if (wait4id) { @@ -125,8 +138,8 @@ actor_config::osal_serialize(simple_checksum &checksum) const { checksum.push(hSignal); } - return format("%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait, - hSignal); + return format("%p.%p.%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait, + hSignal, hProgressActiveEvent, hProgressPassiveEvent); } bool actor_config::osal_deserialize(const char *str, const char *end, @@ -137,17 +150,22 @@ bool actor_config::osal_deserialize(const char *str, const char *end, assert(hBarrierSemaphore == 0); assert(hBarrierEvent == 0); + assert(hProgressActiveEvent == 0); + assert(hProgressPassiveEvent == 0); assert(events.empty()); HANDLE hWait, hSignal; - if (sscanf_s(copy.c_str(), "%p.%p.%p.%p", &hBarrierSemaphore, &hBarrierEvent, - &hWait, &hSignal) != 4) { + if (sscanf_s(copy.c_str(), "%p.%p.%p.%p.%p.%p", &hBarrierSemaphore, + &hBarrierEvent, &hWait, &hSignal, &hProgressActiveEvent, + &hProgressPassiveEvent) != 6) { TRACE("<< osal_deserialize: failed\n"); return false; } checksum.push(hBarrierSemaphore); checksum.push(hBarrierEvent); + checksum.push(hProgressActiveEvent); + checksum.push(hProgressPassiveEvent); if (wait4id) { checksum.push(hWait); @@ -168,6 +186,17 @@ bool actor_config::osal_deserialize(const char *str, const char *end, typedef std::pair<HANDLE, actor_status> child; static std::unordered_map<mdbx_pid_t, child> childs; +bool osal_progress_push(bool active) { + if (!childs.empty()) { + if (!SetEvent(active ? hProgressActiveEvent : hProgressPassiveEvent)) + failure_perror("osal_progress_push: SetEvent(overlord.progress)", + GetLastError()); + return true; + } + + return false; +} + static void ArgvQuote(std::string &CommandLine, const std::string &Argument, bool Force = false) @@ -314,7 +343,7 @@ actor_status osal_actor_info(const mdbx_pid_t pid) { break; case EXCEPTION_BREAKPOINT: case EXCEPTION_SINGLE_STEP: - status = as_debuging; + status = as_debugging; break; case STATUS_CONTROL_C_EXIT: status = as_killed; @@ -344,32 +373,45 @@ void osal_killall_actors(void) { int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { std::vector<HANDLE> handles; - handles.reserve(childs.size()); + handles.reserve(childs.size() + 2); + handles.push_back(hProgressActiveEvent); + handles.push_back(hProgressPassiveEvent); for (const auto &pair : childs) if (pair.second.second <= as_running) handles.push_back(pair.second.first); - DWORD rc = - MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0], - (timeout > 60) ? 60 * 1000 : timeout * 1000, - QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0); + while (true) { + DWORD rc = + MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0], + (timeout > 60) ? 60 * 1000 : timeout * 1000, + QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0); - if (rc >= WAIT_OBJECT_0 && rc < WAIT_OBJECT_0 + handles.size()) { - pid = 0; - for (const auto &pair : childs) - if (pair.second.first == handles[rc - WAIT_OBJECT_0]) { - pid = pair.first; - break; - } - return 0; - } + if (rc == WAIT_OBJECT_0) { + logging::progress_canary(true); + continue; + } + if (rc == WAIT_OBJECT_0 + 1) { + logging::progress_canary(false); + continue; + } - if (rc == WAIT_TIMEOUT) { - pid = 0; - return 0; - } + if (rc >= WAIT_OBJECT_0 + 2 && rc < WAIT_OBJECT_0 + handles.size()) { + pid = 0; + for (const auto &pair : childs) + if (pair.second.first == handles[rc - WAIT_OBJECT_0]) { + pid = pair.first; + break; + } + return 0; + } - return waitstatus2errcode(rc); + if (rc == WAIT_TIMEOUT) { + pid = 0; + return 0; + } + + return waitstatus2errcode(rc); + } } void osal_yield(void) { SwitchToThread(); } diff --git a/libs/libmdbx/src/test/osal.h b/libs/libmdbx/src/test/osal.h index 5acf7ad094..6d0e1c4eb6 100644 --- a/libs/libmdbx/src/test/osal.h +++ b/libs/libmdbx/src/test/osal.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -26,6 +26,8 @@ void osal_killall_actors(void); int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout); void osal_wait4barrier(void); +bool osal_progress_push(bool active); + mdbx_pid_t osal_getpid(void); int osal_delay(unsigned seconds); void osal_udelay(unsigned us); diff --git a/libs/libmdbx/src/test/pcrf/CMakeLists.txt b/libs/libmdbx/src/test/pcrf/CMakeLists.txt index 399c33f88f..8bd3e3d859 100644 --- a/libs/libmdbx/src/test/pcrf/CMakeLists.txt +++ b/libs/libmdbx/src/test/pcrf/CMakeLists.txt @@ -1,7 +1,5 @@ set(TARGET pcrf_test) -project(${TARGET}) - add_executable(${TARGET} pcrf_test.c) - +target_include_directories(${TARGET} PRIVATE "${PROJECT_SOURCE_DIR}") target_link_libraries(${TARGET} mdbx) diff --git a/libs/libmdbx/src/test/pcrf/pcrf_test.c b/libs/libmdbx/src/test/pcrf/pcrf_test.c index 213c8b1d80..206bd21d4e 100644 --- a/libs/libmdbx/src/test/pcrf/pcrf_test.c +++ b/libs/libmdbx/src/test/pcrf/pcrf_test.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2017 Leonid Yuriev <leo@yuriev.ru>. + * Copyright 2016-2019 Leonid Yuriev <leo@yuriev.ru>. * Copyright 2015 Vladimir Romanov * <https://www.linkedin.com/in/vladimirromanov>, Yota Lab. * @@ -36,7 +36,7 @@ (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \ (int)((addr)&0xff) -char opt_db_path[PATH_MAX] = "/root/lmdbx_bench2"; +char opt_db_path[PATH_MAX] = "./mdbx_bench2"; static MDBX_env *env; #define REC_COUNT 10240000 int64_t ids[REC_COUNT * 10]; @@ -105,12 +105,12 @@ static void db_connect() { MDBX_dbi dbi_ip; MDBX_CHECK(mdbx_env_create(&env)); - MDBX_CHECK( - mdbx_env_set_mapsize(env, REC_COUNT * sizeof(session_data_t) * 10)); + MDBX_CHECK(mdbx_env_set_geometry( + env, 0, 0, REC_COUNT * sizeof(session_data_t) * 10, -1, -1, -1)); MDBX_CHECK(mdbx_env_set_maxdbs(env, 30)); MDBX_CHECK(mdbx_env_open(env, opt_db_path, MDBX_CREATE | MDBX_WRITEMAP | MDBX_MAPASYNC | - MDBX_NOSYNC | MDBX_LIFORECLAIM, + MDBX_SAFE_NOSYNC | MDBX_LIFORECLAIM, 0664)); MDBX_txn *txn; @@ -126,7 +126,7 @@ static void db_connect() { printf("Connection open\n"); } -static void create_record(int64_t record_id) { +static void create_record(uint64_t record_id) { MDBX_dbi dbi_session; MDBX_dbi dbi_session_id; MDBX_dbi dbi_event; @@ -136,11 +136,13 @@ static void create_record(int64_t record_id) { session_data_t data; // transaction init snprintf(data.session_id1, sizeof(data.session_id1), - "prefix%02ld_%02ld.fill.fill.fill.fill.fill.fill;%ld", - record_id % 3 + 1, record_id % 9 + 1, record_id); + "prefix%02u_%02u.fill.fill.fill.fill.fill.fill;%" PRIu64, + (unsigned)(record_id % 3) + 1, (unsigned)(record_id % 9) + 1, + record_id); snprintf(data.session_id2, sizeof(data.session_id2), - "dprefix%ld;%ld.fill.fill.;suffix", record_id, - record_id % 1000000000 + 99999); + "dprefix%" PRIu64 ";%" PRIu64 ".fill.fill.;suffix", record_id, + (record_id + UINT64_C(1442695040888963407)) % + UINT64_C(6364136223846793005)); snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); event.obj_id = record_id; @@ -241,8 +243,9 @@ static void get_db_stat(const char *db, int64_t *ms_branch_pages, MDBX_CHECK(mdbx_dbi_open(txn, db, MDBX_CREATE, &dbi)); MDBX_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat))); mdbx_txn_abort(txn); - printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n", db, - stat.ms_branch_pages, stat.ms_depth, stat.ms_entries, + printf("%15s | %15" PRIu64 " | %5u | %10" PRIu64 " | %10" PRIu64 + " | %11" PRIu64 " |\n", + db, stat.ms_branch_pages, stat.ms_depth, stat.ms_entries, stat.ms_leaf_pages, stat.ms_overflow_pages); (*ms_branch_pages) += stat.ms_branch_pages; (*ms_leaf_pages) += stat.ms_leaf_pages; @@ -253,8 +256,8 @@ static void periodic_stat(void) { int64_t ms_leaf_pages = 0; MDBX_stat mst; MDBX_envinfo mei; - MDBX_CHECK(mdbx_env_stat(env, &mst, sizeof(mst))); - MDBX_CHECK(mdbx_env_info(env, &mei, sizeof(mei))); + MDBX_CHECK(mdbx_env_stat_ex(env, NULL, &mst, sizeof(mst))); + MDBX_CHECK(mdbx_env_info_ex(env, NULL, &mei, sizeof(mei))); printf("Environment Info\n"); printf(" Pagesize: %u\n", mst.ms_psize); if (mei.mi_geo.lower != mei.mi_geo.upper) { @@ -287,8 +290,8 @@ static void periodic_stat(void) { get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); - printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, - "", "", ms_leaf_pages, ""); + printf("%15s | %15" PRIu64 " | %5s | %10s | %10" PRIu64 " | %11s |\n", "", + ms_branch_pages, "", "", ms_leaf_pages, ""); static int64_t prev_add_count; static int64_t prev_del_count; @@ -297,26 +300,32 @@ static void periodic_stat(void) { static int64_t t = -1; if (t > 0) { int64_t delta = (getClockUs() - t); - printf( - "CPS: add %ld, delete %ld, items processed - %ldK data=%ldK key=%ldK\n", - (mdbx_add_count - prev_add_count) * 1000000 / delta, - (mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024, - mdbx_data_size / 1024, mdbx_key_size / 1024); - printf("usage data=%ld%%", ((mdbx_data_size + mdbx_key_size) * 100) / - ((ms_leaf_pages + ms_branch_pages) * 4096)); + printf("CPS: add %" PRIu64 ", delete %" PRIu64 + ", items processed - %" PRIu64 "K data=%" PRIu64 "K key=%" PRIu64 + "K\n", + (mdbx_add_count - prev_add_count) * 1000000 / delta, + (mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024, + mdbx_data_size / 1024, mdbx_key_size / 1024); + printf("usage data=%" PRIu64 "%%", + ((mdbx_data_size + mdbx_key_size) * 100) / + ((ms_leaf_pages + ms_branch_pages) * 4096)); if (prev_add_time != mdbx_add_time) { - printf(" Add : %ld c/s", (mdbx_add_count - prev_add_count) * 1000000 / - (mdbx_add_time - prev_add_time)); + printf(" Add : %" PRIu64 " c/s", (mdbx_add_count - prev_add_count) * + 1000000 / + (mdbx_add_time - prev_add_time)); } if (prev_del_time != mdbx_del_time) { - printf(" Del : %ld c/s", (mdbx_del_count - prev_del_count) * 1000000 / - (mdbx_del_time - prev_del_time)); + printf(" Del : %" PRIu64 " c/s", (mdbx_del_count - prev_del_count) * + 1000000 / + (mdbx_del_time - prev_del_time)); } if (mdbx_add_time) { - printf(" tAdd : %ld c/s", mdbx_add_count * 1000000 / mdbx_add_time); + printf(" tAdd : %" PRIu64 " c/s", + mdbx_add_count * 1000000 / mdbx_add_time); } if (mdbx_del_time) { - printf(" tDel : %ld c/s", mdbx_del_count * 1000000 / mdbx_del_time); + printf(" tDel : %" PRIu64 " c/s", + mdbx_del_count * 1000000 / mdbx_del_time); } puts(""); } @@ -385,14 +394,14 @@ int main(int argc, char **argv) { id = get_id_from_pool(); delete_record(id); } - // for (i = 0; i < 50; i++) { - // int64_t id = obj_id++; - // create_record(id); - // add_id_to_pool(id); - // } - // int64_t id = obj_id++; - // create_record(id); - // add_id_to_pool(id); + // for (i = 0; i < 50; i++) { + // int64_t id = obj_id++; + // create_record(id); + // add_id_to_pool(id); + // } + // int64_t id = obj_id++; + // create_record(id); + // add_id_to_pool(id); int64_t now = getClockUs(); if ((now - t) > 10000000L) { periodic_stat(); diff --git a/libs/libmdbx/src/test/test.cc b/libs/libmdbx/src/test/test.cc index e9e925e9a9..ff1676274b 100644 --- a/libs/libmdbx/src/test/test.cc +++ b/libs/libmdbx/src/test/test.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -37,6 +37,8 @@ const char *testcase2str(const actor_testcase testcase) { return "append"; case ac_ttl: return "ttl"; + case ac_nested: + return "nested"; } } @@ -45,8 +47,8 @@ const char *status2str(actor_status status) { default: assert(false); return "?!"; - case as_debuging: - return "debuging"; + case as_debugging: + return "debugging"; case as_running: return "running"; case as_successful: @@ -76,15 +78,16 @@ const char *keygencase2str(const keygen_case keycase) { //----------------------------------------------------------------------------- -int testcase::oom_callback(MDBX_env *env, int pid, mdbx_tid_t tid, uint64_t txn, - unsigned gap, int retry) { +int testcase::oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, + uint64_t txn, unsigned gap, size_t space, + int retry) { testcase *self = (testcase *)mdbx_env_get_userctx(env); if (retry == 0) - log_notice("oom_callback: waitfor pid %u, thread %" PRIuPTR - ", txn #%" PRIu64 ", gap %d", - pid, (size_t)tid, txn, gap); + log_notice("oom_callback: waitfor pid %lu, thread %" PRIuPTR + ", txn #%" PRIu64 ", gap %d, scape %zu", + (long)pid, (size_t)tid, txn, gap, space); if (self->should_continue(true)) { osal_yield(); @@ -186,7 +189,8 @@ int testcase::breakable_commit() { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { rc = err; err = mdbx_txn_abort(txn); - if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH)) + if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && + err != MDBX_BAD_TXN)) failure_perror("mdbx_txn_abort()", err); } else failure_perror("mdbx_txn_commit()", err); @@ -196,6 +200,20 @@ int testcase::breakable_commit() { return rc; } +unsigned testcase::txn_underutilization_x256(MDBX_txn *txn) const { + if (txn) { + MDBX_txn_info info; + int err = mdbx_txn_info(txn, &info, false); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_info()", err); + const size_t left = size_t(info.txn_space_leftover); + const size_t total = + size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty); + return (unsigned)(left / (total >> 8)); + } + return 0; +} + void testcase::txn_end(bool abort) { log_trace(">> txn_end(%s)", abort ? "abort" : "commit"); assert(txn_guard); @@ -203,7 +221,8 @@ void testcase::txn_end(bool abort) { MDBX_txn *txn = txn_guard.release(); if (abort) { int err = mdbx_txn_abort(txn); - if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH)) + if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && + err != MDBX_BAD_TXN)) failure_perror("mdbx_txn_abort()", err); } else { txn_inject_writefault(txn); @@ -215,18 +234,18 @@ void testcase::txn_end(bool abort) { log_trace("<< txn_end(%s)", abort ? "abort" : "commit"); } -void testcase::cursor_open(unsigned dbi) { - log_trace(">> cursor_open(%u)", dbi); +void testcase::cursor_open(MDBX_dbi handle) { + log_trace(">> cursor_open(%u)", handle); assert(!cursor_guard); assert(txn_guard); MDBX_cursor *cursor = nullptr; - int rc = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); + int rc = mdbx_cursor_open(txn_guard.get(), handle, &cursor); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_cursor_open()", rc); cursor_guard.reset(cursor); - log_trace("<< cursor_open(%u)", dbi); + log_trace("<< cursor_open(%u)", handle); } void testcase::cursor_close() { @@ -264,8 +283,9 @@ void testcase::txn_inject_writefault(MDBX_txn *txn) { if (config.params.inject_writefaultn && txn) { if (config.params.inject_writefaultn <= nops_completed && (mdbx_txn_flags(txn) & MDBX_RDONLY) == 0) { - log_info("== txn_inject_writefault(): got %u nops or more, inject FAULT", - config.params.inject_writefaultn); + log_verbose( + "== txn_inject_writefault(): got %u nops or more, inject FAULT", + config.params.inject_writefaultn); log_flush(); #if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) TerminateProcess(GetCurrentProcess(), 42); @@ -305,21 +325,9 @@ bool testcase::wait4start() { } void testcase::kick_progress(bool active) const { - chrono::time now = chrono::now_motonic(); - if (active) { - static int last_point = -1; - int point = (now.fixedpoint >> 29) & 3; - if (point != last_point) { - last.progress_timestamp = now; - fprintf(stderr, "%c\b", "-\\|/"[last_point = point]); - fflush(stderr); - } - } else if (now.fixedpoint - last.progress_timestamp.fixedpoint > - chrono::from_seconds(2).fixedpoint) { - last.progress_timestamp = now; - fprintf(stderr, "%c\b", "@*"[now.utc & 1]); - fflush(stderr); - } + if (!global::config::progress_indicator) + return; + logging::progress_canary(active); } void testcase::report(size_t nops_done) { @@ -328,11 +336,10 @@ void testcase::report(size_t nops_done) { return; nops_completed += nops_done; - log_verbose("== complete +%" PRIuPTR " iteration, total %" PRIuPTR " done", - nops_done, nops_completed); + log_debug("== complete +%" PRIuPTR " iteration, total %" PRIuPTR " done", + nops_done, nops_completed); - if (global::config::progress_indicator) - kick_progress(true); + kick_progress(true); if (config.signal_nops && !signalled && config.signal_nops <= nops_completed) { @@ -387,7 +394,7 @@ bool testcase::should_continue(bool check_timeout_only) const { nops_completed >= config.params.test_nops) result = false; - if (result && global::config::progress_indicator) + if (result) kick_progress(false); return result; @@ -430,14 +437,14 @@ void testcase::update_canary(uint64_t increment) { log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y); } -int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &dbi) { +int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &handle) { db_open(); int err, retry_left = 42; for (;;) { txn_begin(false); - dbi = db_table_open(true); - db_table_clear(dbi); + handle = db_table_open(true); + db_table_clear(handle); err = breakable_commit(); if (likely(err == MDBX_SUCCESS)) { txn_begin(false); @@ -464,7 +471,7 @@ MDBX_dbi testcase::db_table_open(bool create) { failure("snprintf(tablename): %d", rc); tablename = tablename_buf; } - log_verbose("use %s table", tablename ? tablename : "MAINDB"); + log_debug("use %s table", tablename ? tablename : "MAINDB"); MDBX_dbi handle = 0; int rc = mdbx_dbi_open(txn_guard.get(), tablename, @@ -490,9 +497,9 @@ void testcase::db_table_drop(MDBX_dbi handle) { } } -void testcase::db_table_clear(MDBX_dbi handle) { +void testcase::db_table_clear(MDBX_dbi handle, MDBX_txn *txn) { log_trace(">> testcase::db_table_clear, handle %u", handle); - int rc = mdbx_drop(txn_guard.get(), handle, false); + int rc = mdbx_drop(txn ? txn : txn_guard.get(), handle, false); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_drop(delete=false)", rc); log_trace("<< testcase::db_table_clear"); @@ -510,7 +517,7 @@ void testcase::db_table_close(MDBX_dbi handle) { void testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, MDBX_val expected_valued) { MDBX_val actual_value = expected_valued; - int rc = mdbx_get2(txn_guard.get(), handle, &key2check, &actual_value); + int rc = mdbx_get_nearest(txn_guard.get(), handle, &key2check, &actual_value); if (unlikely(rc != MDBX_SUCCESS)) failure_perror(step, rc); if (!is_samedata(&actual_value, &expected_valued)) @@ -560,6 +567,9 @@ bool test_execute(const actor_config &config_const) { case ac_ttl: test.reset(new testcase_ttl(config, pid)); break; + case ac_nested: + test.reset(new testcase_nested(config, pid)); + break; default: test.reset(new testcase(config, pid)); break; @@ -582,13 +592,13 @@ bool test_execute(const actor_config &config_const) { } if (config.params.nrepeat == 1) - log_info("test successed"); + log_verbose("test successed"); else { if (config.params.nrepeat) - log_info("test successed (iteration %zi of %zi)", iter, - size_t(config.params.nrepeat)); + log_verbose("test successed (iteration %zi of %zi)", iter, + size_t(config.params.nrepeat)); else - log_info("test successed (iteration %zi)", iter); + log_verbose("test successed (iteration %zi)", iter); config.params.keygen.seed += INT32_C(0xA4F4D37B); } @@ -599,3 +609,135 @@ bool test_execute(const actor_config &config_const) { return false; } } + +//----------------------------------------------------------------------------- + +int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, + unsigned flags) { + int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags); + if (err == MDBX_SUCCESS && config.params.speculum) { + const auto S_key = S(akey); + const auto S_data = S(adata); + const bool inserted = speculum.emplace(S_key, S_data).second; + assert(inserted); + (void)inserted; + } + return err; +} + +int testcase::replace(const keygen::buffer &akey, + const keygen::buffer &new_data, + const keygen::buffer &old_data, unsigned flags) { + if (config.params.speculum) { + const auto S_key = S(akey); + const auto S_old = S(old_data); + const auto S_new = S(new_data); + const auto removed = speculum.erase(SET::key_type(S_key, S_old)); + assert(removed == 1); + (void)removed; + const bool inserted = speculum.emplace(S_key, S_new).second; + assert(inserted); + (void)inserted; + } + return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value, + &old_data->value, flags); +} + +int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) { + if (config.params.speculum) { + const auto S_key = S(akey); + const auto S_data = S(adata); + const auto removed = speculum.erase(SET::key_type(S_key, S_data)); + assert(removed == 1); + (void)removed; + } + return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value); +} + +bool testcase::speculum_verify() { + if (!config.params.speculum) + return true; + + if (!txn_guard) + txn_begin(true); + + char dump_key[128], dump_value[128]; + char dump_mkey[128], dump_mvalue[128]; + + MDBX_cursor *cursor; + int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); + if (err != MDBX_SUCCESS) + failure_perror("mdbx_cursor_open()", err); + + bool rc = true; + MDBX_val akey, avalue; + MDBX_val mkey, mvalue; + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST); + + unsigned extra = 0, lost = 0, n = 0; + assert(std::is_sorted(speculum.cbegin(), speculum.cend(), ItemCompare(this))); + auto it = speculum.cbegin(); + while (true) { + if (err != MDBX_SUCCESS) { + akey.iov_len = avalue.iov_len = 0; + akey.iov_base = avalue.iov_base = nullptr; + } + const auto S_key = S(akey); + const auto S_data = S(avalue); + if (it != speculum.cend()) { + mkey.iov_base = (void *)it->first.c_str(); + mkey.iov_len = it->first.size(); + mvalue.iov_base = (void *)it->second.c_str(); + mvalue.iov_len = it->second.size(); + } + if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first && + S_data == it->second) { + ++it; + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); + } else if (err == MDBX_SUCCESS && + (it == speculum.cend() || S_key < it->first || + (S_key == it->first && S_data < it->second))) { + extra += 1; + if (it != speculum.cend()) { + log_error("extra pair %u/%u: db{%s, %s} < mi{%s, %s}", n, extra, + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } else { + log_error("extra pair %u/%u: db{%s, %s} < mi.END", n, extra, + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value))); + } + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); + rc = false; + } else if (it != speculum.cend() && + (err == MDBX_NOTFOUND || S_key > it->first || + (S_key == it->first && S_data > it->second))) { + lost += 1; + if (err == MDBX_NOTFOUND) { + log_error("lost pair %u/%u: db.END > mi{%s, %s}", n, lost, + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } else { + log_error("lost pair %u/%u: db{%s, %s} > mi{%s, %s}", n, lost, + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } + ++it; + rc = false; + } else if (err == MDBX_NOTFOUND && it == speculum.cend()) { + break; + } else if (err != MDBX_SUCCESS) { + failure_perror("mdbx_cursor_get()", err); + } else { + assert(!"WTF?"); + } + n += 1; + } + + mdbx_cursor_close(cursor); + return rc; +} diff --git a/libs/libmdbx/src/test/test.h b/libs/libmdbx/src/test/test.h index fb5ad4ee83..630059ac3f 100644 --- a/libs/libmdbx/src/test/test.h +++ b/libs/libmdbx/src/test/test.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -22,6 +22,24 @@ #include "osal.h" #include "utils.h" +#include <deque> +#include <set> +#include <stack> +#include <tuple> + +#ifndef HAVE_cxx17_std_string_view +#if __cplusplus >= 201703L && __has_include(<string_view>) +#include <string_view> +#define HAVE_cxx17_std_string_view 1 +#else +#define HAVE_cxx17_std_string_view 0 +#endif +#endif /* HAVE_cxx17_std_string_view */ + +#if HAVE_cxx17_std_string_view +#include <string_view> +#endif + bool test_execute(const actor_config &config); std::string thunk_param(const actor_config &config); void testcase_setup(const char *casename, actor_params ¶ms, @@ -49,25 +67,26 @@ extern bool cleanup_before; extern bool cleanup_after; extern bool failfast; extern bool progress_indicator; +extern bool console_mode; } /* namespace config */ } /* namespace global */ //----------------------------------------------------------------------------- -struct db_deleter : public std::unary_function<void, MDBX_env *> { +struct db_deleter /* : public std::unary_function<void, MDBX_env *> */ { void operator()(MDBX_env *env) const { mdbx_env_close(env); } }; -struct txn_deleter : public std::unary_function<void, MDBX_txn *> { +struct txn_deleter /* : public std::unary_function<void, MDBX_txn *> */ { void operator()(MDBX_txn *txn) const { int rc = mdbx_txn_abort(txn); if (rc) - log_trouble(mdbx_func_, "mdbx_txn_abort()", rc); + log_trouble(__func__, "mdbx_txn_abort()", rc); } }; -struct cursor_deleter : public std::unary_function<void, MDBX_cursor *> { +struct cursor_deleter /* : public std::unary_function<void, MDBX_cursor *> */ { void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); } }; @@ -79,9 +98,45 @@ typedef std::unique_ptr<MDBX_cursor, cursor_deleter> scoped_cursor_guard; class testcase { protected: +#if HAVE_cxx17_std_string_view + using data_view = std::string_view; +#else + using data_view = std::string; +#endif + static inline data_view S(const MDBX_val &v) { + return data_view(static_cast<const char *>(v.iov_base), v.iov_len); + } + static inline data_view S(const keygen::buffer &b) { return S(b->value); } + + using Item = std::pair<std::string, std::string>; + struct ItemCompare { + const testcase *context; + ItemCompare(const testcase *owner) : context(owner) {} + + bool operator()(const Item &a, const Item &b) const { + MDBX_val va, vb; + va.iov_base = (void *)a.first.data(); + va.iov_len = a.first.size(); + vb.iov_base = (void *)b.first.data(); + vb.iov_len = b.first.size(); + int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb); + if (cmp == 0 && + (context->config.params.table_flags & MDBX_DUPSORT) != 0) { + va.iov_base = (void *)a.second.data(); + va.iov_len = a.second.size(); + vb.iov_base = (void *)b.second.data(); + vb.iov_len = b.second.size(); + cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb); + } + return cmp < 0; + } + }; + using SET = std::set<Item, ItemCompare>; + const actor_config &config; const mdbx_pid_t pid; + MDBX_dbi dbi; scoped_db_guard db_guard; scoped_txn_guard txn_guard; scoped_cursor_guard cursor_guard; @@ -95,12 +150,23 @@ protected: struct { mdbx_canary canary; - mutable chrono::time progress_timestamp; } last; - static int oom_callback(MDBX_env *env, int pid, mdbx_tid_t tid, uint64_t txn, - unsigned gap, int retry); + SET speculum; + bool speculum_verify(); + int insert(const keygen::buffer &akey, const keygen::buffer &adata, + unsigned flags); + int replace(const keygen::buffer &akey, const keygen::buffer &new_value, + const keygen::buffer &old_value, unsigned flags); + int remove(const keygen::buffer &akey, const keygen::buffer &adata); + static int oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, + uint64_t txn, unsigned gap, size_t space, int retry); + + bool is_nested_txn_available() const { + return (config.params.mode_flags & MDBX_WRITEMAP) == 0; + } + void kick_progress(bool active) const; void db_prepare(); void db_open(); void db_close(); @@ -109,21 +175,21 @@ protected: void txn_end(bool abort); int breakable_restart(); void txn_restart(bool abort, bool readonly, unsigned flags = 0); - void cursor_open(unsigned dbi); + void cursor_open(MDBX_dbi handle); void cursor_close(); void txn_inject_writefault(void); void txn_inject_writefault(MDBX_txn *txn); void fetch_canary(); void update_canary(uint64_t increment); - void kick_progress(bool active) const; void checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, MDBX_val expected_valued); + unsigned txn_underutilization_x256(MDBX_txn *txn) const; MDBX_dbi db_table_open(bool create); void db_table_drop(MDBX_dbi handle); - void db_table_clear(MDBX_dbi handle); + void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr); void db_table_close(MDBX_dbi handle); - int db_open__begin__table_create_open_clean(MDBX_dbi &dbi); + int db_open__begin__table_create_open_clean(MDBX_dbi &handle); bool wait4start(); void report(size_t nops_done); @@ -131,13 +197,12 @@ protected: bool should_continue(bool check_timeout_only = false) const; void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key, - keygen::buffer &out_value, keygen::serial_t data_age = 0) { - keyvalue_maker.pair(serial, out_key, out_value, data_age); + keygen::buffer &out_value, keygen::serial_t data_age) { + keyvalue_maker.pair(serial, out_key, out_value, data_age, false); } - void generate_pair(const keygen::serial_t serial, - keygen::serial_t data_age = 0) { - generate_pair(serial, key, data, data_age); + void generate_pair(const keygen::serial_t serial) { + keyvalue_maker.pair(serial, key, data, 0, true); } bool mode_readonly() const { @@ -146,7 +211,8 @@ protected: public: testcase(const actor_config &config, const mdbx_pid_t pid) - : config(config), pid(pid), signalled(false), nops_completed(0) { + : config(config), pid(pid), signalled(false), nops_completed(0), + speculum(ItemCompare(this)) { start_timestamp.reset(); memset(&last, 0, sizeof(last)); } @@ -161,49 +227,52 @@ class testcase_ttl : public testcase { public: testcase_ttl(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_hill : public testcase { + using inherited = testcase; + SET speculum_commited; + public: testcase_hill(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} - bool run(); + : testcase(config, pid), speculum_commited(ItemCompare(this)) {} + bool run() override; }; class testcase_append : public testcase { public: testcase_append(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_deadread : public testcase { public: testcase_deadread(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_deadwrite : public testcase { public: testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_jitter : public testcase { public: testcase_jitter(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_try : public testcase { public: testcase_try(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_copy : public testcase { @@ -214,5 +283,31 @@ public: testcase_copy(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid), copy_pathname(config.params.pathname_db + "-copy") {} - bool run(); + bool run() override; +}; + +class testcase_nested : public testcase { + using inherited = testcase; + using FIFO = std::deque<std::pair<uint64_t, unsigned>>; + + uint64_t serial; + FIFO fifo; + std::stack<std::tuple<scoped_txn_guard, uint64_t, FIFO, SET>> stack; + + bool trim_tail(unsigned window_width); + bool grow_head(unsigned head_count); + bool pop_txn(bool abort); + bool pop_txn() { + return pop_txn(inherited::is_nested_txn_available() ? flipcoin_x3() + : flipcoin_x2()); + } + void push_txn(); + bool stochastic_breakable_restart_with_nested(bool force_restart = false); + +public: + testcase_nested(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool setup() override; + bool run() override; + bool teardown() override; }; diff --git a/libs/libmdbx/src/test/test.vcxproj b/libs/libmdbx/src/test/test.vcxproj deleted file mode 100644 index 93de00e3a0..0000000000 --- a/libs/libmdbx/src/test/test.vcxproj +++ /dev/null @@ -1,209 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> - <ItemGroup Label="ProjectConfigurations"> - <ProjectConfiguration Include="Debug|Win32"> - <Configuration>Debug</Configuration> - <Platform>Win32</Platform> - </ProjectConfiguration> - <ProjectConfiguration Include="Release|Win32"> - <Configuration>Release</Configuration> - <Platform>Win32</Platform> - </ProjectConfiguration> - <ProjectConfiguration Include="Debug|x64"> - <Configuration>Debug</Configuration> - <Platform>x64</Platform> - </ProjectConfiguration> - <ProjectConfiguration Include="Release|x64"> - <Configuration>Release</Configuration> - <Platform>x64</Platform> - </ProjectConfiguration> - </ItemGroup> - <ItemGroup> - <ProjectReference Include="..\dll.vcxproj"> - <Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project> - </ProjectReference> - </ItemGroup> - <PropertyGroup Label="Globals"> - <ProjectGuid>{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}</ProjectGuid> - <Keyword>Win32Proj</Keyword> - <RootNamespace>mdbxtest</RootNamespace> - <WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion> - </PropertyGroup> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration"> - <ConfigurationType>Application</ConfigurationType> - <UseDebugLibraries>true</UseDebugLibraries> - <PlatformToolset>v140</PlatformToolset> - <CharacterSet>MultiByte</CharacterSet> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> - <ConfigurationType>Application</ConfigurationType> - <UseDebugLibraries>false</UseDebugLibraries> - <PlatformToolset>v140</PlatformToolset> - <WholeProgramOptimization>true</WholeProgramOptimization> - <CharacterSet>MultiByte</CharacterSet> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> - <ConfigurationType>Application</ConfigurationType> - <UseDebugLibraries>true</UseDebugLibraries> - <PlatformToolset>v140</PlatformToolset> - <CharacterSet>MultiByte</CharacterSet> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> - <ConfigurationType>Application</ConfigurationType> - <UseDebugLibraries>false</UseDebugLibraries> - <PlatformToolset>v140</PlatformToolset> - <WholeProgramOptimization>true</WholeProgramOptimization> - <CharacterSet>MultiByte</CharacterSet> - </PropertyGroup> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> - <ImportGroup Label="ExtensionSettings"> - </ImportGroup> - <ImportGroup Label="Shared"> - </ImportGroup> - <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> - <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> - </ImportGroup> - <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> - <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> - </ImportGroup> - <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> - <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> - </ImportGroup> - <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> - <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> - </ImportGroup> - <PropertyGroup Label="UserMacros" /> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> - <LinkIncremental>true</LinkIncremental> - <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> - <IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> - <TargetName>mdbx_test</TargetName> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> - <LinkIncremental>false</LinkIncremental> - <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> - <IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> - <TargetName>mdbx_test</TargetName> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> - <LinkIncremental>true</LinkIncremental> - <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> - <IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> - <TargetName>mdbx_test</TargetName> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> - <LinkIncremental>false</LinkIncremental> - <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> - <IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> - <TargetName>mdbx_test</TargetName> - </PropertyGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> - <ClCompile> - <PrecompiledHeader>Use</PrecompiledHeader> - <WarningLevel>Level4</WarningLevel> - <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;MDBX_DEBUG=1;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions> - <SDLCheck>true</SDLCheck> - <PrecompiledHeaderFile>test.h</PrecompiledHeaderFile> - <TreatWarningAsError>true</TreatWarningAsError> - </ClCompile> - <Link> - <SubSystem>Console</SubSystem> - <GenerateDebugInformation>true</GenerateDebugInformation> - <AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies> - </Link> - </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> - <ClCompile> - <PrecompiledHeader>Use</PrecompiledHeader> - <WarningLevel>Level4</WarningLevel> - <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>_DEBUG;_CONSOLE;MDBX_DEBUG=1;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions> - <SDLCheck>true</SDLCheck> - <PrecompiledHeaderFile>test.h</PrecompiledHeaderFile> - <TreatWarningAsError>true</TreatWarningAsError> - </ClCompile> - <Link> - <SubSystem>Console</SubSystem> - <GenerateDebugInformation>true</GenerateDebugInformation> - <AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies> - </Link> - </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> - <ClCompile> - <WarningLevel>Level4</WarningLevel> - <PrecompiledHeader>Use</PrecompiledHeader> - <Optimization>MaxSpeed</Optimization> - <FunctionLevelLinking>true</FunctionLevelLinking> - <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions> - <SDLCheck>true</SDLCheck> - <PrecompiledHeaderFile>test.h</PrecompiledHeaderFile> - <OmitFramePointers>true</OmitFramePointers> - </ClCompile> - <Link> - <SubSystem>Console</SubSystem> - <EnableCOMDATFolding>true</EnableCOMDATFolding> - <OptimizeReferences>true</OptimizeReferences> - <GenerateDebugInformation>true</GenerateDebugInformation> - <AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies> - </Link> - </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> - <ClCompile> - <WarningLevel>Level4</WarningLevel> - <PrecompiledHeader>Use</PrecompiledHeader> - <Optimization>MaxSpeed</Optimization> - <FunctionLevelLinking>true</FunctionLevelLinking> - <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions> - <SDLCheck>true</SDLCheck> - <PrecompiledHeaderFile>test.h</PrecompiledHeaderFile> - <OmitFramePointers>true</OmitFramePointers> - </ClCompile> - <Link> - <SubSystem>Console</SubSystem> - <EnableCOMDATFolding>true</EnableCOMDATFolding> - <OptimizeReferences>true</OptimizeReferences> - <GenerateDebugInformation>true</GenerateDebugInformation> - <AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies> - </Link> - </ItemDefinitionGroup> - <ItemGroup> - <ClInclude Include="base.h" /> - <ClInclude Include="chrono.h" /> - <ClInclude Include="config.h" /> - <ClInclude Include="keygen.h" /> - <ClInclude Include="log.h" /> - <ClInclude Include="osal.h" /> - <ClInclude Include="test.h" /> - <ClInclude Include="utils.h" /> - </ItemGroup> - <ItemGroup> - <ClCompile Include="ttl.cc" /> - <ClCompile Include="append.cc" /> - <ClCompile Include="cases.cc" /> - <ClCompile Include="chrono.cc" /> - <ClCompile Include="config.cc" /> - <ClCompile Include="copy.cc" /> - <ClCompile Include="dead.cc" /> - <ClCompile Include="hill.cc" /> - <ClCompile Include="try.cc" /> - <ClCompile Include="jitter.cc" /> - <ClCompile Include="keygen.cc" /> - <ClCompile Include="log.cc" /> - <ClCompile Include="main.cc" /> - <ClCompile Include="osal-windows.cc"> - <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader> - <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader> - <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader> - <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader> - </ClCompile> - <ClCompile Include="test.cc" /> - <ClCompile Include="utils.cc" /> - </ItemGroup> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> - <ImportGroup Label="ExtensionTargets"> - </ImportGroup> -</Project> diff --git a/libs/libmdbx/src/test/try.cc b/libs/libmdbx/src/test/try.cc index 150abd36de..adb0113096 100644 --- a/libs/libmdbx/src/test/try.cc +++ b/libs/libmdbx/src/test/try.cc @@ -1,4 +1,4 @@ -#include "test.h" +#include "test.h" bool testcase_try::run() { db_open(); diff --git a/libs/libmdbx/src/test/ttl.cc b/libs/libmdbx/src/test/ttl.cc index b2650f0c13..782a8b4d56 100644 --- a/libs/libmdbx/src/test/ttl.cc +++ b/libs/libmdbx/src/test/ttl.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -29,11 +29,10 @@ static unsigned edge2count(uint64_t edge, unsigned count_max) { } bool testcase_ttl::run() { - MDBX_dbi dbi; int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err)); - return true; + return false; } /* LY: тест "эмуляцией time-to-live": @@ -54,18 +53,8 @@ bool testcase_ttl::run() { /* LY: для параметризации используем подходящие параметры, которые не имеют * здесь смысла в первоначальном значении. */ - const unsigned window_max_lower = -#ifdef __APPLE__ - 333; -#else - 999; -#endif - const unsigned count_max_lower = -#ifdef __APPLE__ - 333; -#else - 999; -#endif + const unsigned window_max_lower = 333; + const unsigned count_max_lower = 333; const unsigned window_max = (config.params.batch_read > window_max_lower) ? config.params.batch_read @@ -73,8 +62,8 @@ bool testcase_ttl::run() { const unsigned count_max = (config.params.batch_write > count_max_lower) ? config.params.batch_write : count_max_lower; - log_info("ttl: using `batch_read` value %u for window_max", window_max); - log_info("ttl: using `batch_write` value %u for count_max", count_max); + log_verbose("ttl: using `batch_read` value %u for window_max", window_max); + log_verbose("ttl: using `batch_write` value %u for count_max", count_max); uint64_t seed = prng64_map2_white(config.params.keygen.seed) + config.actor_id; @@ -87,14 +76,16 @@ bool testcase_ttl::run() { std::deque<std::pair<uint64_t, unsigned>> fifo; uint64_t serial = 0; + bool rc = false; while (should_continue()) { const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; - const unsigned window_width = edge2window(salt, window_max); + const unsigned window_width = + flipcoin_x4() ? 0 : edge2window(salt, window_max); unsigned head_count = edge2count(salt, count_max); - log_verbose("ttl: step #%zu (serial %" PRIu64 - ", window %u, count %u) salt %" PRIu64, - nops_completed, serial, window_width, head_count, salt); + log_debug("ttl: step #%zu (serial %" PRIu64 + ", window %u, count %u) salt %" PRIu64, + nops_completed, serial, window_width, head_count, salt); if (window_width) { while (fifo.size() > window_width) { @@ -104,7 +95,7 @@ bool testcase_ttl::run() { tail_count); fifo.pop_back(); for (unsigned n = 0; n < tail_count; ++n) { - log_trace("ttl: remove-tail %" PRIu64, serial); + log_trace("ttl: remove-tail %" PRIu64, tail_serial); generate_pair(tail_serial); err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value); if (unlikely(err != MDBX_SUCCESS)) { @@ -158,7 +149,9 @@ bool testcase_ttl::run() { serial = fifo.front().first; fifo.pop_front(); } + report(1); + rc = true; } bailout: @@ -170,10 +163,10 @@ bailout: err = breakable_commit(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err)); - return true; + return false; } } else db_table_close(dbi); } - return true; + return rc; } diff --git a/libs/libmdbx/src/test/utils.cc b/libs/libmdbx/src/test/utils.cc index d9b3538b99..311cf544ef 100644 --- a/libs/libmdbx/src/test/utils.cc +++ b/libs/libmdbx/src/test/utils.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -331,6 +331,9 @@ double double_from_upper(uint64_t salt) { } bool flipcoin() { return bleach32((uint32_t)entropy_ticks()) & 1; } +bool flipcoin_x2() { return (bleach32((uint32_t)entropy_ticks()) & 3) == 0; } +bool flipcoin_x3() { return (bleach32((uint32_t)entropy_ticks()) & 7) == 0; } +bool flipcoin_x4() { return (bleach32((uint32_t)entropy_ticks()) & 15) == 0; } bool jitter(unsigned probability_percent) { const uint32_t top = UINT32_MAX - UINT32_MAX % 100; diff --git a/libs/libmdbx/src/test/utils.h b/libs/libmdbx/src/test/utils.h index d1b859acd4..2a5a54de6d 100644 --- a/libs/libmdbx/src/test/utils.h +++ b/libs/libmdbx/src/test/utils.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -151,7 +151,7 @@ template <typename T> static __inline T load(const void *ptr) { #if defined(_MSC_VER) && \ (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) return *(const T __unaligned *)ptr; -#elif UNALIGNED_OK +#elif MDBX_UNALIGNED_OK return *(const T *)ptr; #else T local; @@ -161,14 +161,14 @@ template <typename T> static __inline T load(const void *ptr) { memcpy(&local, (const T *)ptr, sizeof(T)); #endif /* __GNUC__ || __clang__ */ return local; -#endif /* UNALIGNED_OK */ +#endif /* MDBX_UNALIGNED_OK */ } template <typename T> static __inline void store(void *ptr, const T &value) { #if defined(_MSC_VER) && \ (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) *((T __unaligned *)ptr) = value; -#elif UNALIGNED_OK +#elif MDBX_UNALIGNED_OK *(volatile T *)ptr = value; #else #if defined(__GNUC__) || defined(__clang__) @@ -176,7 +176,7 @@ template <typename T> static __inline void store(void *ptr, const T &value) { #else memcpy(ptr, &value, sizeof(T)); #endif /* __GNUC__ || __clang__ */ -#endif /* UNALIGNED_OK */ +#endif /* MDBX_UNALIGNED_OK */ } } /* namespace unaligned */ @@ -355,5 +355,8 @@ uint64_t prng64(void); void prng_fill(void *ptr, size_t bytes); bool flipcoin(); +bool flipcoin_x2(); +bool flipcoin_x3(); +bool flipcoin_x4(); bool jitter(unsigned probability_percent); void jitter_delay(bool extra = false); |