diff options
author | George Hazan <ghazan@miranda.im> | 2018-09-14 16:33:56 +0300 |
---|---|---|
committer | George Hazan <ghazan@miranda.im> | 2018-09-14 16:33:56 +0300 |
commit | 50d176bfe78d4b5ffd829a874e503facef398e7d (patch) | |
tree | 3048927747b53a7c79ef73a5671d9ec912322382 /libs/libmdbx/src/test | |
parent | cc03b109287f4c818a4d6df09cbfa48784e1e4a6 (diff) |
merge with libmdbx release
Diffstat (limited to 'libs/libmdbx/src/test')
-rw-r--r-- | libs/libmdbx/src/test/config.cc | 133 | ||||
-rw-r--r-- | libs/libmdbx/src/test/config.h | 35 | ||||
-rw-r--r-- | libs/libmdbx/src/test/gc.sh | 81 | ||||
-rw-r--r-- | libs/libmdbx/src/test/hill.cc | 8 | ||||
-rw-r--r-- | libs/libmdbx/src/test/keygen.cc | 83 | ||||
-rw-r--r-- | libs/libmdbx/src/test/keygen.h | 10 | ||||
-rw-r--r-- | libs/libmdbx/src/test/log.cc | 38 | ||||
-rw-r--r-- | libs/libmdbx/src/test/log.h | 10 | ||||
-rw-r--r-- | libs/libmdbx/src/test/main.cc | 98 | ||||
-rw-r--r-- | libs/libmdbx/src/test/osal-windows.cc | 8 | ||||
-rw-r--r-- | libs/libmdbx/src/test/test.cc | 40 | ||||
-rw-r--r-- | libs/libmdbx/src/test/utils.cc | 2 |
12 files changed, 422 insertions, 124 deletions
diff --git a/libs/libmdbx/src/test/config.cc b/libs/libmdbx/src/test/config.cc index cbff68ce4e..619bd35727 100644 --- a/libs/libmdbx/src/test/config.cc +++ b/libs/libmdbx/src/test/config.cc @@ -43,6 +43,11 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) { *value = argv[narg + 1]; + if (strcmp(*value, "default") == 0) { + if (!default_value) + failure("Option '--%s' doen't accept default value\n", option); + *value = default_value; + } ++narg; return true; } @@ -57,9 +62,15 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty) { + return parse_option(argc, argv, narg, option, value, allow_empty, + allow_empty ? "" : nullptr); +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty, + const char *default_value) { const char *value_cstr; - if (!parse_option(argc, argv, narg, option, &value_cstr, - allow_empty ? "" : nullptr)) + if (!parse_option(argc, argv, narg, option, &value_cstr, default_value)) return false; if (!allow_empty && strlen(value_cstr) == 0) @@ -75,7 +86,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, if (!parse_option(argc, argv, narg, option, &list)) return false; - mask = 0; + unsigned clear = 0; while (*list) { if (*list == ',' || *list == ' ' || *list == '\t') { ++list; @@ -83,14 +94,21 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, } const char *const comma = strchr(list, ','); + const bool strikethrough = *list == '-' || *list == '~'; + if (strikethrough || *list == '+') + ++list; + else + mask = clear; const size_t len = (comma) ? comma - list : strlen(list); const option_verb *scan = verbs; + while (true) { if (!scan->verb) failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list, option); if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) { - mask |= scan->mask; + mask = strikethrough ? mask & ~scan->mask : mask | scan->mask; + clear = strikethrough ? clear & ~scan->mask : clear | scan->mask; list += len; break; } @@ -103,15 +121,36 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint64_t &value, const scale_mode scale, - const uint64_t minval, const uint64_t maxval) { + const uint64_t minval, const uint64_t maxval, + const uint64_t default_value) { const char *value_cstr; if (!parse_option(argc, argv, narg, option, &value_cstr)) return false; + if (default_value && strcmp(value_cstr, "default") == 0) { + value = default_value; + return true; + } + + if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0) { + value = minval; + return true; + } + + if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0) { + value = maxval; + return true; + } + char *suffix = nullptr; errno = 0; - unsigned long raw = strtoul(value_cstr, &suffix, 0); + unsigned long long raw = strtoull(value_cstr, &suffix, 0); + if ((suffix && *suffix) || errno) { + suffix = nullptr; + errno = 0; + raw = strtoull(value_cstr, &suffix, 10); + } if (errno) failure("Option '--%s' expects a numeric value (%s)\n", option, test_strerror(errno)); @@ -167,28 +206,58 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &value, const scale_mode scale, - const unsigned minval, const unsigned maxval) { + const unsigned minval, const unsigned maxval, + const unsigned default_value) { uint64_t huge; - if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval)) + if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval, + default_value)) return false; value = (unsigned)huge; return true; } bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint8_t &value, const uint8_t minval, const uint8_t maxval) { + uint8_t &value, const uint8_t minval, const uint8_t maxval, + const uint8_t default_value) { uint64_t huge; - if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval)) + if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval, + default_value)) return false; value = (uint8_t)huge; return true; } bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int64_t &value, const int64_t minval, const int64_t maxval, + const int64_t default_value) { + uint64_t proxy = (uint64_t)value; + if (parse_option(argc, argv, narg, option, proxy, config::binary, + (uint64_t)minval, (uint64_t)maxval, + (uint64_t)default_value)) { + value = (int64_t)proxy; + return true; + } + return false; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int32_t &value, const int32_t minval, const int32_t maxval, + const int32_t default_value) { + uint64_t proxy = (uint64_t)value; + if (parse_option(argc, argv, narg, option, proxy, config::binary, + (uint64_t)minval, (uint64_t)maxval, + (uint64_t)default_value)) { + value = (int32_t)proxy; + return true; + } + return false; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool &value) { - const char *value_cstr = NULL; + const char *value_cstr = nullptr; if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) { const char *current = argv[narg]; if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) { @@ -257,7 +326,7 @@ static void dump_verbs(const char *caption, size_t bits, ++verbs; } - logging::feed("\n"); + logging::feed("%s\n", (*comma == '\0') ? "none" : ""); } static void dump_duration(const char *caption, unsigned duration) { @@ -288,8 +357,12 @@ void dump(const char *title) { : i->params.pathname_log.c_str()); } - log_info("database: %s, size %" PRIu64 "\n", i->params.pathname_db.c_str(), - i->params.size); + log_info("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR + ", %i %i, %i]\n", + i->params.pathname_db.c_str(), i->params.size_now, + i->params.size_lower, i->params.size_upper, + i->params.shrink_threshold, i->params.growth_step, + i->params.pagesize); dump_verbs("mode", i->params.mode_flags, mode_bits); dump_verbs("table", i->params.table_flags, table_bits); @@ -306,7 +379,13 @@ void dump(const char *title) { log_info("threads %u\n", i->params.nthreads); - log_info("keygen.case: %s\n", keygencase2str(i->params.keygen.keycase)); + log_info( + "keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64 + ", split %u/%u\n", + keygencase2str(i->params.keygen.keycase), i->params.keygen.width, + i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset, + i->params.keygen.split, + i->params.keygen.width - i->params.keygen.split); log_info("keygen.seed: %u\n", i->params.keygen.seed); log_info("key: minlen %u, maxlen %u\n", i->params.keylen_min, i->params.keylen_max); @@ -469,3 +548,27 @@ bool actor_config::deserialize(const char *str, actor_config &config) { TRACE("<< actor_config::deserialize: OK\n"); return true; } + +unsigned actor_params::mdbx_keylen_min() const { + return (table_flags & MDBX_INTEGERKEY) ? 4 : 0; +} + +unsigned actor_params::mdbx_keylen_max() const { + return (table_flags & MDBX_INTEGERKEY) + ? 8 + : std::min((unsigned)mdbx_limits_keysize_max(pagesize), + (unsigned)UINT16_MAX); +} + +unsigned actor_params::mdbx_datalen_min() const { + return (table_flags & MDBX_INTEGERDUP) ? 4 : 0; +} + +unsigned actor_params::mdbx_datalen_max() const { + return (table_flags & MDBX_INTEGERDUP) + ? 8 + : std::min((table_flags & MDBX_DUPSORT) + ? (unsigned)mdbx_limits_keysize_max(pagesize) + : (unsigned)MDBX_MAXDATASIZE, + (unsigned)UINT16_MAX); +} diff --git a/libs/libmdbx/src/test/config.h b/libs/libmdbx/src/test/config.h index 86f37fbed8..2d0fede046 100644 --- a/libs/libmdbx/src/test/config.h +++ b/libs/libmdbx/src/test/config.h @@ -63,6 +63,10 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty = false); bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty, + const char *default_value); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool &value); struct option_verb { @@ -75,16 +79,25 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint64_t &value, const scale_mode scale, - const uint64_t minval = 0, const uint64_t maxval = INT64_MAX); + const uint64_t minval = 0, const uint64_t maxval = INT64_MAX, + const uint64_t default_value = 0); bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &value, const scale_mode scale, - const unsigned minval = 0, const unsigned maxval = INT32_MAX); + const unsigned minval = 0, const unsigned maxval = INT32_MAX, + const unsigned default_value = 0); bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint8_t &value, const uint8_t minval = 0, - const uint8_t maxval = 255); + const uint8_t maxval = 255, const uint8_t default_value = 0); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int64_t &value, const int64_t minval, const int64_t maxval, + const int64_t default_value = -1); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int32_t &value, const int32_t minval, const int32_t maxval, + const int32_t default_value = -1); //----------------------------------------------------------------------------- #pragma pack(push, 1) @@ -121,6 +134,8 @@ struct keygen_params_pod { * Иначе говоря, нет смысла в со-координации генерации паттернов для * ключей и значений. Более того, генерацию значений всегда необходимо * рассматривать в контексте связки с одним значением ключа. + * - Тем не менее, во всех случаях достаточно важным является равномерная + * всех возможных сочетаний длин ключей и данных. * * width: * Большинство тестов предполагают создание или итерирование некоторого @@ -156,7 +171,7 @@ struct keygen_params_pod { * псевдо-случайные значений ключей без псевдо-случайности в значениях. * * Такое ограничение соответствуют внутренней алгоритмике libmdbx. Проще - * говоря мы можем проверить движок псевдо-случайной последовательностью + * говоря, мы можем проверить движок псевдо-случайной последовательностью * ключей на таблицах без дубликатов (без multi-value), а затем проверить * корректность работу псевдо-случайной последовательностью значений на * таблицах с дубликатами (с multi-value), опционально добавляя @@ -203,7 +218,12 @@ struct actor_params_pod { unsigned mode_flags; unsigned table_flags; - uint64_t size; + intptr_t size_lower; + intptr_t size_now; + intptr_t size_upper; + int shrink_threshold; + int growth_step; + int pagesize; unsigned test_duration; unsigned test_nops; @@ -246,6 +266,11 @@ struct actor_params : public config::actor_params_pod { std::string pathname_log; std::string pathname_db; void set_defaults(const std::string &tmpdir); + + unsigned mdbx_keylen_min() const; + unsigned mdbx_keylen_max() const; + unsigned mdbx_datalen_min() const; + unsigned mdbx_datalen_max() const; }; struct actor_config : public config::actor_config_pod { diff --git a/libs/libmdbx/src/test/gc.sh b/libs/libmdbx/src/test/gc.sh new file mode 100644 index 0000000000..bddd92af24 --- /dev/null +++ b/libs/libmdbx/src/test/gc.sh @@ -0,0 +1,81 @@ +#!/bin/bash +set -euo pipefail +make check +TESTDB_PREFIX=${1:-/dev/shm/mdbx-gc-test}. + +function rep9 { printf "%*s" $1 '' | tr ' ' '9'; } +function join { local IFS="$1"; shift; echo "$*"; } +function bit2option { local -n arr=$1; (( ($2&(1<<$3)) != 0 )) && echo -n '+' || echo -n '-'; echo "${arr[$3]}"; } + +options=(writemap coalesce lifo) + +function bits2list { + local -n arr=$1 + local i + local list=() + for ((i=0; i<${#arr[@]}; ++i)) do + list[$i]=$(bit2option $1 $2 $i) + done + join , "${list[@]}" +} + +function probe { + echo "=============================================== $(date)" + echo "${caption}: $*" + rm -f ${TESTDB_PREFIX}* \ + && ./mdbx_test --pathname=${TESTDB_PREFIX}db "$@" | lz4 > log.lz4 \ + && ./mdbx_chk -nvvv ${TESTDB_PREFIX}db | tee ${TESTDB_PREFIX}chk \ + || (echo "FAILED"; exit 1) +} + +############################################################################### + +caption="Failfast #1" probe \ + --pagesize=min --size=6G --table=+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --nops=99999 --batch.write=9 --mode=-writemap,-coalesce,+lifo --keygen.seed=248240655 --hill + +caption="Failfast #2" probe \ + --pagesize=min --size=6G --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=999999 --batch.write=999 --mode=+writemap,+coalesce,+lifo --keygen.seed=259083046 --hill + +caption="Failfast #3" probe \ + --pagesize=min --size=6G --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=999999 --batch.write=999 --mode=+writemap,+coalesce,+lifo --keygen.seed=522365681 --hill + +caption="Failfast #4" probe \ + --pagesize=min --size=6G --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=999999 --batch.write=9999 --mode=-writemap,+coalesce,+lifo --keygen.seed=866083781 --hill + +caption="Failfast #5" probe \ + --pagesize=min --size=6G --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=999999 --batch.write=999 --mode=+writemap,-coalesce,+lifo --keygen.seed=246539192 --hill + +caption="Failfast #6" probe \ + --pagesize=min --size=6G --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=999999 --batch.write=999 --mode=+writemap,+coalesce,+lifo --keygen.seed=540406278 --hill + +caption="Failfast #7" probe \ + --pagesize=min --size=6G --table=+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --nops=999999 --batch.write=999 --mode=-writemap,+coalesce,+lifo --keygen.seed=619798690 --hill + +count=0 +for nops in {2..7}; do + for ((wbatch=nops-1; wbatch > 0; --wbatch)); do + loops=$((1111/nops + 2)) + for ((rep=0; rep++ < loops; )); do + for ((bits=2**${#options[@]}; --bits >= 0; )); do + seed=$(date +%N) + caption="Probe #$((++count)) w/o-dups, repeat ${rep} of ${loops}" probe \ + --pagesize=min --size=6G --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \ + --keygen.seed=${seed} --hill + caption="Probe #$((++count)) with-dups, repeat ${rep} of ${loops}" probe \ + --pagesize=min --size=6G --table=+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \ + --keygen.seed=${seed} --hill + done + done + done +done + +echo "=== ALL DONE ====================== $(date)" diff --git a/libs/libmdbx/src/test/hill.cc b/libs/libmdbx/src/test/hill.cc index c9115784d4..0193c4f2f5 100644 --- a/libs/libmdbx/src/test/hill.cc +++ b/libs/libmdbx/src/test/hill.cc @@ -65,7 +65,9 @@ bool testcase_hill::run() { ? MDBX_NODUPDATA : MDBX_NODUPDATA | MDBX_NOOVERWRITE; const unsigned update_flags = - MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE; + (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE + : MDBX_NODUPDATA; uint64_t serial_count = 0; unsigned txn_nops = 0; @@ -115,7 +117,7 @@ bool testcase_hill::run() { rc = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value, &a_data_1->value, update_flags); if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_put(update-a: 1->0)", rc); + failure_perror("mdbx_replace(update-a: 1->0)", rc); if (++txn_nops >= config.params.batch_write) { txn_restart(false, false); @@ -156,8 +158,6 @@ bool testcase_hill::run() { a_serial); generate_pair(a_serial, a_key, a_data_0, 0); generate_pair(a_serial, a_key, a_data_1, age_shift); - if (a_serial == 808) - log_trace("!!!"); int rc = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value, &a_data_0->value, update_flags); if (unlikely(rc != MDBX_SUCCESS)) diff --git a/libs/libmdbx/src/test/keygen.cc b/libs/libmdbx/src/test/keygen.cc index 99b46f2976..0b68194dc1 100644 --- a/libs/libmdbx/src/test/keygen.cc +++ b/libs/libmdbx/src/test/keygen.cc @@ -30,7 +30,7 @@ serial_t injective(const serial_t serial, /* LY: All these "magic" prime numbers were found * and verified with a bit of brute force. */ - static const uint64_t m[64 - serial_minwith] = { + static const uint64_t m[64 - serial_minwith + 1] = { /* 8 - 24 */ 113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937, 309277, 834497, 1499933, 4373441, 10184137, @@ -43,26 +43,31 @@ serial_t injective(const serial_t serial, 2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153, 23171543400565993, 53353226456762893, 155627817337932409, 227827205384840249, 816509268558278821, 576933057762605689, - 2623957345935638441, 5048241705479929949, 4634245581946485653}; - static const uint8_t s[64 - serial_minwith] = { + 2623957345935638441, 5048241705479929949, 4634245581946485653, + 4613509448041658233, 4952535426879925961}; + static const uint8_t s[64 - serial_minwith + 1] = { /* 8 - 24 */ 2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11, /* 25 - 64 */ 11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20, - 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14}; + 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20, + 19}; - serial_t result = serial * m[bits - 8]; + const auto mult = m[bits - 8]; + const auto shift = s[bits - 8]; + serial_t result = serial * mult; if (salt) { const unsigned left = bits / 2; const unsigned right = bits - left; result = (result << left) | ((result & mask(bits)) >> right); - result = (result ^ salt) * m[bits - 8]; + result = (result ^ salt) * mult; } - result ^= result << s[bits - 8]; + result ^= result << shift; result &= mask(bits); - log_trace("keygen-injective: serial %" PRIu64 " into %" PRIu64, serial, - result); + log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64 + " => %" PRIu64 "/%u", + serial, bits, mult, shift, salt, result, bits); return result; } @@ -73,8 +78,9 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, assert(mapping.mesh <= mapping.width); assert(mapping.rotate <= mapping.width); assert(mapping.offset <= mask(mapping.width)); - assert(!(key_essentials.flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP))); - assert(!(value_essentials.flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY))); + assert(!(key_essentials.flags & + ~(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT))); + assert(!(value_essentials.flags & ~(MDBX_INTEGERDUP | MDBX_REVERSEDUP))); log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial, value_age); @@ -82,31 +88,49 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, if (mapping.mesh >= serial_minwith) { serial = (serial & ~mask(mapping.mesh)) | injective(serial, mapping.mesh, salt); - log_trace("keygen-pair: mesh %" PRIu64, serial); + log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial); } if (mapping.rotate) { const unsigned right = mapping.rotate; const unsigned left = mapping.width - right; serial = (serial << left) | ((serial & mask(mapping.width)) >> right); - log_trace("keygen-pair: rotate %" PRIu64 ", 0x%" PRIx64, serial, serial); + log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64, + mapping.rotate, serial, serial); } - serial = (serial + mapping.offset) & mask(mapping.width); - log_trace("keygen-pair: offset %" PRIu64, serial); - serial += base; + if (mapping.offset) { + serial = (serial + mapping.offset) & mask(mapping.width); + log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset, + serial); + } + if (base) { + serial += base; + log_trace("keygen-pair: base@%" PRIu64 " => %" PRIu64, base, serial); + } serial_t key_serial = serial; - serial_t value_serial = value_age; + serial_t value_serial = value_age << mapping.split; if (mapping.split) { - key_serial = serial >> mapping.split; - value_serial = - (serial & mask(mapping.split)) | (value_age << mapping.split); + if (key_essentials.flags & MDBX_DUPSORT) { + key_serial >>= mapping.split; + value_serial += serial & mask(mapping.split); + } else { + /* Без MDBX_DUPSORT требуется уникальность ключей, а для этого нельзя + * отбрасывать какие-либо биты serial после инъективного преобразования. + * Поэтому key_serial не трогаем, а в value_serial нелинейно вмешиваем + * запрошенное количество бит из serial */ + value_serial += + (serial ^ (serial >> mapping.split)) & mask(mapping.split); + } + + value_serial |= value_age << mapping.split; + log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split, + key_serial, value_serial); } log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial, value_serial); - mk(key_serial, key_essentials, *key); mk(value_serial, value_essentials, *value); @@ -121,17 +145,17 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, void maker::setup(const config::actor_params_pod &actor, unsigned thread_number) { key_essentials.flags = - actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY); - assert(actor.keylen_min < UINT8_MAX); + actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT); + assert(actor.keylen_min <= UINT8_MAX); key_essentials.minlen = (uint8_t)actor.keylen_min; - assert(actor.keylen_max < UINT16_MAX); + assert(actor.keylen_max <= UINT16_MAX); key_essentials.maxlen = (uint16_t)actor.keylen_max; value_essentials.flags = actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP); - assert(actor.datalen_min < UINT8_MAX); + assert(actor.datalen_min <= UINT8_MAX); value_essentials.minlen = (uint8_t)actor.datalen_min; - assert(actor.datalen_max < UINT16_MAX); + assert(actor.datalen_max <= UINT16_MAX); value_essentials.maxlen = (uint16_t)actor.datalen_max; assert(thread_number < 2); @@ -165,7 +189,7 @@ bool maker::increment(serial_t &serial, int delta) { //----------------------------------------------------------------------------- -size_t length(serial_t serial) { +static size_t length(serial_t serial) { size_t n = 0; if (serial > UINT32_MAX) { n = 4; @@ -199,7 +223,10 @@ void __hot maker::mk(const serial_t serial, const essentials ¶ms, assert(params.maxlen >= length(serial)); out.value.iov_base = out.bytes; - out.value.iov_len = params.minlen; + out.value.iov_len = + (params.maxlen > params.minlen) + ? params.minlen + serial % (params.maxlen - params.minlen) + : params.minlen; if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) { assert(params.maxlen == params.minlen); diff --git a/libs/libmdbx/src/test/keygen.h b/libs/libmdbx/src/test/keygen.h index c1e907bc0b..449165ae9a 100644 --- a/libs/libmdbx/src/test/keygen.h +++ b/libs/libmdbx/src/test/keygen.h @@ -44,7 +44,7 @@ namespace keygen { * - абсолютное значение ключей или разность между отдельными значениями; * * Соответственно, в общих чертах, схема генерации следующая: - * - вводится плоская одномерная "координата" uint64_t; + * - вводится плоская одномерная "координата" serial (uint64_t); * - генерация специфических паттернов (последовательностей) * реализуется посредством соответствующих преобразований "координат", при * этом все подобные преобразования выполняются только над "координатой"; @@ -74,7 +74,7 @@ typedef uint64_t serial_t; enum : serial_t { serial_minwith = 8, serial_maxwith = sizeof(serial_t) * 8, - serial_allones = ~(serial_t)0 + serial_allones = ~(serial_t)0u }; struct result { @@ -85,6 +85,10 @@ struct result { uint32_t u32; uint64_t u64; }; + + std::string as_string() const { + return std::string((const char *)value.iov_base, value.iov_len); + } }; //----------------------------------------------------------------------------- @@ -120,6 +124,4 @@ public: bool increment(serial_t &serial, int delta); }; -size_t length(serial_t serial); - } /* namespace keygen */ diff --git a/libs/libmdbx/src/test/log.cc b/libs/libmdbx/src/test/log.cc index 521e1d6900..7bc3ecf613 100644 --- a/libs/libmdbx/src/test/log.cc +++ b/libs/libmdbx/src/test/log.cc @@ -37,6 +37,31 @@ void __noreturn failure_perror(const char *what, int errnum) { //----------------------------------------------------------------------------- +static void mdbx_logger(int type, const char *function, int line, + const char *msg, va_list args) { + logging::loglevel level = logging::info; + if (type & MDBX_DBG_EXTRA) + level = logging::extra; + if (type & MDBX_DBG_TRACE) + level = logging::trace; + if (type & MDBX_DBG_PRINT) + level = logging::verbose; + + if (!function) + function = "unknown"; + if (type & MDBX_DBG_ASSERT) { + log_error("mdbx: assertion failure: %s, %d", function, line); + level = logging::failure; + } + + if (logging::output( + level, + strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx: %s: ", function)) + logging::feed(msg, args); + if (type & MDBX_DBG_ASSERT) + abort(); +} + namespace logging { static std::string prefix; @@ -44,8 +69,19 @@ static std::string suffix; static loglevel level; static FILE *last; -void setup(loglevel _level, const std::string &_prefix) { +void setlevel(loglevel _level) { level = (_level > error) ? failure : _level; + int mdbx_dbg_opts = MDBX_DBG_ASSERT | MDBX_DBG_JITTER | MDBX_DBG_DUMP; + if (level <= trace) + mdbx_dbg_opts |= MDBX_DBG_TRACE; + if (level <= verbose) + mdbx_dbg_opts |= MDBX_DBG_PRINT; + int rc = mdbx_setup_debug(mdbx_dbg_opts, mdbx_logger); + log_trace("set mdbx debug-opts: 0x%02x", rc); +} + +void setup(loglevel _level, const std::string &_prefix) { + setlevel(_level); prefix = _prefix; } diff --git a/libs/libmdbx/src/test/log.h b/libs/libmdbx/src/test/log.h index e97e954cea..7350f1b9b1 100644 --- a/libs/libmdbx/src/test/log.h +++ b/libs/libmdbx/src/test/log.h @@ -17,16 +17,7 @@ #include "base.h" void __noreturn usage(void); - -#ifdef __GNUC__ -#define __printf_args(format_index, first_arg) \ - __attribute__((format(printf, format_index, first_arg))) -#else -#define __printf_args(format_index, first_arg) -#endif - void __noreturn __printf_args(1, 2) failure(const char *fmt, ...); - void __noreturn failure_perror(const char *what, int errnum); const char *test_strerror(int errnum); @@ -46,6 +37,7 @@ enum loglevel { const char *level2str(const loglevel level); void setup(loglevel level, const std::string &prefix); void setup(const std::string &prefix); +void setlevel(loglevel level); bool output(const loglevel priority, const char *format, va_list ap); bool __printf_args(2, 3) diff --git a/libs/libmdbx/src/test/main.cc b/libs/libmdbx/src/test/main.cc index bc3198ed3a..7493ab75c3 100644 --- a/libs/libmdbx/src/test/main.cc +++ b/libs/libmdbx/src/test/main.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2018 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -35,25 +35,31 @@ void actor_params::set_defaults(const std::string &tmpdir) { mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NORDAHEAD | MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_LIFORECLAIM; table_flags = MDBX_DUPSORT; - size = 1024 * 1024 * 4; + + size_lower = -1; + size_now = 1024 * 1024 * ((table_flags & MDBX_DUPSORT) ? 4 : 256); + size_upper = -1; + shrink_threshold = -1; + growth_step = -1; + pagesize = -1; keygen.seed = 1; keygen.keycase = kc_random; - keygen.width = 32; - keygen.mesh = 32; + keygen.width = (table_flags & MDBX_DUPSORT) ? 32 : 64; + keygen.mesh = keygen.width; keygen.split = keygen.width / 2; - keygen.rotate = 0; - keygen.offset = 0; + keygen.rotate = 3; + keygen.offset = 41; test_duration = 0; test_nops = 1000; nrepeat = 1; nthreads = 1; - keylen_min = 0; - keylen_max = 42; - datalen_min = 0; - datalen_max = 256; + keylen_min = mdbx_keylen_min(); + keylen_max = mdbx_keylen_max(); + datalen_min = mdbx_datalen_min(); + datalen_max = std::min(mdbx_datalen_max(), 256u * 1024 + 42); batch_read = 4; batch_write = 4; @@ -150,8 +156,45 @@ int main(int argc, char *const argv[]) { if (config::parse_option(argc, argv, narg, "table", params.table_flags, config::table_bits)) continue; - if (config::parse_option(argc, argv, narg, "size", params.size, - config::binary, 4096 * 4)) + + if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize, + mdbx_limits_pgsize_min(), + mdbx_limits_pgsize_max())) { + const unsigned keylen_max = params.mdbx_keylen_max(); + if (params.keylen_min > keylen_max) + params.keylen_min = keylen_max; + if (params.keylen_max > keylen_max) + params.keylen_max = keylen_max; + const unsigned datalen_max = params.mdbx_datalen_max(); + if (params.datalen_min > datalen_max) + params.datalen_min = datalen_max; + if (params.datalen_max > datalen_max) + params.datalen_max = datalen_max; + continue; + } + if (config::parse_option(argc, argv, narg, "size-lower", params.size_lower, + mdbx_limits_dbsize_min(params.pagesize), + mdbx_limits_dbsize_max(params.pagesize))) + continue; + if (config::parse_option(argc, argv, narg, "size", params.size_now, + mdbx_limits_dbsize_min(params.pagesize), + mdbx_limits_dbsize_max(params.pagesize))) + continue; + if (config::parse_option(argc, argv, narg, "size-upper", params.size_upper, + mdbx_limits_dbsize_min(params.pagesize), + mdbx_limits_dbsize_max(params.pagesize))) + continue; + if (config::parse_option( + argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0, + (int)std::min((intptr_t)INT_MAX, + mdbx_limits_dbsize_max(params.pagesize) - + mdbx_limits_dbsize_min(params.pagesize)))) + continue; + if (config::parse_option( + argc, argv, narg, "growth-step", params.growth_step, 0, + (int)std::min((intptr_t)INT_MAX, + mdbx_limits_dbsize_max(params.pagesize) - + mdbx_limits_dbsize_min(params.pagesize)))) continue; if (config::parse_option(argc, argv, narg, "keygen.width", @@ -188,20 +231,39 @@ int main(int argc, char *const argv[]) { config::duration, 1)) continue; if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min, - config::no_scale, 0, params.keylen_max)) + config::no_scale, params.mdbx_keylen_min(), + params.mdbx_keylen_max())) { + if ((params.table_flags & MDBX_INTEGERKEY) || + params.keylen_max < params.keylen_min) + params.keylen_max = params.keylen_min; continue; + } if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max, - config::no_scale, params.keylen_min, - mdbx_get_maxkeysize(0))) + config::no_scale, params.mdbx_keylen_min(), + params.mdbx_keylen_max())) { + if ((params.table_flags & MDBX_INTEGERKEY) || + params.keylen_min > params.keylen_max) + params.keylen_min = params.keylen_max; continue; + } if (config::parse_option(argc, argv, narg, "datalen.min", - params.datalen_min, config::no_scale, 0, - params.datalen_max)) + params.datalen_min, config::no_scale, + params.mdbx_datalen_min(), + params.mdbx_datalen_max())) { + if ((params.table_flags & MDBX_DUPFIXED) || + params.datalen_max < params.datalen_min) + params.datalen_max = params.datalen_min; continue; + } if (config::parse_option(argc, argv, narg, "datalen.max", params.datalen_max, config::no_scale, - params.datalen_min, MDBX_MAXDATASIZE)) + params.mdbx_datalen_min(), + params.mdbx_datalen_max())) { + if ((params.table_flags & MDBX_DUPFIXED) || + params.datalen_min > params.datalen_max) + params.datalen_min = params.datalen_max; continue; + } if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read, config::no_scale, 1)) continue; diff --git a/libs/libmdbx/src/test/osal-windows.cc b/libs/libmdbx/src/test/osal-windows.cc index 109c835a96..b8cdb53513 100644 --- a/libs/libmdbx/src/test/osal-windows.cc +++ b/libs/libmdbx/src/test/osal-windows.cc @@ -53,7 +53,7 @@ void osal_wait4barrier(void) { } } -static HANDLE make_inharitable(HANDLE hHandle) { +static HANDLE make_inheritable(HANDLE hHandle) { assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE); if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(), &hHandle, 0, TRUE, @@ -71,7 +71,7 @@ void osal_setup(const std::vector<actor_config> &actors) { HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); if (!hEvent) failure_perror("CreateEvent()", GetLastError()); - hEvent = make_inharitable(hEvent); + hEvent = make_inheritable(hEvent); log_trace("osal_setup: event %" PRIuPTR " -> %p", i, hEvent); events[i] = hEvent; } @@ -79,12 +79,12 @@ void osal_setup(const std::vector<actor_config> &actors) { hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL); if (!hBarrierSemaphore) failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError()); - hBarrierSemaphore = make_inharitable(hBarrierSemaphore); + hBarrierSemaphore = make_inheritable(hBarrierSemaphore); hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL); if (!hBarrierEvent) failure_perror("CreateEvent(BarrierEvent)", GetLastError()); - hBarrierEvent = make_inharitable(hBarrierEvent); + hBarrierEvent = make_inheritable(hBarrierEvent); } void osal_broadcast(unsigned id) { diff --git a/libs/libmdbx/src/test/test.cc b/libs/libmdbx/src/test/test.cc index 3750af525f..c28bbd221e 100644 --- a/libs/libmdbx/src/test/test.cc +++ b/libs/libmdbx/src/test/test.cc @@ -68,31 +68,6 @@ const char *keygencase2str(const keygen_case keycase) { //----------------------------------------------------------------------------- -static void mdbx_logger(int type, const char *function, int line, - const char *msg, va_list args) { - logging::loglevel level = logging::info; - if (type & MDBX_DBG_EXTRA) - level = logging::extra; - if (type & MDBX_DBG_TRACE) - level = logging::trace; - if (type & MDBX_DBG_PRINT) - level = logging::verbose; - - if (!function) - function = "unknown"; - if (type & MDBX_DBG_ASSERT) { - log_error("mdbx: assertion failure: %s, %d", function, line); - level = logging::failure; - } - - if (logging::output( - level, - strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx: %s: ", function)) - logging::feed(msg, args); - if (type & MDBX_DBG_ASSERT) - abort(); -} - int testcase::oom_callback(MDBX_env *env, int pid, mdbx_tid_t tid, uint64_t txn, unsigned gap, int retry) { @@ -117,16 +92,8 @@ void testcase::db_prepare() { log_trace(">> db_prepare"); assert(!db_guard); - int mdbx_dbg_opts = MDBX_DBG_ASSERT | MDBX_DBG_JITTER | MDBX_DBG_DUMP; - if (config.params.loglevel <= logging::trace) - mdbx_dbg_opts |= MDBX_DBG_TRACE; - if (config.params.loglevel <= logging::verbose) - mdbx_dbg_opts |= MDBX_DBG_PRINT; - int rc = mdbx_setup_debug(mdbx_dbg_opts, mdbx_logger); - log_trace("set mdbx debug-opts: 0x%02x", rc); - MDBX_env *env = nullptr; - rc = mdbx_env_create(&env); + int rc = mdbx_env_create(&env); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_env_create()", rc); @@ -149,7 +116,10 @@ void testcase::db_prepare() { if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_env_set_oomfunc()", rc); - rc = mdbx_env_set_mapsize(env, (size_t)config.params.size); + rc = mdbx_env_set_geometry( + env, config.params.size_lower, config.params.size_now, + config.params.size_upper, config.params.growth_step, + config.params.shrink_threshold, config.params.pagesize); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_env_set_mapsize()", rc); diff --git a/libs/libmdbx/src/test/utils.cc b/libs/libmdbx/src/test/utils.cc index 0855c7eef3..53a750e314 100644 --- a/libs/libmdbx/src/test/utils.cc +++ b/libs/libmdbx/src/test/utils.cc @@ -93,7 +93,7 @@ bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, //----------------------------------------------------------------------------- -/* TODO: replace my 'libmera' fomr t1ha. */ +/* TODO: replace my 'libmera' from t1ha. */ uint64_t entropy_ticks(void) { #if defined(EMSCRIPTEN) return (uint64_t)emscripten_get_now(); |