diff options
Diffstat (limited to 'tools')
-rw-r--r-- | tools/mdbx_chk/src/mdbx_chk.cc | 106 | ||||
-rw-r--r-- | tools/mdbx_chk/src/stdafx.h | 9 | ||||
-rw-r--r-- | tools/mdbx_dump/src/mdbx_dump.cc | 101 | ||||
-rw-r--r-- | tools/mdbx_dump/src/stdafx.h | 9 | ||||
-rw-r--r-- | tools/mdbx_load/src/mdbx_load.cc | 235 | ||||
-rw-r--r-- | tools/mdbx_load/src/stdafx.h | 61 |
6 files changed, 314 insertions, 207 deletions
diff --git a/tools/mdbx_chk/src/mdbx_chk.cc b/tools/mdbx_chk/src/mdbx_chk.cc index 6676cee912..5b41402f85 100644 --- a/tools/mdbx_chk/src/mdbx_chk.cc +++ b/tools/mdbx_chk/src/mdbx_chk.cc @@ -1,7 +1,7 @@ /* mdbx_chk.c - memory-mapped database check tool */ /* - * Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru> + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. * @@ -13,6 +13,8 @@ * top-level directory of the distribution or, alternatively, at * <http://www.OpenLDAP.org/license.html>. */ +#include "stdafx.h" + #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -94,7 +96,7 @@ size_t userdb_count, skipped_subdb; uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages, unused_pages, backed_pages; unsigned verbose; -bool ignore_wrong_order, quiet; +bool ignore_wrong_order, quiet, dont_traversal; const char *only_subdb; struct problem { @@ -627,22 +629,23 @@ static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler, error("too many DBIs or out of memory\n"); return MDBX_ENOMEM; } - const uint64_t subtotal_pages = - ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages; - if (subtotal_pages != dbi->pages.total) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "subtotal", - subtotal_pages, dbi->pages.total); - if (ms.ms_branch_pages != dbi->pages.branch) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch", - ms.ms_branch_pages, dbi->pages.branch); - const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed; - if (ms.ms_leaf_pages != allleaf_pages) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "all-leaf", - ms.ms_leaf_pages, allleaf_pages); - if (ms.ms_overflow_pages != dbi->pages.large_volume) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", - "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume); - + if (!dont_traversal) { + const uint64_t subtotal_pages = + ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages; + if (subtotal_pages != dbi->pages.total) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", + "subtotal", subtotal_pages, dbi->pages.total); + if (ms.ms_branch_pages != dbi->pages.branch) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch", + ms.ms_branch_pages, dbi->pages.branch); + const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed; + if (ms.ms_leaf_pages != allleaf_pages) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", + "all-leaf", ms.ms_leaf_pages, allleaf_pages); + if (ms.ms_overflow_pages != dbi->pages.large_volume) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", + "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume); + } rc = mdbx_cursor_open(txn, dbi_handle, &mc); if (rc) { error("mdbx_cursor_open failed, error %d %s\n", rc, mdbx_strerror(rc)); @@ -759,18 +762,19 @@ bailout: } static void usage(char *prog) { - fprintf(stderr, - "usage: %s [-V] [-v] [-n] [-q] [-c] [-w] [-d] [-i] [-s subdb] dbpath\n" - " -V\t\tprint version and exit\n" - " -v\t\tmore verbose, could be used multiple times\n" - " -n\t\tNOSUBDIR mode for open\n" - " -q\t\tbe quiet\n" - " -c\t\tforce cooperative mode (don't try exclusive)\n" - " -w\t\tlock DB for writing while checking\n" - " -d\t\tdisable page-by-page traversal of B-tree\n" - " -i\t\tignore wrong order errors (for custom comparators case)\n" - " -s subdb\tprocess a specific subdatabase only\n", - prog); + fprintf( + stderr, + "usage: %s [-V] [-v] [-n] [-q] [-c] [-w] [-d] [-i] [-s subdb] dbpath\n" + " -V\t\tprint version and exit\n" + " -v\t\tmore verbose, could be used multiple times\n" + " -n\t\tNOSUBDIR mode for open\n" + " -q\t\tbe quiet\n" + " -c\t\tforce cooperative mode (don't try exclusive)\n" + " -w\t\tlock DB for writing while checking\n" + " -d\t\tdisable page-by-page traversal of B-tree\n" + " -i\t\tignore wrong order errors (for custom comparators case)\n" + " -s subdb\tprocess a specific subdatabase only\n", + prog); exit(EXIT_INTERRUPTED); } @@ -913,7 +917,6 @@ int main(int argc, char *argv[]) { char *prog = argv[0]; char *envname; int problems_maindb = 0, problems_freedb = 0, problems_meta = 0; - bool dont_traversal = false; bool locked = false; double elapsed; @@ -928,6 +931,7 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE_SYS; } #endif + DECLARE_VERSION(); dbi_meta.name = "@META"; dbi_free.name = "@GC"; @@ -940,18 +944,18 @@ int main(int argc, char *argv[]) { for (int i; (i = getopt(argc, argv, "Vvqnwcdsi:")) != EOF;) { switch (i) { case 'V': - //printf("mdbx_chk version %d.%d.%d.%d\n" - // " - source: %s %s, commit %s, tree %s\n" - // " - anchor: %s\n" - // " - build: %s for %s by %s\n" - // " - flags: %s\n" - // " - options: %s\n", - // mdbx_version.major, mdbx_version.minor, mdbx_version.release, - // mdbx_version.revision, mdbx_version.git.describe, - // mdbx_version.git.datetime, mdbx_version.git.commit, - // mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - // mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - // mdbx_build.options); + printf("mdbx_chk version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + MDBX_version.major, MDBX_version.minor, MDBX_version.release, + MDBX_version.revision, MDBX_version.git.describe, + MDBX_version.git.datetime, MDBX_version.git.commit, + MDBX_version.git.tree, MDBX_sourcery_anchor, MDBX_build.datetime, + MDBX_build.target, MDBX_build.compiler, MDBX_build.flags, + MDBX_build.options); return EXIT_SUCCESS; case 'v': verbose++; @@ -1001,10 +1005,10 @@ int main(int argc, char *argv[]) { #endif /* !WINDOWS */ envname = argv[optind]; - // print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n", - // mdbx_version.git.describe, mdbx_version.git.datetime, - // mdbx_version.git.tree, envname, - // (envflags & MDBX_RDONLY) ? "only" : "write"); + print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n", + MDBX_version.git.describe, MDBX_version.git.datetime, + MDBX_version.git.tree, envname, + (envflags & MDBX_RDONLY) ? "only" : "write"); fflush(NULL); rc = mdbx_env_create(&env); @@ -1172,6 +1176,14 @@ int main(int argc, char *argv[]) { } printf(", %" PRIu64 " pages\n", envinfo.mi_geo.current / envinfo.mi_dxb_pagesize); +#if defined(_WIN32) || defined(_WIN64) + if (envinfo.mi_geo.shrink && envinfo.mi_geo.current != envinfo.mi_geo.upper) + print(" WARNING: Due Windows system limitations a " + "file couldn't\n be truncated while database " + "is opened. So, the size of\n database file " + "may by large than the database itself,\n " + "until it will be closed or reopened in read-write mode.\n"); +#endif print(" - transactions: recent %" PRIu64 ", latter reader %" PRIu64 ", lag %" PRIi64 "\n", envinfo.mi_recent_txnid, envinfo.mi_latter_reader_txnid, diff --git a/tools/mdbx_chk/src/stdafx.h b/tools/mdbx_chk/src/stdafx.h index 540b7b9a99..f22afc24b5 100644 --- a/tools/mdbx_chk/src/stdafx.h +++ b/tools/mdbx_chk/src/stdafx.h @@ -19,3 +19,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. #include <stdio.h> #include <windows.h> + +#define DECLARE_VERSION() \ + mdbx_version_info MDBX_version; \ + mdbx_build_info MDBX_build; \ + char* MDBX_sourcery_anchor; \ + HINSTANCE hDll = LoadLibraryA("libmdbx.mir"); \ + MDBX_version = *(mdbx_version_info *)GetProcAddress(hDll, "mdbx_version"); \ + MDBX_build = *(mdbx_build_info*)GetProcAddress(hDll, "mdbx_build"); \ + MDBX_sourcery_anchor = (char*)GetProcAddress(hDll, "mdbx_sourcery_MDBX_BUILD_SOURCERY"); diff --git a/tools/mdbx_dump/src/mdbx_dump.cc b/tools/mdbx_dump/src/mdbx_dump.cc index af5776d27e..f51903df4c 100644 --- a/tools/mdbx_dump/src/mdbx_dump.cc +++ b/tools/mdbx_dump/src/mdbx_dump.cc @@ -1,7 +1,7 @@ /* mdbx_dump.c - memory-mapped database dump tool */ /* - * Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru> + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. * @@ -13,6 +13,8 @@ * top-level directory of the distribution or, alternatively, at * <http://www.OpenLDAP.org/license.html>. */ +#include "stdafx.h" + #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -116,7 +118,7 @@ static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) { if (rc) return rc; - rc = mdbx_env_info(mdbx_txn_env(txn), &info, sizeof(info)); + rc = mdbx_env_info_ex(mdbx_txn_env(txn), txn, &info, sizeof(info)); if (rc) return rc; @@ -125,7 +127,7 @@ static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) { if (name) printf("database=%s\n", name); printf("type=btree\n"); - printf("mapsize=%" PRIu64 "\n", info.mi_mapsize); + printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper); printf("maxreaders=%u\n", info.mi_maxreaders); for (i = 0; dbflags[i].bit; i++) @@ -161,7 +163,18 @@ static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) { static void usage(char *prog) { fprintf(stderr, - "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", + "usage: %s [-V] [-q] [-f file] [-l] [-p] [-a|-s subdb] [-r] [-n] " + "dbpath\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -f\t\twrite to file instead of stdout\n" + " -l\t\tlist subDBs and exit\n" + " -p\t\tuse printable characters\n" + " -a\t\tdump main DB and all subDBs,\n" + " \t\tby default dump only the main DB\n" + " -s\t\tdump only the named subDB\n" + " -r\t\trescure mode (ignore errors to dump corrupted DB)\n" + " -n\t\tNOSUBDIR mode for open\n", prog); exit(EXIT_FAILURE); } @@ -174,26 +187,28 @@ int main(int argc, char *argv[]) { char *prog = argv[0]; char *envname; char *subname = NULL; - int alldbs = 0, envflags = 0, list = 0; + int alldbs = 0, envflags = 0, list = 0, quiet = 0, rescue = 0; + DECLARE_VERSION(); - if (argc < 2) { + if (argc < 2) usage(prog); - } - /* -a: dump main DB and all subDBs - * -s: dump only the named subDB - * -n: use NOSUBDIR flag on env_open - * -p: use printable characters - * -f: write to file instead of stdout - * -V: print version and exit - * (default) dump only the main DB - */ - while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { + while ((i = getopt(argc, argv, "af:lnps:Vrq")) != EOF) { switch (i) { case 'V': - //printf("%s (%s, build %s)\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_build.datetime); - exit(EXIT_SUCCESS); - break; + printf("mdbx_dump version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + MDBX_version.major, MDBX_version.minor, MDBX_version.release, + MDBX_version.revision, MDBX_version.git.describe, + MDBX_version.git.datetime, MDBX_version.git.commit, + MDBX_version.git.tree, MDBX_sourcery_anchor, MDBX_build.datetime, + MDBX_build.target, MDBX_build.compiler, MDBX_build.flags, + MDBX_build.options); + return EXIT_SUCCESS; case 'l': list = 1; /*FALLTHROUGH*/; @@ -205,7 +220,8 @@ int main(int argc, char *argv[]) { break; case 'f': if (freopen(optarg, "w", stdout) == NULL) { - fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno)); + fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, + mdbx_strerror(errno)); exit(EXIT_FAILURE); } break; @@ -220,6 +236,12 @@ int main(int argc, char *argv[]) { usage(prog); subname = optarg; break; + case 'q': + quiet = 1; + break; + case 'r': + rescue = 1; + break; default: usage(prog); } @@ -242,6 +264,13 @@ int main(int argc, char *argv[]) { #endif /* !WINDOWS */ envname = argv[optind]; + if (!quiet) { + fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", + MDBX_version.git.describe, MDBX_version.git.datetime, + MDBX_version.git.tree, envname); + fflush(NULL); + } + rc = mdbx_env_create(&env); if (rc) { fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc, @@ -253,7 +282,9 @@ int main(int argc, char *argv[]) { mdbx_env_set_maxdbs(env, 2); } - rc = mdbx_env_open(env, envname, envflags | MDBX_EXCLUSIVE | MDBX_RDONLY, 0664); + rc = mdbx_env_open( + env, envname, + envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE : MDBX_RDONLY), 0); if (rc) { fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc, mdbx_strerror(rc)); @@ -304,8 +335,32 @@ int main(int argc, char *argv[]) { list++; } else { rc = dumpit(txn, db2, str); - if (rc) - break; + if (rc) { + if (!rescue) + break; + fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, + envname, mdbx_strerror(rc), str); + /* Here is a hack for rescue mode, don't do that: + * - we should restart transaction in case error due + * database corruption; + * - but we won't close cursor, reopen and re-positioning it + * for new a transaction; + * - this is possible since DB is opened in read-only exclusive + * mode and transaction is the same, i.e. has the same address + * and so on. */ + rc = mdbx_txn_reset(txn); + if (rc) { + fprintf(stderr, "mdbx_txn_reset failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + rc = mdbx_txn_renew(txn); + if (rc) { + fprintf(stderr, "mdbx_txn_renew failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + } } mdbx_dbi_close(env, db2); } diff --git a/tools/mdbx_dump/src/stdafx.h b/tools/mdbx_dump/src/stdafx.h index 540b7b9a99..f22afc24b5 100644 --- a/tools/mdbx_dump/src/stdafx.h +++ b/tools/mdbx_dump/src/stdafx.h @@ -19,3 +19,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. #include <stdio.h> #include <windows.h> + +#define DECLARE_VERSION() \ + mdbx_version_info MDBX_version; \ + mdbx_build_info MDBX_build; \ + char* MDBX_sourcery_anchor; \ + HINSTANCE hDll = LoadLibraryA("libmdbx.mir"); \ + MDBX_version = *(mdbx_version_info *)GetProcAddress(hDll, "mdbx_version"); \ + MDBX_build = *(mdbx_build_info*)GetProcAddress(hDll, "mdbx_build"); \ + MDBX_sourcery_anchor = (char*)GetProcAddress(hDll, "mdbx_sourcery_MDBX_BUILD_SOURCERY"); diff --git a/tools/mdbx_load/src/mdbx_load.cc b/tools/mdbx_load/src/mdbx_load.cc index deceaca224..567ad5b93a 100644 --- a/tools/mdbx_load/src/mdbx_load.cc +++ b/tools/mdbx_load/src/mdbx_load.cc @@ -1,7 +1,7 @@ /* mdbx_load.c - memory-mapped database load tool */ /* - * Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru> + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. * @@ -58,7 +58,7 @@ static int version; static int dbi_flags; static char *prog; -static int Eof; +static bool Eof; static MDBX_envinfo envinfo; static MDBX_val kbuf, dbuf; @@ -86,22 +86,34 @@ static void readhdr(void) { char *ptr; dbi_flags = 0; - while (fgets((char *)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) { + while (fgets((char*)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) { lineno++; - if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) || - !strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) { - /* LY: silently ignore information fields. */ + + if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize="))) { + envinfo.mi_dxb_pagesize = + atoi((char *)dbuf.iov_base + STRLENOF("db_pagesize=")); + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) { + dbi_flags |= MDBX_DUPSORT; continue; - } else if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) { + } + + if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) { version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION=")); if (version > 3) { fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n", prog, lineno, version); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) { - break; - } else if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) + return; + + if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) { if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print", STRLENOF("print"))) mode |= PRINT; @@ -111,21 +123,30 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) { ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) *ptr = '\0'; if (subname) - free(subname); - subname = strdup((char *)dbuf.iov_base + STRLENOF("database=")); - } else if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) { + mdbx_free(subname); + subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database=")); + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) { if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree", STRLENOF("btree"))) { fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog, lineno, (char *)dbuf.iov_base + STRLENOF("type=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) { int i; ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) @@ -137,7 +158,10 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) { int i; ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) @@ -149,7 +173,10 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("mapsize=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) { int i; ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) @@ -161,31 +188,33 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders=")); exit(EXIT_FAILURE); } - } else { - int i; - for (i = 0; dbflags[i].bit; i++) { - if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) && - ((char *)dbuf.iov_base)[dbflags[i].len] == '=') { - if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1') - dbi_flags |= dbflags[i].bit; - break; - } + continue; + } + + int i; + for (i = 0; dbflags[i].bit; i++) { + if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) && + ((char *)dbuf.iov_base)[dbflags[i].len] == '=') { + if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1') + dbi_flags |= dbflags[i].bit; + break; } - if (!dbflags[i].bit) { - ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len); - if (!ptr) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog, - lineno); - exit(EXIT_FAILURE); - } else { - *ptr = '\0'; - fprintf(stderr, - "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", - prog, lineno, (char *)dbuf.iov_base); - } + } + if (!dbflags[i].bit) { + ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len); + if (!ptr) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog, + lineno); + exit(EXIT_FAILURE); + } else { + *ptr = '\0'; + fprintf(stderr, + "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", + prog, lineno, (char *)dbuf.iov_base); } } } + Eof = true; } static void badend(void) { @@ -214,14 +243,14 @@ static int readline(MDBX_val *out, MDBX_val *buf) { if (!(mode & NOHDR)) { c = fgetc(stdin); if (c == EOF) { - Eof = 1; + Eof = true; return EOF; } if (c != ' ') { lineno++; if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) { badend: - Eof = 1; + Eof = true; badend(); return EOF; } @@ -231,28 +260,28 @@ static int readline(MDBX_val *out, MDBX_val *buf) { } } if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) { - Eof = 1; + Eof = true; return EOF; } lineno++; - c1 = (BYTE *)buf->iov_base; + c1 = (unsigned char *)buf->iov_base; len = strlen((char *)c1); l2 = len; /* Is buffer too short? */ while (c1[len - 1] != '\n') { - buf->iov_base = realloc(buf->iov_base, buf->iov_len * 2); + buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2); if (!buf->iov_base) { - Eof = 1; + Eof = true; fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog, lineno); return EOF; } - c1 = (BYTE *)buf->iov_base; + c1 = (unsigned char *)buf->iov_base; c1 += l2; if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) { - Eof = 1; + Eof = true; badend(); return EOF; } @@ -260,26 +289,25 @@ static int readline(MDBX_val *out, MDBX_val *buf) { len = strlen((char *)c1); l2 += len; } - c1 = c2 = (BYTE*)buf->iov_base; + c1 = c2 = (unsigned char *)buf->iov_base; len = l2; c1[--len] = '\0'; end = c1 + len; if (mode & PRINT) { while (c2 < end) { - if (*c2 == '\\') { + if (unlikely(*c2 == '\\')) { if (c2[1] == '\\') { - c1++; - c2 += 2; + *c1++ = '\\'; } else { if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { - Eof = 1; + Eof = true; badend(); return EOF; } *c1++ = (char)unhex(++c2); - c2 += 2; } + c2 += 2; } else { /* copies are redundant when no escapes were used */ *c1++ = *c2++; @@ -288,13 +316,13 @@ static int readline(MDBX_val *out, MDBX_val *buf) { } else { /* odd length not allowed */ if (len & 1) { - Eof = 1; + Eof = true; badend(); return EOF; } while (c2 < end) { if (!isxdigit(*c2) || !isxdigit(c2[1])) { - Eof = 1; + Eof = true; badend(); return EOF; } @@ -302,7 +330,7 @@ static int readline(MDBX_val *out, MDBX_val *buf) { c2 += 2; } } - c2 = (BYTE*)(out->iov_base = buf->iov_base); + c2 = (unsigned char *)(out->iov_base = buf->iov_base); out->iov_len = c1 - c2; return 0; @@ -310,7 +338,17 @@ static int readline(MDBX_val *out, MDBX_val *buf) { static void usage(void) { fprintf(stderr, - "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", + "usage: %s [-V] [-q] [-a] [-f file] [-s name] [-N] [-T] [-r] [-n] " + "dbpath\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -a\t\tappend records in input order\n" + " -f file\tread from file instead of stdin\n" + " -s name\tload into named subDB\n" + " -N\t\tuse NOOVERWRITE on puts\n" + " -T\t\tread plaintext\n" + " -r\t\trescure mode (ignore errors to load corrupted DB dump)\n" + " -n\t\tNOSUBDIR mode for open\n", prog); exit(EXIT_FAILURE); } @@ -330,32 +368,38 @@ int main(int argc, char *argv[]) { char *envname = NULL; int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0; int append = 0; + int quiet = 0; + int rescue = 0; MDBX_val prevk; + DECLARE_VERSION(); prog = argv[0]; if (argc < 2) usage(); - /* -a: append records in input order - * -f: load file instead of stdin - * -n: use NOSUBDIR flag on env_open - * -s: load into named subDB - * -N: use NOOVERWRITE on puts - * -T: read plaintext - * -V: print version and exit - */ - while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) { + while ((i = getopt(argc, argv, "af:ns:NTVrq")) != EOF) { switch (i) { case 'V': - // printf("%s (%s, build %s)\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_build.datetime); - exit(EXIT_SUCCESS); - break; + printf("mdbx_load version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + MDBX_version.major, MDBX_version.minor, MDBX_version.release, + MDBX_version.revision, MDBX_version.git.describe, + MDBX_version.git.datetime, MDBX_version.git.commit, + MDBX_version.git.tree, MDBX_sourcery_anchor, MDBX_build.datetime, + MDBX_build.target, MDBX_build.compiler, MDBX_build.flags, + MDBX_build.options); + return EXIT_SUCCESS; case 'a': append = 1; break; case 'f': if (freopen(optarg, "r", stdin) == NULL) { - fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno)); + fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, + mdbx_strerror(errno)); exit(EXIT_FAILURE); } break; @@ -371,6 +415,12 @@ int main(int argc, char *argv[]) { case 'T': mode |= NOHDR | PRINT; break; + case 'q': + quiet = 1; + break; + case 'r': + rescue = 1; + break; default: usage(); } @@ -392,6 +442,13 @@ int main(int argc, char *argv[]) { signal(SIGTERM, signal_handler); #endif /* !WINDOWS */ + envname = argv[optind]; + if (!quiet) + printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", + MDBX_version.git.describe, MDBX_version.git.datetime, + MDBX_version.git.tree, envname); + fflush(NULL); + dbuf.iov_len = 4096; dbuf.iov_base = mdbx_malloc(dbuf.iov_len); @@ -399,7 +456,6 @@ int main(int argc, char *argv[]) { if (!(mode & NOHDR)) readhdr(); - envname = argv[optind]; rc = mdbx_env_create(&env); if (rc) { fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc, @@ -409,23 +465,28 @@ int main(int argc, char *argv[]) { mdbx_env_set_maxdbs(env, 2); - if (envinfo.mi_maxreaders) - mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders); - - if (envinfo.mi_mapsize) { - if (envinfo.mi_mapsize > SIZE_MAX) { - fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc, - mdbx_strerror(MDBX_TOO_LARGE)); - return EXIT_FAILURE; - } - mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize); - } - #ifdef MDBX_FIXEDMAP if (info.mi_mapaddr) envflags |= MDBX_FIXEDMAP; #endif + if (envinfo.mi_mapsize) { + if (envinfo.mi_mapsize > INTPTR_MAX) { + fprintf(stderr, + "Database size is too large for current system (mapsize=%" PRIu64 + " is great than system-limit %zi)\n", + envinfo.mi_mapsize, INTPTR_MAX); + goto env_close; + } + rc = mdbx_env_set_geometry(env, 0, 0, (intptr_t)envinfo.mi_mapsize, -1, -1, + -1); + if (rc) { + fprintf(stderr, "mdbx_env_set_geometry failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + } + rc = mdbx_env_open(env, envname, envflags, 0664); if (rc) { fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc, @@ -433,8 +494,8 @@ int main(int argc, char *argv[]) { goto env_close; } - kbuf.iov_len = mdbx_env_get_maxkeysize(env); - if (kbuf.iov_len >= SIZE_MAX / 4) { + kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, MDBX_DUPSORT); + if (kbuf.iov_len >= INTPTR_MAX / 4) { fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n", kbuf.iov_len); goto env_close; @@ -503,6 +564,11 @@ int main(int argc, char *argv[]) { rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag); if (rc == MDBX_KEYEXIST && putflags) continue; + if (rc == MDBX_BAD_VALSIZE && rescue) { + fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, + mdbx_strerror(rc)); + continue; + } if (rc) { fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc, mdbx_strerror(rc)); @@ -539,6 +605,7 @@ int main(int argc, char *argv[]) { goto env_close; } mdbx_dbi_close(env, dbi); + subname = NULL; /* try read next header */ if (!(mode & NOHDR)) diff --git a/tools/mdbx_load/src/stdafx.h b/tools/mdbx_load/src/stdafx.h index c54474f43b..518027200a 100644 --- a/tools/mdbx_load/src/stdafx.h +++ b/tools/mdbx_load/src/stdafx.h @@ -21,56 +21,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. #include <stdint.h> #include <windows.h> -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "../../libs/libmdbx/src/src/elements/internals.h" - -#define CMP_UINT(x, y) { if ((x) != (y)) return (x) < (y) ? -1 : 1; } - -struct DBEventSortingKey -{ - uint32_t hContact; - uint32_t hEvent; - uint64_t ts; - - static int Compare(const MDBX_val *ax, const MDBX_val *bx) - { - const DBEventSortingKey *a = (DBEventSortingKey*)ax->iov_base; - const DBEventSortingKey *b = (DBEventSortingKey*)bx->iov_base; - - CMP_UINT(a->hContact, b->hContact); - CMP_UINT(a->ts, b->ts); - CMP_UINT(a->hEvent, b->hEvent); - return 0; - } -}; - -struct DBEventIdKey -{ - uint32_t iModuleId; // offset to a DBModuleName struct of the name of - char szEventId[256]; // string id - - static int Compare(const MDBX_val *ax, const MDBX_val *bx) - { - const DBEventIdKey *a = (DBEventIdKey*)ax->iov_base; - const DBEventIdKey *b = (DBEventIdKey*)bx->iov_base; - CMP_UINT(a->iModuleId, b->iModuleId); - return strcmp(a->szEventId, b->szEventId); - } -}; - -struct DBSettingKey -{ - uint32_t hContact; - uint32_t dwModuleId; - char szSettingName[1]; - - static int Compare(const MDBX_val *ax, const MDBX_val *bx) - { - const DBSettingKey *a = (DBSettingKey*)ax->iov_base; - const DBSettingKey *b = (DBSettingKey*)bx->iov_base; - - CMP_UINT(a->hContact, b->hContact); - CMP_UINT(a->dwModuleId, b->dwModuleId); - return strcmp(a->szSettingName, b->szSettingName); - } -}; +#define DECLARE_VERSION() \ + mdbx_version_info MDBX_version; \ + mdbx_build_info MDBX_build; \ + char* MDBX_sourcery_anchor; \ + HINSTANCE hDll = LoadLibraryA("libmdbx.mir"); \ + MDBX_version = *(mdbx_version_info *)GetProcAddress(hDll, "mdbx_version"); \ + MDBX_build = *(mdbx_build_info*)GetProcAddress(hDll, "mdbx_build"); \ + MDBX_sourcery_anchor = (char*)GetProcAddress(hDll, "mdbx_sourcery_MDBX_BUILD_SOURCERY"); |