diff options
Diffstat (limited to 'tools/mdbx_load/src')
| -rw-r--r-- | tools/mdbx_load/src/mdbx_load.cc | 235 | ||||
| -rw-r--r-- | tools/mdbx_load/src/stdafx.h | 61 |
2 files changed, 159 insertions, 137 deletions
diff --git a/tools/mdbx_load/src/mdbx_load.cc b/tools/mdbx_load/src/mdbx_load.cc index deceaca224..567ad5b93a 100644 --- a/tools/mdbx_load/src/mdbx_load.cc +++ b/tools/mdbx_load/src/mdbx_load.cc @@ -1,7 +1,7 @@ /* mdbx_load.c - memory-mapped database load tool */ /* - * Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru> + * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> * and other libmdbx authors: please see AUTHORS file. * All rights reserved. * @@ -58,7 +58,7 @@ static int version; static int dbi_flags; static char *prog; -static int Eof; +static bool Eof; static MDBX_envinfo envinfo; static MDBX_val kbuf, dbuf; @@ -86,22 +86,34 @@ static void readhdr(void) { char *ptr; dbi_flags = 0; - while (fgets((char *)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) { + while (fgets((char*)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) { lineno++; - if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) || - !strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) { - /* LY: silently ignore information fields. */ + + if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize="))) { + envinfo.mi_dxb_pagesize = + atoi((char *)dbuf.iov_base + STRLENOF("db_pagesize=")); + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) { + dbi_flags |= MDBX_DUPSORT; continue; - } else if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) { + } + + if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) { version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION=")); if (version > 3) { fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n", prog, lineno, version); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) { - break; - } else if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) + return; + + if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) { if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print", STRLENOF("print"))) mode |= PRINT; @@ -111,21 +123,30 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) { ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) *ptr = '\0'; if (subname) - free(subname); - subname = strdup((char *)dbuf.iov_base + STRLENOF("database=")); - } else if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) { + mdbx_free(subname); + subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database=")); + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) { if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree", STRLENOF("btree"))) { fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog, lineno, (char *)dbuf.iov_base + STRLENOF("type=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) { int i; ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) @@ -137,7 +158,10 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) { int i; ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) @@ -149,7 +173,10 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("mapsize=")); exit(EXIT_FAILURE); } - } else if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) { + continue; + } + + if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) { int i; ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); if (ptr) @@ -161,31 +188,33 @@ static void readhdr(void) { lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders=")); exit(EXIT_FAILURE); } - } else { - int i; - for (i = 0; dbflags[i].bit; i++) { - if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) && - ((char *)dbuf.iov_base)[dbflags[i].len] == '=') { - if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1') - dbi_flags |= dbflags[i].bit; - break; - } + continue; + } + + int i; + for (i = 0; dbflags[i].bit; i++) { + if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) && + ((char *)dbuf.iov_base)[dbflags[i].len] == '=') { + if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1') + dbi_flags |= dbflags[i].bit; + break; } - if (!dbflags[i].bit) { - ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len); - if (!ptr) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog, - lineno); - exit(EXIT_FAILURE); - } else { - *ptr = '\0'; - fprintf(stderr, - "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", - prog, lineno, (char *)dbuf.iov_base); - } + } + if (!dbflags[i].bit) { + ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len); + if (!ptr) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog, + lineno); + exit(EXIT_FAILURE); + } else { + *ptr = '\0'; + fprintf(stderr, + "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", + prog, lineno, (char *)dbuf.iov_base); } } } + Eof = true; } static void badend(void) { @@ -214,14 +243,14 @@ static int readline(MDBX_val *out, MDBX_val *buf) { if (!(mode & NOHDR)) { c = fgetc(stdin); if (c == EOF) { - Eof = 1; + Eof = true; return EOF; } if (c != ' ') { lineno++; if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) { badend: - Eof = 1; + Eof = true; badend(); return EOF; } @@ -231,28 +260,28 @@ static int readline(MDBX_val *out, MDBX_val *buf) { } } if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) { - Eof = 1; + Eof = true; return EOF; } lineno++; - c1 = (BYTE *)buf->iov_base; + c1 = (unsigned char *)buf->iov_base; len = strlen((char *)c1); l2 = len; /* Is buffer too short? */ while (c1[len - 1] != '\n') { - buf->iov_base = realloc(buf->iov_base, buf->iov_len * 2); + buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2); if (!buf->iov_base) { - Eof = 1; + Eof = true; fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog, lineno); return EOF; } - c1 = (BYTE *)buf->iov_base; + c1 = (unsigned char *)buf->iov_base; c1 += l2; if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) { - Eof = 1; + Eof = true; badend(); return EOF; } @@ -260,26 +289,25 @@ static int readline(MDBX_val *out, MDBX_val *buf) { len = strlen((char *)c1); l2 += len; } - c1 = c2 = (BYTE*)buf->iov_base; + c1 = c2 = (unsigned char *)buf->iov_base; len = l2; c1[--len] = '\0'; end = c1 + len; if (mode & PRINT) { while (c2 < end) { - if (*c2 == '\\') { + if (unlikely(*c2 == '\\')) { if (c2[1] == '\\') { - c1++; - c2 += 2; + *c1++ = '\\'; } else { if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { - Eof = 1; + Eof = true; badend(); return EOF; } *c1++ = (char)unhex(++c2); - c2 += 2; } + c2 += 2; } else { /* copies are redundant when no escapes were used */ *c1++ = *c2++; @@ -288,13 +316,13 @@ static int readline(MDBX_val *out, MDBX_val *buf) { } else { /* odd length not allowed */ if (len & 1) { - Eof = 1; + Eof = true; badend(); return EOF; } while (c2 < end) { if (!isxdigit(*c2) || !isxdigit(c2[1])) { - Eof = 1; + Eof = true; badend(); return EOF; } @@ -302,7 +330,7 @@ static int readline(MDBX_val *out, MDBX_val *buf) { c2 += 2; } } - c2 = (BYTE*)(out->iov_base = buf->iov_base); + c2 = (unsigned char *)(out->iov_base = buf->iov_base); out->iov_len = c1 - c2; return 0; @@ -310,7 +338,17 @@ static int readline(MDBX_val *out, MDBX_val *buf) { static void usage(void) { fprintf(stderr, - "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", + "usage: %s [-V] [-q] [-a] [-f file] [-s name] [-N] [-T] [-r] [-n] " + "dbpath\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -a\t\tappend records in input order\n" + " -f file\tread from file instead of stdin\n" + " -s name\tload into named subDB\n" + " -N\t\tuse NOOVERWRITE on puts\n" + " -T\t\tread plaintext\n" + " -r\t\trescure mode (ignore errors to load corrupted DB dump)\n" + " -n\t\tNOSUBDIR mode for open\n", prog); exit(EXIT_FAILURE); } @@ -330,32 +368,38 @@ int main(int argc, char *argv[]) { char *envname = NULL; int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0; int append = 0; + int quiet = 0; + int rescue = 0; MDBX_val prevk; + DECLARE_VERSION(); prog = argv[0]; if (argc < 2) usage(); - /* -a: append records in input order - * -f: load file instead of stdin - * -n: use NOSUBDIR flag on env_open - * -s: load into named subDB - * -N: use NOOVERWRITE on puts - * -T: read plaintext - * -V: print version and exit - */ - while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) { + while ((i = getopt(argc, argv, "af:ns:NTVrq")) != EOF) { switch (i) { case 'V': - // printf("%s (%s, build %s)\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_build.datetime); - exit(EXIT_SUCCESS); - break; + printf("mdbx_load version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + MDBX_version.major, MDBX_version.minor, MDBX_version.release, + MDBX_version.revision, MDBX_version.git.describe, + MDBX_version.git.datetime, MDBX_version.git.commit, + MDBX_version.git.tree, MDBX_sourcery_anchor, MDBX_build.datetime, + MDBX_build.target, MDBX_build.compiler, MDBX_build.flags, + MDBX_build.options); + return EXIT_SUCCESS; case 'a': append = 1; break; case 'f': if (freopen(optarg, "r", stdin) == NULL) { - fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno)); + fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, + mdbx_strerror(errno)); exit(EXIT_FAILURE); } break; @@ -371,6 +415,12 @@ int main(int argc, char *argv[]) { case 'T': mode |= NOHDR | PRINT; break; + case 'q': + quiet = 1; + break; + case 'r': + rescue = 1; + break; default: usage(); } @@ -392,6 +442,13 @@ int main(int argc, char *argv[]) { signal(SIGTERM, signal_handler); #endif /* !WINDOWS */ + envname = argv[optind]; + if (!quiet) + printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", + MDBX_version.git.describe, MDBX_version.git.datetime, + MDBX_version.git.tree, envname); + fflush(NULL); + dbuf.iov_len = 4096; dbuf.iov_base = mdbx_malloc(dbuf.iov_len); @@ -399,7 +456,6 @@ int main(int argc, char *argv[]) { if (!(mode & NOHDR)) readhdr(); - envname = argv[optind]; rc = mdbx_env_create(&env); if (rc) { fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc, @@ -409,23 +465,28 @@ int main(int argc, char *argv[]) { mdbx_env_set_maxdbs(env, 2); - if (envinfo.mi_maxreaders) - mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders); - - if (envinfo.mi_mapsize) { - if (envinfo.mi_mapsize > SIZE_MAX) { - fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc, - mdbx_strerror(MDBX_TOO_LARGE)); - return EXIT_FAILURE; - } - mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize); - } - #ifdef MDBX_FIXEDMAP if (info.mi_mapaddr) envflags |= MDBX_FIXEDMAP; #endif + if (envinfo.mi_mapsize) { + if (envinfo.mi_mapsize > INTPTR_MAX) { + fprintf(stderr, + "Database size is too large for current system (mapsize=%" PRIu64 + " is great than system-limit %zi)\n", + envinfo.mi_mapsize, INTPTR_MAX); + goto env_close; + } + rc = mdbx_env_set_geometry(env, 0, 0, (intptr_t)envinfo.mi_mapsize, -1, -1, + -1); + if (rc) { + fprintf(stderr, "mdbx_env_set_geometry failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + } + rc = mdbx_env_open(env, envname, envflags, 0664); if (rc) { fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc, @@ -433,8 +494,8 @@ int main(int argc, char *argv[]) { goto env_close; } - kbuf.iov_len = mdbx_env_get_maxkeysize(env); - if (kbuf.iov_len >= SIZE_MAX / 4) { + kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, MDBX_DUPSORT); + if (kbuf.iov_len >= INTPTR_MAX / 4) { fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n", kbuf.iov_len); goto env_close; @@ -503,6 +564,11 @@ int main(int argc, char *argv[]) { rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag); if (rc == MDBX_KEYEXIST && putflags) continue; + if (rc == MDBX_BAD_VALSIZE && rescue) { + fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, + mdbx_strerror(rc)); + continue; + } if (rc) { fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc, mdbx_strerror(rc)); @@ -539,6 +605,7 @@ int main(int argc, char *argv[]) { goto env_close; } mdbx_dbi_close(env, dbi); + subname = NULL; /* try read next header */ if (!(mode & NOHDR)) diff --git a/tools/mdbx_load/src/stdafx.h b/tools/mdbx_load/src/stdafx.h index c54474f43b..518027200a 100644 --- a/tools/mdbx_load/src/stdafx.h +++ b/tools/mdbx_load/src/stdafx.h @@ -21,56 +21,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. #include <stdint.h> #include <windows.h> -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "../../libs/libmdbx/src/src/elements/internals.h" - -#define CMP_UINT(x, y) { if ((x) != (y)) return (x) < (y) ? -1 : 1; } - -struct DBEventSortingKey -{ - uint32_t hContact; - uint32_t hEvent; - uint64_t ts; - - static int Compare(const MDBX_val *ax, const MDBX_val *bx) - { - const DBEventSortingKey *a = (DBEventSortingKey*)ax->iov_base; - const DBEventSortingKey *b = (DBEventSortingKey*)bx->iov_base; - - CMP_UINT(a->hContact, b->hContact); - CMP_UINT(a->ts, b->ts); - CMP_UINT(a->hEvent, b->hEvent); - return 0; - } -}; - -struct DBEventIdKey -{ - uint32_t iModuleId; // offset to a DBModuleName struct of the name of - char szEventId[256]; // string id - - static int Compare(const MDBX_val *ax, const MDBX_val *bx) - { - const DBEventIdKey *a = (DBEventIdKey*)ax->iov_base; - const DBEventIdKey *b = (DBEventIdKey*)bx->iov_base; - CMP_UINT(a->iModuleId, b->iModuleId); - return strcmp(a->szEventId, b->szEventId); - } -}; - -struct DBSettingKey -{ - uint32_t hContact; - uint32_t dwModuleId; - char szSettingName[1]; - - static int Compare(const MDBX_val *ax, const MDBX_val *bx) - { - const DBSettingKey *a = (DBSettingKey*)ax->iov_base; - const DBSettingKey *b = (DBSettingKey*)bx->iov_base; - - CMP_UINT(a->hContact, b->hContact); - CMP_UINT(a->dwModuleId, b->dwModuleId); - return strcmp(a->szSettingName, b->szSettingName); - } -}; +#define DECLARE_VERSION() \ + mdbx_version_info MDBX_version; \ + mdbx_build_info MDBX_build; \ + char* MDBX_sourcery_anchor; \ + HINSTANCE hDll = LoadLibraryA("libmdbx.mir"); \ + MDBX_version = *(mdbx_version_info *)GetProcAddress(hDll, "mdbx_version"); \ + MDBX_build = *(mdbx_build_info*)GetProcAddress(hDll, "mdbx_build"); \ + MDBX_sourcery_anchor = (char*)GetProcAddress(hDll, "mdbx_sourcery_MDBX_BUILD_SOURCERY"); |