From 4c9c071567afd9e759a92a3fbaf7bcca623c1822 Mon Sep 17 00:00:00 2001
From: George Hazan <ghazan@miranda.im>
Date: Mon, 2 Mar 2020 13:20:17 +0300
Subject: mdbx utilities:
- fixes #2239 (-a parameter is missing in fixme.cmd);
- merge of recent changes in mdbx utilities;
- crutch for Windows to print dynamically exported data
---
tools/mdbx_load/src/mdbx_load.cc | 235 +++++++++++++++++++++++++--------------
tools/mdbx_load/src/stdafx.h | 61 ++--------
2 files changed, 159 insertions(+), 137 deletions(-)
(limited to 'tools/mdbx_load')
diff --git a/tools/mdbx_load/src/mdbx_load.cc b/tools/mdbx_load/src/mdbx_load.cc
index deceaca224..567ad5b93a 100644
--- a/tools/mdbx_load/src/mdbx_load.cc
+++ b/tools/mdbx_load/src/mdbx_load.cc
@@ -1,7 +1,7 @@
/* mdbx_load.c - memory-mapped database load tool */
/*
- * Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
+ * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
@@ -58,7 +58,7 @@ static int version;
static int dbi_flags;
static char *prog;
-static int Eof;
+static bool Eof;
static MDBX_envinfo envinfo;
static MDBX_val kbuf, dbuf;
@@ -86,22 +86,34 @@ static void readhdr(void) {
char *ptr;
dbi_flags = 0;
- while (fgets((char *)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
+ while (fgets((char*)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
lineno++;
- if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) ||
- !strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
- /* LY: silently ignore information fields. */
+
+ if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize="))) {
+ envinfo.mi_dxb_pagesize =
+ atoi((char *)dbuf.iov_base + STRLENOF("db_pagesize="));
+ continue;
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
+ dbi_flags |= MDBX_DUPSORT;
continue;
- } else if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n",
prog, lineno, version);
exit(EXIT_FAILURE);
}
- } else if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) {
- break;
- } else if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) {
+ continue;
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END")))
+ return;
+
+ if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) {
if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print",
STRLENOF("print")))
mode |= PRINT;
@@ -111,21 +123,30 @@ static void readhdr(void) {
lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
- } else if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) {
+ continue;
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) {
ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
if (subname)
- free(subname);
- subname = strdup((char *)dbuf.iov_base + STRLENOF("database="));
- } else if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) {
+ mdbx_free(subname);
+ subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database="));
+ continue;
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree",
STRLENOF("btree"))) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("type="));
exit(EXIT_FAILURE);
}
- } else if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
+ continue;
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
@@ -137,7 +158,10 @@ static void readhdr(void) {
lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
- } else if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
+ continue;
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
@@ -149,7 +173,10 @@ static void readhdr(void) {
lineno, (char *)dbuf.iov_base + STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
- } else if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) {
+ continue;
+ }
+
+ if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) {
int i;
ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
@@ -161,31 +188,33 @@ static void readhdr(void) {
lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
- } else {
- int i;
- for (i = 0; dbflags[i].bit; i++) {
- if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
- ((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
- if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
- dbi_flags |= dbflags[i].bit;
- break;
- }
+ continue;
+ }
+
+ int i;
+ for (i = 0; dbflags[i].bit; i++) {
+ if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
+ ((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
+ if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
+ dbi_flags |= dbflags[i].bit;
+ break;
}
- if (!dbflags[i].bit) {
- ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len);
- if (!ptr) {
- fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog,
- lineno);
- exit(EXIT_FAILURE);
- } else {
- *ptr = '\0';
- fprintf(stderr,
- "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
- prog, lineno, (char *)dbuf.iov_base);
- }
+ }
+ if (!dbflags[i].bit) {
+ ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len);
+ if (!ptr) {
+ fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog,
+ lineno);
+ exit(EXIT_FAILURE);
+ } else {
+ *ptr = '\0';
+ fprintf(stderr,
+ "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
+ prog, lineno, (char *)dbuf.iov_base);
}
}
}
+ Eof = true;
}
static void badend(void) {
@@ -214,14 +243,14 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
- Eof = 1;
+ Eof = true;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
badend:
- Eof = 1;
+ Eof = true;
badend();
return EOF;
}
@@ -231,28 +260,28 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
}
}
if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
- Eof = 1;
+ Eof = true;
return EOF;
}
lineno++;
- c1 = (BYTE *)buf->iov_base;
+ c1 = (unsigned char *)buf->iov_base;
len = strlen((char *)c1);
l2 = len;
/* Is buffer too short? */
while (c1[len - 1] != '\n') {
- buf->iov_base = realloc(buf->iov_base, buf->iov_len * 2);
+ buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2);
if (!buf->iov_base) {
- Eof = 1;
+ Eof = true;
fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n",
prog, lineno);
return EOF;
}
- c1 = (BYTE *)buf->iov_base;
+ c1 = (unsigned char *)buf->iov_base;
c1 += l2;
if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) {
- Eof = 1;
+ Eof = true;
badend();
return EOF;
}
@@ -260,26 +289,25 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
len = strlen((char *)c1);
l2 += len;
}
- c1 = c2 = (BYTE*)buf->iov_base;
+ c1 = c2 = (unsigned char *)buf->iov_base;
len = l2;
c1[--len] = '\0';
end = c1 + len;
if (mode & PRINT) {
while (c2 < end) {
- if (*c2 == '\\') {
+ if (unlikely(*c2 == '\\')) {
if (c2[1] == '\\') {
- c1++;
- c2 += 2;
+ *c1++ = '\\';
} else {
if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
- Eof = 1;
+ Eof = true;
badend();
return EOF;
}
*c1++ = (char)unhex(++c2);
- c2 += 2;
}
+ c2 += 2;
} else {
/* copies are redundant when no escapes were used */
*c1++ = *c2++;
@@ -288,13 +316,13 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
} else {
/* odd length not allowed */
if (len & 1) {
- Eof = 1;
+ Eof = true;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
- Eof = 1;
+ Eof = true;
badend();
return EOF;
}
@@ -302,7 +330,7 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
c2 += 2;
}
}
- c2 = (BYTE*)(out->iov_base = buf->iov_base);
+ c2 = (unsigned char *)(out->iov_base = buf->iov_base);
out->iov_len = c1 - c2;
return 0;
@@ -310,7 +338,17 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
static void usage(void) {
fprintf(stderr,
- "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
+ "usage: %s [-V] [-q] [-a] [-f file] [-s name] [-N] [-T] [-r] [-n] "
+ "dbpath\n"
+ " -V\t\tprint version and exit\n"
+ " -q\t\tbe quiet\n"
+ " -a\t\tappend records in input order\n"
+ " -f file\tread from file instead of stdin\n"
+ " -s name\tload into named subDB\n"
+ " -N\t\tuse NOOVERWRITE on puts\n"
+ " -T\t\tread plaintext\n"
+ " -r\t\trescure mode (ignore errors to load corrupted DB dump)\n"
+ " -n\t\tNOSUBDIR mode for open\n",
prog);
exit(EXIT_FAILURE);
}
@@ -330,32 +368,38 @@ int main(int argc, char *argv[]) {
char *envname = NULL;
int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0;
int append = 0;
+ int quiet = 0;
+ int rescue = 0;
MDBX_val prevk;
+ DECLARE_VERSION();
prog = argv[0];
if (argc < 2)
usage();
- /* -a: append records in input order
- * -f: load file instead of stdin
- * -n: use NOSUBDIR flag on env_open
- * -s: load into named subDB
- * -N: use NOOVERWRITE on puts
- * -T: read plaintext
- * -V: print version and exit
- */
- while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
+ while ((i = getopt(argc, argv, "af:ns:NTVrq")) != EOF) {
switch (i) {
case 'V':
- // printf("%s (%s, build %s)\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_build.datetime);
- exit(EXIT_SUCCESS);
- break;
+ printf("mdbx_load version %d.%d.%d.%d\n"
+ " - source: %s %s, commit %s, tree %s\n"
+ " - anchor: %s\n"
+ " - build: %s for %s by %s\n"
+ " - flags: %s\n"
+ " - options: %s\n",
+ MDBX_version.major, MDBX_version.minor, MDBX_version.release,
+ MDBX_version.revision, MDBX_version.git.describe,
+ MDBX_version.git.datetime, MDBX_version.git.commit,
+ MDBX_version.git.tree, MDBX_sourcery_anchor, MDBX_build.datetime,
+ MDBX_build.target, MDBX_build.compiler, MDBX_build.flags,
+ MDBX_build.options);
+ return EXIT_SUCCESS;
case 'a':
append = 1;
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
- fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno));
+ fprintf(stderr, "%s: %s: open: %s\n", prog, optarg,
+ mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
@@ -371,6 +415,12 @@ int main(int argc, char *argv[]) {
case 'T':
mode |= NOHDR | PRINT;
break;
+ case 'q':
+ quiet = 1;
+ break;
+ case 'r':
+ rescue = 1;
+ break;
default:
usage();
}
@@ -392,6 +442,13 @@ int main(int argc, char *argv[]) {
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
+ envname = argv[optind];
+ if (!quiet)
+ printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n",
+ MDBX_version.git.describe, MDBX_version.git.datetime,
+ MDBX_version.git.tree, envname);
+ fflush(NULL);
+
dbuf.iov_len = 4096;
dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
@@ -399,7 +456,6 @@ int main(int argc, char *argv[]) {
if (!(mode & NOHDR))
readhdr();
- envname = argv[optind];
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
@@ -409,23 +465,28 @@ int main(int argc, char *argv[]) {
mdbx_env_set_maxdbs(env, 2);
- if (envinfo.mi_maxreaders)
- mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
-
- if (envinfo.mi_mapsize) {
- if (envinfo.mi_mapsize > SIZE_MAX) {
- fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc,
- mdbx_strerror(MDBX_TOO_LARGE));
- return EXIT_FAILURE;
- }
- mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize);
- }
-
#ifdef MDBX_FIXEDMAP
if (info.mi_mapaddr)
envflags |= MDBX_FIXEDMAP;
#endif
+ if (envinfo.mi_mapsize) {
+ if (envinfo.mi_mapsize > INTPTR_MAX) {
+ fprintf(stderr,
+ "Database size is too large for current system (mapsize=%" PRIu64
+ " is great than system-limit %zi)\n",
+ envinfo.mi_mapsize, INTPTR_MAX);
+ goto env_close;
+ }
+ rc = mdbx_env_set_geometry(env, 0, 0, (intptr_t)envinfo.mi_mapsize, -1, -1,
+ -1);
+ if (rc) {
+ fprintf(stderr, "mdbx_env_set_geometry failed, error %d %s\n", rc,
+ mdbx_strerror(rc));
+ goto env_close;
+ }
+ }
+
rc = mdbx_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
@@ -433,8 +494,8 @@ int main(int argc, char *argv[]) {
goto env_close;
}
- kbuf.iov_len = mdbx_env_get_maxkeysize(env);
- if (kbuf.iov_len >= SIZE_MAX / 4) {
+ kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, MDBX_DUPSORT);
+ if (kbuf.iov_len >= INTPTR_MAX / 4) {
fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n",
kbuf.iov_len);
goto env_close;
@@ -503,6 +564,11 @@ int main(int argc, char *argv[]) {
rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag);
if (rc == MDBX_KEYEXIST && putflags)
continue;
+ if (rc == MDBX_BAD_VALSIZE && rescue) {
+ fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno,
+ mdbx_strerror(rc));
+ continue;
+ }
if (rc) {
fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc,
mdbx_strerror(rc));
@@ -539,6 +605,7 @@ int main(int argc, char *argv[]) {
goto env_close;
}
mdbx_dbi_close(env, dbi);
+ subname = NULL;
/* try read next header */
if (!(mode & NOHDR))
diff --git a/tools/mdbx_load/src/stdafx.h b/tools/mdbx_load/src/stdafx.h
index c54474f43b..518027200a 100644
--- a/tools/mdbx_load/src/stdafx.h
+++ b/tools/mdbx_load/src/stdafx.h
@@ -21,56 +21,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
#include <stdint.h>
#include <windows.h>
-#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "../../libs/libmdbx/src/src/elements/internals.h"
-
-#define CMP_UINT(x, y) { if ((x) != (y)) return (x) < (y) ? -1 : 1; }
-
-struct DBEventSortingKey
-{
- uint32_t hContact;
- uint32_t hEvent;
- uint64_t ts;
-
- static int Compare(const MDBX_val *ax, const MDBX_val *bx)
- {
- const DBEventSortingKey *a = (DBEventSortingKey*)ax->iov_base;
- const DBEventSortingKey *b = (DBEventSortingKey*)bx->iov_base;
-
- CMP_UINT(a->hContact, b->hContact);
- CMP_UINT(a->ts, b->ts);
- CMP_UINT(a->hEvent, b->hEvent);
- return 0;
- }
-};
-
-struct DBEventIdKey
-{
- uint32_t iModuleId; // offset to a DBModuleName struct of the name of
- char szEventId[256]; // string id
-
- static int Compare(const MDBX_val *ax, const MDBX_val *bx)
- {
- const DBEventIdKey *a = (DBEventIdKey*)ax->iov_base;
- const DBEventIdKey *b = (DBEventIdKey*)bx->iov_base;
- CMP_UINT(a->iModuleId, b->iModuleId);
- return strcmp(a->szEventId, b->szEventId);
- }
-};
-
-struct DBSettingKey
-{
- uint32_t hContact;
- uint32_t dwModuleId;
- char szSettingName[1];
-
- static int Compare(const MDBX_val *ax, const MDBX_val *bx)
- {
- const DBSettingKey *a = (DBSettingKey*)ax->iov_base;
- const DBSettingKey *b = (DBSettingKey*)bx->iov_base;
-
- CMP_UINT(a->hContact, b->hContact);
- CMP_UINT(a->dwModuleId, b->dwModuleId);
- return strcmp(a->szSettingName, b->szSettingName);
- }
-};
+#define DECLARE_VERSION() \
+ mdbx_version_info MDBX_version; \
+ mdbx_build_info MDBX_build; \
+ char* MDBX_sourcery_anchor; \
+ HINSTANCE hDll = LoadLibraryA("libmdbx.mir"); \
+ MDBX_version = *(mdbx_version_info *)GetProcAddress(hDll, "mdbx_version"); \
+ MDBX_build = *(mdbx_build_info*)GetProcAddress(hDll, "mdbx_build"); \
+ MDBX_sourcery_anchor = (char*)GetProcAddress(hDll, "mdbx_sourcery_MDBX_BUILD_SOURCERY");
--
cgit v1.2.3