mdbx utilities:

- fixes #2239 (-a parameter is missing in fixme.cmd);
- merge of recent changes in mdbx utilities;
- crutch for Windows to print dynamically exported data

2 files changed, 159 insertions, 137 deletions

diff --git a/tools/mdbx_load/src/mdbx_load.cc b/tools/mdbx_load/src/mdbx_load.cc
index deceaca224..567ad5b93a 100644
--- a/tools/mdbx_load/src/mdbx_load.cc
+++ b/tools/mdbx_load/src/mdbx_load.cc
@@ -1,7 +1,7 @@
 /* mdbx_load.c - memory-mapped database load tool */
 
 /*
- * Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
+ * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>
  * and other libmdbx authors: please see AUTHORS file.
  * All rights reserved.
  *
@@ -58,7 +58,7 @@ static int version;
 
 static int dbi_flags;
 static char *prog;
-static int Eof;
+static bool Eof;
 
 static MDBX_envinfo envinfo;
 static MDBX_val kbuf, dbuf;
@@ -86,22 +86,34 @@ static void readhdr(void) {
   char *ptr;
 
   dbi_flags = 0;
-  while (fgets((char *)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
+  while (fgets((char*)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
     lineno++;
-    if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) ||
-        !strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
-      /* LY: silently ignore information fields. */
+
+    if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize="))) {
+      envinfo.mi_dxb_pagesize =
+          atoi((char *)dbuf.iov_base + STRLENOF("db_pagesize="));
+      continue;
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
+      dbi_flags |= MDBX_DUPSORT;
       continue;
-    } else if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
       version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION="));
       if (version > 3) {
         fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n",
                 prog, lineno, version);
         exit(EXIT_FAILURE);
       }
-    } else if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) {
-      break;
-    } else if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) {
+      continue;
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END")))
+      return;
+
+    if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) {
       if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print",
                    STRLENOF("print")))
         mode |= PRINT;
@@ -111,21 +123,30 @@ static void readhdr(void) {
                 lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT="));
         exit(EXIT_FAILURE);
       }
-    } else if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) {
+      continue;
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) {
       ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
       if (ptr)
         *ptr = '\0';
       if (subname)
-        free(subname);
-      subname = strdup((char *)dbuf.iov_base + STRLENOF("database="));
-    } else if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) {
+        mdbx_free(subname);
+      subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database="));
+      continue;
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) {
       if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree",
                   STRLENOF("btree"))) {
         fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog,
                 lineno, (char *)dbuf.iov_base + STRLENOF("type="));
         exit(EXIT_FAILURE);
       }
-    } else if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
+      continue;
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
       int i;
       ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
       if (ptr)
@@ -137,7 +158,10 @@ static void readhdr(void) {
                 lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr="));
         exit(EXIT_FAILURE);
       }
-    } else if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
+      continue;
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
       int i;
       ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
       if (ptr)
@@ -149,7 +173,10 @@ static void readhdr(void) {
                 lineno, (char *)dbuf.iov_base + STRLENOF("mapsize="));
         exit(EXIT_FAILURE);
       }
-    } else if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) {
+      continue;
+    }
+
+    if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) {
       int i;
       ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len);
       if (ptr)
@@ -161,31 +188,33 @@ static void readhdr(void) {
                 lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders="));
         exit(EXIT_FAILURE);
       }
-    } else {
-      int i;
-      for (i = 0; dbflags[i].bit; i++) {
-        if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
-            ((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
-          if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
-            dbi_flags |= dbflags[i].bit;
-          break;
-        }
+      continue;
+    }
+
+    int i;
+    for (i = 0; dbflags[i].bit; i++) {
+      if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
+          ((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
+        if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
+          dbi_flags |= dbflags[i].bit;
+        break;
       }
-      if (!dbflags[i].bit) {
-        ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len);
-        if (!ptr) {
-          fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog,
-                  lineno);
-          exit(EXIT_FAILURE);
-        } else {
-          *ptr = '\0';
-          fprintf(stderr,
-                  "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
-                  prog, lineno, (char *)dbuf.iov_base);
-        }
+    }
+    if (!dbflags[i].bit) {
+      ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len);
+      if (!ptr) {
+        fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog,
+                lineno);
+        exit(EXIT_FAILURE);
+      } else {
+        *ptr = '\0';
+        fprintf(stderr,
+                "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
+                prog, lineno, (char *)dbuf.iov_base);
       }
     }
   }
+  Eof = true;
 }
 
 static void badend(void) {
@@ -214,14 +243,14 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
   if (!(mode & NOHDR)) {
     c = fgetc(stdin);
     if (c == EOF) {
-      Eof = 1;
+      Eof = true;
       return EOF;
     }
     if (c != ' ') {
       lineno++;
       if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
       badend:
-        Eof = 1;
+        Eof = true;
         badend();
         return EOF;
       }
@@ -231,28 +260,28 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
     }
   }
   if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
-    Eof = 1;
+    Eof = true;
     return EOF;
   }
   lineno++;
 
-  c1 = (BYTE *)buf->iov_base;
+  c1 = (unsigned char *)buf->iov_base;
   len = strlen((char *)c1);
   l2 = len;
 
   /* Is buffer too short? */
   while (c1[len - 1] != '\n') {
-    buf->iov_base = realloc(buf->iov_base, buf->iov_len * 2);
+    buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2);
     if (!buf->iov_base) {
-      Eof = 1;
+      Eof = true;
       fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n",
               prog, lineno);
       return EOF;
     }
-    c1 = (BYTE *)buf->iov_base;
+    c1 = (unsigned char *)buf->iov_base;
     c1 += l2;
     if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) {
-      Eof = 1;
+      Eof = true;
       badend();
       return EOF;
     }
@@ -260,26 +289,25 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
     len = strlen((char *)c1);
     l2 += len;
   }
-  c1 = c2 = (BYTE*)buf->iov_base;
+  c1 = c2 = (unsigned char *)buf->iov_base;
   len = l2;
   c1[--len] = '\0';
   end = c1 + len;
 
   if (mode & PRINT) {
     while (c2 < end) {
-      if (*c2 == '\\') {
+      if (unlikely(*c2 == '\\')) {
         if (c2[1] == '\\') {
-          c1++;
-          c2 += 2;
+          *c1++ = '\\';
         } else {
           if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
-            Eof = 1;
+            Eof = true;
             badend();
             return EOF;
           }
           *c1++ = (char)unhex(++c2);
-          c2 += 2;
         }
+        c2 += 2;
       } else {
         /* copies are redundant when no escapes were used */
         *c1++ = *c2++;
@@ -288,13 +316,13 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
   } else {
     /* odd length not allowed */
     if (len & 1) {
-      Eof = 1;
+      Eof = true;
       badend();
       return EOF;
     }
     while (c2 < end) {
       if (!isxdigit(*c2) || !isxdigit(c2[1])) {
-        Eof = 1;
+        Eof = true;
         badend();
         return EOF;
       }
@@ -302,7 +330,7 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
       c2 += 2;
     }
   }
-  c2 = (BYTE*)(out->iov_base = buf->iov_base);
+  c2 = (unsigned char *)(out->iov_base = buf->iov_base);
   out->iov_len = c1 - c2;
 
   return 0;
@@ -310,7 +338,17 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
 
 static void usage(void) {
   fprintf(stderr,
-          "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
+          "usage: %s [-V] [-q] [-a] [-f file] [-s name] [-N] [-T] [-r] [-n] "
+          "dbpath\n"
+          "  -V\t\tprint version and exit\n"
+          "  -q\t\tbe quiet\n"
+          "  -a\t\tappend records in input order\n"
+          "  -f file\tread from file instead of stdin\n"
+          "  -s name\tload into named subDB\n"
+          "  -N\t\tuse NOOVERWRITE on puts\n"
+          "  -T\t\tread plaintext\n"
+          "  -r\t\trescure mode (ignore errors to load corrupted DB dump)\n"
+          "  -n\t\tNOSUBDIR mode for open\n",
           prog);
   exit(EXIT_FAILURE);
 }
@@ -330,32 +368,38 @@ int main(int argc, char *argv[]) {
   char *envname = NULL;
   int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0;
   int append = 0;
+  int quiet = 0;
+  int rescue = 0;
   MDBX_val prevk;
+  DECLARE_VERSION();
 
   prog = argv[0];
   if (argc < 2)
     usage();
 
-  /* -a: append records in input order
-   * -f: load file instead of stdin
-   * -n: use NOSUBDIR flag on env_open
-   * -s: load into named subDB
-   * -N: use NOOVERWRITE on puts
-   * -T: read plaintext
-   * -V: print version and exit
-   */
-  while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
+  while ((i = getopt(argc, argv, "af:ns:NTVrq")) != EOF) {
     switch (i) {
     case 'V':
-      // printf("%s (%s, build %s)\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_build.datetime);
-      exit(EXIT_SUCCESS);
-      break;
+      printf("mdbx_load version %d.%d.%d.%d\n"
+             " - source: %s %s, commit %s, tree %s\n"
+             " - anchor: %s\n"
+             " - build: %s for %s by %s\n"
+             " - flags: %s\n"
+             " - options: %s\n",
+             MDBX_version.major, MDBX_version.minor, MDBX_version.release,
+             MDBX_version.revision, MDBX_version.git.describe,
+             MDBX_version.git.datetime, MDBX_version.git.commit,
+             MDBX_version.git.tree, MDBX_sourcery_anchor, MDBX_build.datetime,
+             MDBX_build.target, MDBX_build.compiler, MDBX_build.flags,
+             MDBX_build.options);
+      return EXIT_SUCCESS;
     case 'a':
       append = 1;
       break;
     case 'f':
       if (freopen(optarg, "r", stdin) == NULL) {
-        fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno));
+        fprintf(stderr, "%s: %s: open: %s\n", prog, optarg,
+                mdbx_strerror(errno));
         exit(EXIT_FAILURE);
       }
       break;
@@ -371,6 +415,12 @@ int main(int argc, char *argv[]) {
     case 'T':
       mode |= NOHDR | PRINT;
       break;
+    case 'q':
+      quiet = 1;
+      break;
+    case 'r':
+      rescue = 1;
+      break;
     default:
       usage();
     }
@@ -392,6 +442,13 @@ int main(int argc, char *argv[]) {
   signal(SIGTERM, signal_handler);
 #endif /* !WINDOWS */
 
+  envname = argv[optind];
+  if (!quiet)
+    printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n",
+           MDBX_version.git.describe, MDBX_version.git.datetime,
+           MDBX_version.git.tree, envname);
+  fflush(NULL);
+
   dbuf.iov_len = 4096;
   dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
 
@@ -399,7 +456,6 @@ int main(int argc, char *argv[]) {
   if (!(mode & NOHDR))
     readhdr();
 
-  envname = argv[optind];
   rc = mdbx_env_create(&env);
   if (rc) {
     fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
@@ -409,23 +465,28 @@ int main(int argc, char *argv[]) {
 
   mdbx_env_set_maxdbs(env, 2);
 
-  if (envinfo.mi_maxreaders)
-    mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
-
-  if (envinfo.mi_mapsize) {
-    if (envinfo.mi_mapsize > SIZE_MAX) {
-      fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc,
-              mdbx_strerror(MDBX_TOO_LARGE));
-      return EXIT_FAILURE;
-    }
-    mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize);
-  }
-
 #ifdef MDBX_FIXEDMAP
   if (info.mi_mapaddr)
     envflags |= MDBX_FIXEDMAP;
 #endif
 
+  if (envinfo.mi_mapsize) {
+    if (envinfo.mi_mapsize > INTPTR_MAX) {
+      fprintf(stderr,
+              "Database size is too large for current system (mapsize=%" PRIu64
+              " is great than system-limit %zi)\n",
+              envinfo.mi_mapsize, INTPTR_MAX);
+      goto env_close;
+    }
+    rc = mdbx_env_set_geometry(env, 0, 0, (intptr_t)envinfo.mi_mapsize, -1, -1,
+                               -1);
+    if (rc) {
+      fprintf(stderr, "mdbx_env_set_geometry failed, error %d %s\n", rc,
+              mdbx_strerror(rc));
+      goto env_close;
+    }
+  }
+
   rc = mdbx_env_open(env, envname, envflags, 0664);
   if (rc) {
     fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
@@ -433,8 +494,8 @@ int main(int argc, char *argv[]) {
     goto env_close;
   }
 
-  kbuf.iov_len = mdbx_env_get_maxkeysize(env);
-  if (kbuf.iov_len >= SIZE_MAX / 4) {
+  kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, MDBX_DUPSORT);
+  if (kbuf.iov_len >= INTPTR_MAX / 4) {
     fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n",
             kbuf.iov_len);
     goto env_close;
@@ -503,6 +564,11 @@ int main(int argc, char *argv[]) {
       rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag);
       if (rc == MDBX_KEYEXIST && putflags)
         continue;
+      if (rc == MDBX_BAD_VALSIZE && rescue) {
+        fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno,
+                mdbx_strerror(rc));
+        continue;
+      }
       if (rc) {
         fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc,
                 mdbx_strerror(rc));
@@ -539,6 +605,7 @@ int main(int argc, char *argv[]) {
       goto env_close;
     }
     mdbx_dbi_close(env, dbi);
+    subname = NULL;
 
     /* try read next header */
     if (!(mode & NOHDR))
diff --git a/tools/mdbx_load/src/stdafx.h b/tools/mdbx_load/src/stdafx.h
index c54474f43b..518027200a 100644
--- a/tools/mdbx_load/src/stdafx.h
+++ b/tools/mdbx_load/src/stdafx.h
@@ -21,56 +21,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 #include <stdint.h>
 #include <windows.h>
 
-#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
-#include "../../libs/libmdbx/src/src/elements/internals.h"
-
-#define CMP_UINT(x, y) { if ((x) != (y)) return (x) < (y) ? -1 : 1; }
-
-struct DBEventSortingKey
-{
-	uint32_t hContact;
-	uint32_t hEvent;
-	uint64_t ts;
-
-	static int Compare(const MDBX_val *ax, const MDBX_val *bx)
-	{
-		const DBEventSortingKey *a = (DBEventSortingKey*)ax->iov_base;
-		const DBEventSortingKey *b = (DBEventSortingKey*)bx->iov_base;
-
-		CMP_UINT(a->hContact, b->hContact);
-		CMP_UINT(a->ts, b->ts);
-		CMP_UINT(a->hEvent, b->hEvent);
-		return 0;
-	}
-};
-
-struct DBEventIdKey
-{
-	uint32_t iModuleId;	    // offset to a DBModuleName struct of the name of
-	char     szEventId[256]; // string id
-
-	static int Compare(const MDBX_val *ax, const MDBX_val *bx)
-	{
-		const DBEventIdKey *a = (DBEventIdKey*)ax->iov_base;
-		const DBEventIdKey *b = (DBEventIdKey*)bx->iov_base;
-		CMP_UINT(a->iModuleId, b->iModuleId);
-		return strcmp(a->szEventId, b->szEventId);
-	}
-};
-
-struct DBSettingKey
-{
-	uint32_t hContact;
-	uint32_t dwModuleId;
-	char     szSettingName[1];
-
-	static int Compare(const MDBX_val *ax, const MDBX_val *bx)
-	{
-		const DBSettingKey *a = (DBSettingKey*)ax->iov_base;
-		const DBSettingKey *b = (DBSettingKey*)bx->iov_base;
-
-		CMP_UINT(a->hContact, b->hContact);
-		CMP_UINT(a->dwModuleId, b->dwModuleId);
-		return strcmp(a->szSettingName, b->szSettingName);
-	}
-};
+#define DECLARE_VERSION() \
+   mdbx_version_info MDBX_version; \
+   mdbx_build_info MDBX_build; \
+   char* MDBX_sourcery_anchor; \
+   HINSTANCE hDll = LoadLibraryA("libmdbx.mir"); \
+   MDBX_version = *(mdbx_version_info *)GetProcAddress(hDll, "mdbx_version"); \
+   MDBX_build = *(mdbx_build_info*)GetProcAddress(hDll, "mdbx_build"); \
+   MDBX_sourcery_anchor = (char*)GetProcAddress(hDll, "mdbx_sourcery_MDBX_BUILD_SOURCERY");