summaryrefslogtreecommitdiff
path: root/tools/mdbx_load/src/mdbx_load.cc
diff options
context:
space:
mode:
Diffstat (limited to 'tools/mdbx_load/src/mdbx_load.cc')
-rw-r--r--tools/mdbx_load/src/mdbx_load.cc79
1 files changed, 55 insertions, 24 deletions
diff --git a/tools/mdbx_load/src/mdbx_load.cc b/tools/mdbx_load/src/mdbx_load.cc
index 599677a160..0d7068a7e8 100644
--- a/tools/mdbx_load/src/mdbx_load.cc
+++ b/tools/mdbx_load/src/mdbx_load.cc
@@ -1,7 +1,7 @@
/* mdbx_load.c - memory-mapped database load tool */
/*
- * Copyright 2015-2018 Leonid Yuriev <leo@yuriev.ru>
+ * Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
@@ -59,6 +59,7 @@ static int Eof;
static MDBX_envinfo envinfo;
static MDBX_val kbuf, dbuf;
+static MDBX_val k0buf;
#define STRLENOF(s) (sizeof(s) - 1)
@@ -305,11 +306,18 @@ static int readline(MDBX_val *out, MDBX_val *buf) {
}
static void usage(void) {
- fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
+ fprintf(stderr,
+ "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
+static int anyway_greater(const MDBX_val *a, const MDBX_val *b) {
+ (void)a;
+ (void)b;
+ return 1;
+}
+
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env = NULL;
@@ -317,27 +325,31 @@ int main(int argc, char *argv[]) {
MDBX_cursor *mc = NULL;
MDBX_dbi dbi;
char *envname = NULL;
- int envflags = MDBX_EXCLUSIVE, putflags = 0;
+ int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0;
+ int append = 0;
+ MDBX_val prevk;
prog = argv[0];
-
- if (argc < 2) {
+ if (argc < 2)
usage();
- }
- /* -f: load file instead of stdin
+ /* -a: append records in input order
+ * -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
- while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
+ while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
switch (i) {
case 'V':
// printf("%s (%s, build %s)\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
break;
+ case 'a':
+ append = 1;
+ break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno));
@@ -348,7 +360,7 @@ int main(int argc, char *argv[]) {
envflags |= MDBX_NOSUBDIR;
break;
case 's':
- subname = strdup(optarg);
+ subname = mdbx_strdup(optarg);
break;
case 'N':
putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA;
@@ -378,8 +390,9 @@ int main(int argc, char *argv[]) {
#endif /* !WINDOWS */
dbuf.iov_len = 4096;
- dbuf.iov_base = malloc(dbuf.iov_len);
+ dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
+ /* read first header for mapsize= */
if (!(mode & NOHDR))
readhdr();
@@ -417,8 +430,17 @@ int main(int argc, char *argv[]) {
goto env_close;
}
- kbuf.iov_len = mdbx_env_get_maxkeysize(env) * 2 + 2;
- kbuf.iov_base = malloc(kbuf.iov_len);
+ kbuf.iov_len = mdbx_env_get_maxkeysize(env);
+ if (kbuf.iov_len >= SIZE_MAX / 4) {
+ fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n",
+ kbuf.iov_len);
+ goto env_close;
+ }
+ kbuf.iov_len = (kbuf.iov_len + 1) * 2;
+ kbuf.iov_base = malloc(kbuf.iov_len * 2);
+ k0buf.iov_len = kbuf.iov_len;
+ k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len;
+ prevk.iov_base = k0buf.iov_base;
while (!Eof) {
if (user_break) {
@@ -426,9 +448,6 @@ int main(int argc, char *argv[]) {
break;
}
- MDBX_val key, data;
- int batch = 0;
-
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
@@ -436,14 +455,9 @@ int main(int argc, char *argv[]) {
goto env_close;
}
- if (!stricmp(subname, "eventids"))
- rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, &DBEventIdKey::Compare, nullptr);
- else if (!stricmp(subname, "eventsrt"))
- rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, &DBEventSortingKey::Compare, nullptr);
- else if (!stricmp(subname, "settings"))
- rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, &DBSettingKey::Compare, nullptr);
- else
- rc = mdbx_dbi_open(txn, subname, dbi_flags | MDBX_CREATE, &dbi);
+ rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
+ append ? anyway_greater : NULL,
+ append ? anyway_greater : NULL);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
@@ -456,11 +470,15 @@ int main(int argc, char *argv[]) {
goto txn_abort;
}
+ int batch = 0;
+ prevk.iov_len = 0;
while (1) {
+ MDBX_val key;
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
+ MDBX_val data;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
@@ -468,7 +486,18 @@ int main(int argc, char *argv[]) {
goto txn_abort;
}
- rc = mdbx_cursor_put(mc, &key, &data, putflags);
+ int appflag = 0;
+ if (append) {
+ appflag = MDBX_APPEND;
+ if (dbi_flags & MDBX_DUPSORT) {
+ if (prevk.iov_len == key.iov_len &&
+ memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0)
+ appflag = MDBX_APPEND | MDBX_APPENDDUP;
+ else
+ memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len);
+ }
+ }
+ rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag);
if (rc == MDBX_KEYEXIST && putflags)
continue;
if (rc) {
@@ -507,6 +536,8 @@ int main(int argc, char *argv[]) {
goto env_close;
}
mdbx_dbi_close(env, dbi);
+
+ /* try read next header */
if (!(mode & NOHDR))
readhdr();
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-16 09:40:03 +0000