diff options
author | George Hazan <ghazan@miranda.im> | 2020-05-22 20:25:34 +0300 |
---|---|---|
committer | George Hazan <ghazan@miranda.im> | 2020-05-22 20:25:34 +0300 |
commit | 4ba812b71eb6e3cbefcea50d71dd8b7b98779a73 (patch) | |
tree | 89f8527d0e01ccaa96b3f9498dad1073e512b8c6 /tools/mdbx_load/src | |
parent | e705d77b5ce00ddcddb4584b6b3df3e538a86b46 (diff) |
Database
- libmdbx inlined not to change the code of utilities
- libmdbx.mir removed from build scripts
Diffstat (limited to 'tools/mdbx_load/src')
-rw-r--r-- | tools/mdbx_load/src/mdbx_load.cc | 622 | ||||
-rw-r--r-- | tools/mdbx_load/src/stdafx.cxx | 20 | ||||
-rw-r--r-- | tools/mdbx_load/src/stdafx.h | 31 |
3 files changed, 0 insertions, 673 deletions
diff --git a/tools/mdbx_load/src/mdbx_load.cc b/tools/mdbx_load/src/mdbx_load.cc deleted file mode 100644 index d9f7f872bd..0000000000 --- a/tools/mdbx_load/src/mdbx_load.cc +++ /dev/null @@ -1,622 +0,0 @@ -/* mdbx_load.c - memory-mapped database load tool */ - -/* - * Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru> - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * <http://www.OpenLDAP.org/license.html>. */ - -#include "stdafx.h" - -#ifdef _MSC_VER -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif -#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ -#endif /* _MSC_VER (warnings) */ - -#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ -#include "../../libs/libmdbx/src/src/internals.h" - -#include <ctype.h> - -#if defined(_WIN32) || defined(_WIN64) -#include "../../libs/libmdbx/src/src/wingetopt.h" -#include "../../libs/libmdbx/src/src/wingetopt.c" - -static volatile BOOL user_break; -static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { - (void)dwCtrlType; - user_break = true; - return true; -} - -#else /* WINDOWS */ - -static volatile sig_atomic_t user_break; -static void signal_handler(int sig) { - (void)sig; - user_break = 1; -} - -#endif /* !WINDOWS */ - -#define PRINT 1 -#define NOHDR 2 -static int mode; - -static char *subname = NULL; -static size_t lineno; -static int version; - -static int dbi_flags; -static char *prog; -static bool Eof; - -static MDBX_envinfo envinfo; -static MDBX_val kbuf, dbuf; -static MDBX_val k0buf; - -#define STRLENOF(s) (sizeof(s) - 1) - -typedef struct flagbit { - int bit; - char *name; - int len; -} flagbit; - -#define S(s) s, STRLENOF(s) - -flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")}, - {MDBX_DUPSORT, S("dupsort")}, - {MDBX_INTEGERKEY, S("integerkey")}, - {MDBX_DUPFIXED, S("dupfixed")}, - {MDBX_INTEGERDUP, S("integerdup")}, - {MDBX_REVERSEDUP, S("reversedup")}, - {0, NULL, 0}}; - -static void readhdr(void) { - char *ptr; - - dbi_flags = 0; - while (fgets((char*)dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) { - lineno++; - - if (!strncmp((char *)dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize="))) { - envinfo.mi_dxb_pagesize = - atoi((char *)dbuf.iov_base + STRLENOF("db_pagesize=")); - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) { - dbi_flags |= MDBX_DUPSORT; - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) { - version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION=")); - if (version > 3) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n", - prog, lineno, version); - exit(EXIT_FAILURE); - } - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) - return; - - if (!strncmp((char *)dbuf.iov_base, "format=", STRLENOF("format="))) { - if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print", - STRLENOF("print"))) - mode |= PRINT; - else if (strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "bytevalue", - STRLENOF("bytevalue"))) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported FORMAT %s\n", prog, - lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT=")); - exit(EXIT_FAILURE); - } - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "database=", STRLENOF("database="))) { - ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); - if (ptr) - *ptr = '\0'; - if (subname) - mdbx_free(subname); - subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database=")); - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "type=", STRLENOF("type="))) { - if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree", - STRLENOF("btree"))) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog, - lineno, (char *)dbuf.iov_base + STRLENOF("type=")); - exit(EXIT_FAILURE); - } - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) { - int i; - ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); - if (ptr) - *ptr = '\0'; - void *unused; - i = sscanf((char *)dbuf.iov_base + STRLENOF("mapaddr="), "%p", &unused); - if (i != 1) { - fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapaddr %s\n", prog, - lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr=")); - exit(EXIT_FAILURE); - } - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) { - int i; - ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); - if (ptr) - *ptr = '\0'; - i = sscanf((char *)dbuf.iov_base + STRLENOF("mapsize="), "%" PRIu64, - &envinfo.mi_mapsize); - if (i != 1) { - fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapsize %s\n", prog, - lineno, (char *)dbuf.iov_base + STRLENOF("mapsize=")); - exit(EXIT_FAILURE); - } - continue; - } - - if (!strncmp((char *)dbuf.iov_base, "maxreaders=", STRLENOF("maxreaders="))) { - int i; - ptr = (char *)memchr(dbuf.iov_base, '\n', dbuf.iov_len); - if (ptr) - *ptr = '\0'; - i = sscanf((char *)dbuf.iov_base + STRLENOF("maxreaders="), "%u", - &envinfo.mi_maxreaders); - if (i != 1) { - fprintf(stderr, "%s: line %" PRIiSIZE ": invalid maxreaders %s\n", prog, - lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders=")); - exit(EXIT_FAILURE); - } - continue; - } - - int i; - for (i = 0; dbflags[i].bit; i++) { - if (!strncmp((char *)dbuf.iov_base, dbflags[i].name, dbflags[i].len) && - ((char *)dbuf.iov_base)[dbflags[i].len] == '=') { - if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1') - dbi_flags |= dbflags[i].bit; - break; - } - } - if (!dbflags[i].bit) { - ptr = (char *)memchr(dbuf.iov_base, '=', dbuf.iov_len); - if (!ptr) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog, - lineno); - exit(EXIT_FAILURE); - } else { - *ptr = '\0'; - fprintf(stderr, - "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", - prog, lineno, (char *)dbuf.iov_base); - } - } - } - Eof = true; -} - -static void badend(void) { - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, - lineno); -} - -static int unhex(unsigned char *c2) { - int x, c; - x = *c2++ & 0x4f; - if (x & 0x40) - x -= 55; - c = x << 4; - x = *c2 & 0x4f; - if (x & 0x40) - x -= 55; - c |= x; - return c; -} - -static int readline(MDBX_val *out, MDBX_val *buf) { - unsigned char *c1, *c2, *end; - size_t len, l2; - int c; - - if (!(mode & NOHDR)) { - c = fgetc(stdin); - if (c == EOF) { - Eof = true; - return EOF; - } - if (c != ' ') { - lineno++; - if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) { - badend: - Eof = true; - badend(); - return EOF; - } - if (c == 'D' && !strncmp((char *)buf->iov_base, "ATA=END", STRLENOF("ATA=END"))) - return EOF; - goto badend; - } - } - if (fgets((char *)buf->iov_base, (int)buf->iov_len, stdin) == NULL) { - Eof = true; - return EOF; - } - lineno++; - - c1 = (unsigned char *)buf->iov_base; - len = strlen((char *)c1); - l2 = len; - - /* Is buffer too short? */ - while (c1[len - 1] != '\n') { - buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2); - if (!buf->iov_base) { - Eof = true; - fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", - prog, lineno); - return EOF; - } - c1 = (unsigned char *)buf->iov_base; - c1 += l2; - if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) { - Eof = true; - badend(); - return EOF; - } - buf->iov_len *= 2; - len = strlen((char *)c1); - l2 += len; - } - c1 = c2 = (unsigned char *)buf->iov_base; - len = l2; - c1[--len] = '\0'; - end = c1 + len; - - if (mode & PRINT) { - while (c2 < end) { - if (unlikely(*c2 == '\\')) { - if (c2[1] == '\\') { - *c1++ = '\\'; - } else { - if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { - Eof = true; - badend(); - return EOF; - } - *c1++ = (char)unhex(++c2); - } - c2 += 2; - } else { - /* copies are redundant when no escapes were used */ - *c1++ = *c2++; - } - } - } else { - /* odd length not allowed */ - if (len & 1) { - Eof = true; - badend(); - return EOF; - } - while (c2 < end) { - if (!isxdigit(*c2) || !isxdigit(c2[1])) { - Eof = true; - badend(); - return EOF; - } - *c1++ = (char)unhex(c2); - c2 += 2; - } - } - c2 = (unsigned char *)(out->iov_base = buf->iov_base); - out->iov_len = c1 - c2; - - return 0; -} - -static void usage(void) { - fprintf(stderr, - "usage: %s [-V] [-q] [-a] [-f file] [-s name] [-N] [-T] [-r] [-n] " - "dbpath\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -a\t\tappend records in input order (required for custom " - "comparators)\n" - " -f file\tread from file instead of stdin\n" - " -s name\tload into named subDB\n" - " -N\t\tuse NOOVERWRITE on puts\n" - " -T\t\tread plaintext\n" - " -r\t\trescure mode (ignore errors to load corrupted DB dump)\n" - " -n\t\tNOSUBDIR mode for open\n", - prog); - exit(EXIT_FAILURE); -} - -static int anyway_greater(const MDBX_val *a, const MDBX_val *b) { - (void)a; - (void)b; - return 1; -} - -int main(int argc, char *argv[]) { - int i, rc; - MDBX_env *env = NULL; - MDBX_txn *txn = NULL; - MDBX_cursor *mc = NULL; - MDBX_dbi dbi; - char *envname = NULL; - int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0; - int append = 0; - int quiet = 0; - int rescue = 0; - MDBX_val prevk; - DECLARE_VERSION(); - - prog = argv[0]; - if (argc < 2) - usage(); - - while ((i = getopt(argc, argv, "af:ns:NTVrq")) != EOF) { - switch (i) { - case 'V': - printf("mdbx_load version %d.%d.%d.%d\n" - " - source: %s %s, commit %s, tree %s\n" - " - anchor: %s\n" - " - build: %s for %s by %s\n" - " - flags: %s\n" - " - options: %s\n", - MDBX_version.major, MDBX_version.minor, MDBX_version.release, - MDBX_version.revision, MDBX_version.git.describe, - MDBX_version.git.datetime, MDBX_version.git.commit, - MDBX_version.git.tree, MDBX_sourcery_anchor, MDBX_build.datetime, - MDBX_build.target, MDBX_build.compiler, MDBX_build.flags, - MDBX_build.options); - return EXIT_SUCCESS; - case 'a': - append = 1; - break; - case 'f': - if (freopen(optarg, "r", stdin) == NULL) { - fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, - mdbx_strerror(errno)); - exit(EXIT_FAILURE); - } - break; - case 'n': - envflags |= MDBX_NOSUBDIR; - break; - case 's': - subname = mdbx_strdup(optarg); - break; - case 'N': - putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA; - break; - case 'T': - mode |= NOHDR | PRINT; - break; - case 'q': - quiet = 1; - break; - case 'r': - rescue = 1; - break; - default: - usage(); - } - } - - if (optind != argc - 1) - usage(); - -#if defined(_WIN32) || defined(_WIN64) - SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); -#else -#ifdef SIGPIPE - signal(SIGPIPE, signal_handler); -#endif -#ifdef SIGHUP - signal(SIGHUP, signal_handler); -#endif - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); -#endif /* !WINDOWS */ - - envname = argv[optind]; - if (!quiet) - printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", - MDBX_version.git.describe, MDBX_version.git.datetime, - MDBX_version.git.tree, envname); - fflush(NULL); - - dbuf.iov_len = 4096; - dbuf.iov_base = mdbx_malloc(dbuf.iov_len); - - /* read first header for mapsize= */ - if (!(mode & NOHDR)) - readhdr(); - - rc = mdbx_env_create(&env); - if (rc) { - fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc, - mdbx_strerror(rc)); - return EXIT_FAILURE; - } - - mdbx_env_set_maxdbs(env, 2); - -#ifdef MDBX_FIXEDMAP - if (info.mi_mapaddr) - envflags |= MDBX_FIXEDMAP; -#endif - - if (envinfo.mi_mapsize) { - if (envinfo.mi_mapsize > INTPTR_MAX) { - fprintf(stderr, - "Database size is too large for current system (mapsize=%" PRIu64 - " is great than system-limit %zi)\n", - envinfo.mi_mapsize, INTPTR_MAX); - goto env_close; - } - rc = mdbx_env_set_geometry(env, 0, 0, (intptr_t)envinfo.mi_mapsize, -1, -1, - -1); - if (rc) { - fprintf(stderr, "mdbx_env_set_geometry failed, error %d %s\n", rc, - mdbx_strerror(rc)); - goto env_close; - } - } - - rc = mdbx_env_open(env, envname, envflags, 0664); - if (rc) { - fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc, - mdbx_strerror(rc)); - goto env_close; - } - - kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, MDBX_DUPSORT); - if (kbuf.iov_len >= INTPTR_MAX / 4) { - fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n", - kbuf.iov_len); - goto env_close; - } - kbuf.iov_len = (kbuf.iov_len + 1) * 2; - kbuf.iov_base = malloc(kbuf.iov_len * 2); - k0buf.iov_len = kbuf.iov_len; - k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len; - prevk.iov_base = k0buf.iov_base; - - while (!Eof) { - if (user_break) { - rc = MDBX_EINTR; - break; - } - - rc = mdbx_txn_begin(env, NULL, 0, &txn); - if (rc) { - fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc, - mdbx_strerror(rc)); - goto env_close; - } - - rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, - append ? anyway_greater : NULL, - append ? anyway_greater : NULL); - if (rc) { - fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto txn_abort; - } - - rc = mdbx_cursor_open(txn, dbi, &mc); - if (rc) { - fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc, - mdbx_strerror(rc)); - goto txn_abort; - } - - int batch = 0; - prevk.iov_len = 0; - while (1) { - MDBX_val key; - rc = readline(&key, &kbuf); - if (rc) /* rc == EOF */ - break; - - MDBX_val data; - rc = readline(&data, &dbuf); - if (rc) { - fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", - prog, lineno); - goto txn_abort; - } - - int appflag = 0; - if (append) { - appflag = MDBX_APPEND; - if (dbi_flags & MDBX_DUPSORT) { - if (prevk.iov_len == key.iov_len && - memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0) - appflag = MDBX_APPEND | MDBX_APPENDDUP; - else - memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len); - } - } - rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag); - if (rc == MDBX_KEYEXIST && putflags) - continue; - if (rc == MDBX_BAD_VALSIZE && rescue) { - fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, - mdbx_strerror(rc)); - continue; - } - if (rc) { - fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc, - mdbx_strerror(rc)); - goto txn_abort; - } - batch++; - if (batch == 100) { - rc = mdbx_txn_commit(txn); - if (rc) { - fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog, - lineno, mdbx_strerror(rc)); - goto env_close; - } - rc = mdbx_txn_begin(env, NULL, 0, &txn); - if (rc) { - fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc, - mdbx_strerror(rc)); - goto env_close; - } - rc = mdbx_cursor_open(txn, dbi, &mc); - if (rc) { - fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc, - mdbx_strerror(rc)); - goto txn_abort; - } - batch = 0; - } - } - rc = mdbx_txn_commit(txn); - txn = NULL; - if (rc) { - fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog, lineno, - mdbx_strerror(rc)); - goto env_close; - } - mdbx_dbi_close(env, dbi); - subname = NULL; - - /* try read next header */ - if (!(mode & NOHDR)) - readhdr(); - } - -txn_abort: - mdbx_txn_abort(txn); -env_close: - mdbx_env_close(env); - - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/tools/mdbx_load/src/stdafx.cxx b/tools/mdbx_load/src/stdafx.cxx deleted file mode 100644 index 22c4818d32..0000000000 --- a/tools/mdbx_load/src/stdafx.cxx +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright (C) 2018-20 Miranda NG team (https://miranda-ng.org) - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation version 2 -of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -#include "stdafx.h" - -#include "..\..\build\appstub\appstub.cpp"
\ No newline at end of file diff --git a/tools/mdbx_load/src/stdafx.h b/tools/mdbx_load/src/stdafx.h deleted file mode 100644 index 518027200a..0000000000 --- a/tools/mdbx_load/src/stdafx.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright (C) 2018-20 Miranda NG team (https://miranda-ng.org) - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation version 2 -of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -#pragma once - -#include <stdio.h> -#include <stdint.h> -#include <windows.h> - -#define DECLARE_VERSION() \ - mdbx_version_info MDBX_version; \ - mdbx_build_info MDBX_build; \ - char* MDBX_sourcery_anchor; \ - HINSTANCE hDll = LoadLibraryA("libmdbx.mir"); \ - MDBX_version = *(mdbx_version_info *)GetProcAddress(hDll, "mdbx_version"); \ - MDBX_build = *(mdbx_build_info*)GetProcAddress(hDll, "mdbx_build"); \ - MDBX_sourcery_anchor = (char*)GetProcAddress(hDll, "mdbx_sourcery_MDBX_BUILD_SOURCERY"); |