diff options
| author | George Hazan <george.hazan@gmail.com> | 2015-04-02 17:28:07 +0000 |
|---|---|---|
| committer | George Hazan <george.hazan@gmail.com> | 2015-04-02 17:28:07 +0000 |
| commit | 2e511ab1b1ff3d78c695874e3b28ff4ce7680cc8 (patch) | |
| tree | 9c3588c82da7ad3e326f51d899800ad183f0d826 /plugins/Dbx_kyoto/src | |
| parent | 0f73f1572a03e5bae2664c1b2bb2cd18a1e33fca (diff) | |
kyotocabinet based db driver
first version that compiles
DO NOT USE IT, dragons live there
git-svn-id: http://svn.miranda-ng.org/main/trunk@12580 1316c22d-e87f-b044-9b9b-93d7a3e3ba9c
Diffstat (limited to 'plugins/Dbx_kyoto/src')
83 files changed, 86771 insertions, 0 deletions
diff --git a/plugins/Dbx_kyoto/src/commonheaders.h b/plugins/Dbx_kyoto/src/commonheaders.h new file mode 100644 index 0000000000..d8662967e9 --- /dev/null +++ b/plugins/Dbx_kyoto/src/commonheaders.h @@ -0,0 +1,73 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#define _CRT_SECURE_NO_WARNINGS
+#define _WIN32_WINNT 0x0501
+
+#pragma warning(disable:4509)
+
+#include <windows.h>
+#include <time.h>
+#include <process.h>
+#include <memory>
+
+#include <newpluginapi.h>
+#include <win2k.h>
+#include <m_system_cpp.h>
+#include <m_database.h>
+#include <m_langpack.h>
+#include <m_clist.h>
+#include <m_icolib.h>
+#include <m_options.h>
+#include <m_crypto.h>
+#include <m_metacontacts.h>
+#include <m_protocols.h>
+#include <m_netlib.h>
+
+#define OWN_CACHED_CONTACT
+#include <m_db_int.h>
+
+#include "dbintf.h"
+#include "resource.h"
+#include "version.h"
+
+extern HINSTANCE g_hInst;
+extern LIST<CDbxKV> g_Dbs;
+
+class cursor_ptr
+{
+ TreeDB::Cursor *m_cursor;
+
+public:
+ __forceinline cursor_ptr(TreeDB &_db)
+ {
+ m_cursor = _db.cursor();
+ }
+
+ __forceinline ~cursor_ptr()
+ {
+ delete m_cursor;
+ }
+
+ __forceinline TreeDB::Cursor* operator->() const { return m_cursor; }
+};
diff --git a/plugins/Dbx_kyoto/src/dbcontacts.cpp b/plugins/Dbx_kyoto/src/dbcontacts.cpp new file mode 100644 index 0000000000..c7b74bc038 --- /dev/null +++ b/plugins/Dbx_kyoto/src/dbcontacts.cpp @@ -0,0 +1,253 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+int CDbxKV::CheckProto(DBCachedContact *cc, const char *proto)
+{
+ if (cc->szProto == NULL) {
+ char protobuf[MAX_PATH] = { 0 };
+ DBVARIANT dbv;
+ dbv.type = DBVT_ASCIIZ;
+ dbv.pszVal = protobuf;
+ dbv.cchVal = sizeof(protobuf);
+ if (GetContactSettingStatic(cc->contactID, "Protocol", "p", &dbv) != 0 || (dbv.type != DBVT_ASCIIZ))
+ return 0;
+
+ cc->szProto = m_cache->GetCachedSetting(NULL, protobuf, 0, (int)strlen(protobuf));
+ }
+
+ return !strcmp(cc->szProto, proto);
+}
+
+STDMETHODIMP_(LONG) CDbxKV::GetContactCount(void)
+{
+ mir_cslock lck(m_csDbAccess);
+ return m_contactCount;
+}
+
+STDMETHODIMP_(LONG) CDbxKV::GetContactSize(void)
+{
+ return sizeof(DBCachedContact);
+}
+
+STDMETHODIMP_(MCONTACT) CDbxKV::FindFirstContact(const char *szProto)
+{
+ mir_cslock lck(m_csDbAccess);
+ DBCachedContact *cc = m_cache->GetFirstContact();
+ if (cc == NULL)
+ return NULL;
+
+ if (cc->contactID == 0)
+ if ((cc = m_cache->GetNextContact(0)) == NULL)
+ return NULL;
+
+ if (!szProto || CheckProto(cc, szProto))
+ return cc->contactID;
+
+ return FindNextContact(cc->contactID, szProto);
+}
+
+STDMETHODIMP_(MCONTACT) CDbxKV::FindNextContact(MCONTACT contactID, const char *szProto)
+{
+ mir_cslock lck(m_csDbAccess);
+ while (contactID) {
+ DBCachedContact *cc = m_cache->GetNextContact(contactID);
+ if (cc == NULL)
+ break;
+
+ if (!szProto || CheckProto(cc, szProto))
+ return cc->contactID;
+
+ contactID = cc->contactID;
+ }
+
+ return NULL;
+}
+
+STDMETHODIMP_(LONG) CDbxKV::DeleteContact(MCONTACT contactID)
+{
+ if (contactID == 0) // global contact cannot be removed
+ return 1;
+
+ // call notifier while outside mutex
+ NotifyEventHooks(hContactDeletedEvent, contactID, 0);
+
+ // delete
+ mir_cslock lck(m_csDbAccess);
+ m_dbContacts.remove((LPCSTR)&contactID, sizeof(MCONTACT));
+ return 0;
+}
+
+STDMETHODIMP_(MCONTACT) CDbxKV::AddContact()
+{
+ DWORD dwContactId;
+ {
+ mir_cslock lck(m_csDbAccess);
+ dwContactId = m_dwMaxContactId++;
+
+ DBCachedContact *cc = m_cache->AddContactToCache(dwContactId);
+ cc->dbc.dwSignature = DBCONTACT_SIGNATURE;
+ m_dbContacts.set((LPCSTR)&dwContactId, sizeof(MCONTACT), (LPCSTR)&cc->dbc, sizeof(cc->dbc));
+ }
+
+ NotifyEventHooks(hContactAddedEvent, dwContactId, 0);
+ return dwContactId;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::IsDbContact(MCONTACT contactID)
+{
+ DBCachedContact *cc = m_cache->GetCachedContact(contactID);
+ return (cc != NULL);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+// metacontacts support
+
+BOOL CDbxKV::MetaDetouchSub(DBCachedContact *cc, int nSub)
+{
+ CallService(MS_DB_MODULE_DELETE, cc->pSubs[nSub], (LPARAM)META_PROTO);
+ return 0;
+}
+
+BOOL CDbxKV::MetaSetDefault(DBCachedContact *cc)
+{
+ return db_set_dw(cc->contactID, META_PROTO, "Default", cc->nDefault);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+BOOL CDbxKV::MetaMergeHistory(DBCachedContact *ccMeta, DBCachedContact *ccSub)
+{
+ DBEventSortingKey keyVal = { ccSub->contactID, 0, 0 }, insVal = { ccMeta->contactID, 0, 0 };
+ cursor_ptr cursor(m_dbEventsSort);
+ cursor->jump((LPCSTR)&keyVal, sizeof(keyVal));
+
+ size_t key_size;
+ while (cursor->step()) {
+ DBEventSortingKey *pKey = (DBEventSortingKey*)cursor->get_key(&key_size);
+ if (pKey->dwContactId != ccSub->contactID) {
+ delete[] pKey;
+ break;
+ }
+
+ insVal.ts = pKey->ts;
+ insVal.dwEventId = pKey->dwEventId;
+ m_dbEventsSort.set((LPCSTR)&insVal, sizeof(insVal), "", 1);
+
+ ccMeta->dbc.dwEventCount++;
+ delete[] pKey;
+ }
+
+ // now update the number of events in a metacontact
+ m_dbContacts.set((LPCSTR)&ccMeta->contactID, sizeof(int), (LPCSTR)&ccMeta->dbc, sizeof(ccMeta->dbc));
+ return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+BOOL CDbxKV::MetaSplitHistory(DBCachedContact *ccMeta, DBCachedContact *ccSub)
+{
+ DBEventSortingKey keyVal = { ccSub->contactID, 0, 0 }, delVal = { ccMeta->contactID, 0, 0 };
+
+ cursor_ptr cursor(m_dbEventsSort);
+ cursor->jump((LPCSTR)&keyVal, sizeof(keyVal));
+
+ size_t key_size;
+ while (cursor->step()) {
+ DBEventSortingKey *pKey = (DBEventSortingKey*)cursor->get_key(&key_size);
+ if (pKey->dwContactId != ccSub->contactID) {
+ delete[] pKey;
+ break;
+ }
+
+ delVal.ts = pKey->ts;
+ delVal.dwEventId = pKey->dwEventId;
+ m_dbEventsSort.remove((LPCSTR)&delVal, sizeof(delVal));
+
+ ccMeta->dbc.dwEventCount--;
+ delete[] pKey;
+ };
+
+ // now update the number of events in a metacontact
+ m_dbContacts.set((LPCSTR)&ccMeta->contactID, sizeof(int), (LPCSTR)&ccMeta->dbc, sizeof(ccMeta->dbc));
+ return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+void DBCachedContact::Advance(DWORD id, DBEvent &dbe)
+{
+ dbc.dwEventCount++;
+
+ if (dbe.flags & (DBEF_READ | DBEF_SENT))
+ return;
+
+ if (dbe.timestamp < dbc.tsFirstUnread || dbc.tsFirstUnread == 0) {
+ dbc.tsFirstUnread = dbe.timestamp;
+ dbc.dwFirstUnread = id;
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+// initial cycle to fill the contacts' cache
+
+void CDbxKV::FillContacts()
+{
+ m_contactCount = 0;
+
+ size_t size;
+ cursor_ptr cursor(m_dbContacts);
+ for (cursor->jump(); cursor->step();) {
+ const char *pRec, *pKey = cursor->get(&size, &pRec, &size);
+ DBContact *dbc = (DBContact*)pRec;
+ if (dbc->dwSignature != DBCONTACT_SIGNATURE)
+ DatabaseCorruption(NULL);
+
+ DBCachedContact *cc = m_cache->AddContactToCache(*(DWORD*)pKey);
+ cc->dbc.dwSignature = DBCONTACT_SIGNATURE;
+ cc->dbc.dwEventCount = dbc->dwEventCount;
+ cc->dbc.dwFirstUnread = dbc->dwFirstUnread;
+ cc->dbc.tsFirstUnread = dbc->tsFirstUnread;
+
+ CheckProto(cc, "");
+
+ m_dwMaxContactId = cc->contactID+1;
+ m_contactCount++;
+
+ DBVARIANT dbv; dbv.type = DBVT_DWORD;
+ cc->nSubs = (0 != GetContactSetting(cc->contactID, META_PROTO, "NumContacts", &dbv)) ? -1 : dbv.dVal;
+ if (cc->nSubs != -1) {
+ cc->pSubs = (MCONTACT*)mir_alloc(cc->nSubs*sizeof(MCONTACT));
+ for (int i = 0; i < cc->nSubs; i++) {
+ char setting[100];
+ mir_snprintf(setting, SIZEOF(setting), "Handle%d", i);
+ cc->pSubs[i] = (0 != GetContactSetting(cc->contactID, META_PROTO, setting, &dbv)) ? NULL : dbv.dVal;
+ }
+ }
+ cc->nDefault = (0 != GetContactSetting(cc->contactID, META_PROTO, "Default", &dbv)) ? -1 : dbv.dVal;
+ cc->parentID = (0 != GetContactSetting(cc->contactID, META_PROTO, "ParentMeta", &dbv)) ? NULL : dbv.dVal;
+
+ delete[] pKey;
+ }
+}
diff --git a/plugins/Dbx_kyoto/src/dbcrypt.cpp b/plugins/Dbx_kyoto/src/dbcrypt.cpp new file mode 100644 index 0000000000..b79639fa61 --- /dev/null +++ b/plugins/Dbx_kyoto/src/dbcrypt.cpp @@ -0,0 +1,263 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+//VERY VERY VERY BASIC ENCRYPTION FUNCTION
+
+static void Encrypt(char *msg, BOOL up)
+{
+ int jump = (up) ? 5 : -5;
+ for (int i = 0; msg[i]; i++)
+ msg[i] = msg[i] + jump;
+}
+
+__forceinline void DecodeString(LPSTR buf)
+{
+ Encrypt(buf, FALSE);
+}
+
+struct VarDescr
+{
+ VarDescr(LPCSTR var, LPCSTR value) :
+ szVar(mir_strdup(var)),
+ szValue(mir_strdup(value))
+ {}
+
+ VarDescr(LPCSTR var, LPSTR value) :
+ szVar(mir_strdup(var)),
+ szValue(value)
+ {}
+
+ VarDescr(LPCSTR var, PBYTE value, int len) :
+ szVar(mir_strdup(var)),
+ szValue((char*)memcpy(mir_alloc(len), value, len)),
+ iLen(len)
+ {}
+
+ ptrA szVar, szValue;
+ int iLen;
+};
+
+struct SettingUgraderParam
+{
+ CDbxKV *db;
+ LPCSTR szModule;
+ MCONTACT contactID;
+ OBJLIST<VarDescr>* pList;
+};
+
+int sttSettingUgrader(const char *szSetting, LPARAM lParam)
+{
+ SettingUgraderParam *param = (SettingUgraderParam*)lParam;
+ if (param->db->IsSettingEncrypted(param->szModule, szSetting)) {
+ DBVARIANT dbv = { DBVT_UTF8 };
+ if (!param->db->GetContactSettingStr(param->contactID, param->szModule, szSetting, &dbv)) {
+ if (dbv.type == DBVT_UTF8) {
+ DecodeString(dbv.pszVal);
+ param->pList->insert(new VarDescr(szSetting, (LPCSTR)dbv.pszVal));
+ }
+ param->db->FreeVariant(&dbv);
+ }
+ }
+ return 0;
+}
+
+void sttContactEnum(MCONTACT contactID, const char *szModule, CDbxKV *db)
+{
+ OBJLIST<VarDescr> arSettings(1);
+ SettingUgraderParam param = { db, szModule, contactID, &arSettings };
+
+ DBCONTACTENUMSETTINGS dbces = { 0 };
+ dbces.pfnEnumProc = sttSettingUgrader;
+ dbces.szModule = szModule;
+ dbces.lParam = (LPARAM)¶m;
+ db->EnumContactSettings(NULL, &dbces);
+
+ for (int i = 0; i < arSettings.getCount(); i++) {
+ VarDescr &p = arSettings[i];
+
+ size_t len;
+ BYTE *pResult = db->m_crypto->encodeString(p.szValue, &len);
+ if (pResult != NULL) {
+ DBCONTACTWRITESETTING dbcws = { szModule, p.szVar };
+ dbcws.value.type = DBVT_ENCRYPTED;
+ dbcws.value.pbVal = pResult;
+ dbcws.value.cpbVal = (WORD)len;
+ db->WriteContactSetting(contactID, &dbcws);
+
+ mir_free(pResult);
+ }
+ }
+}
+
+int sttModuleEnum(const char *szModule, DWORD, LPARAM lParam)
+{
+ CDbxKV *db = (CDbxKV*)lParam;
+ sttContactEnum(NULL, szModule, db);
+
+ for (MCONTACT contactID = db->FindFirstContact(); contactID; contactID = db->FindNextContact(contactID))
+ sttContactEnum(contactID, szModule, db);
+
+ return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+int CDbxKV::InitCrypt()
+{
+ CRYPTO_PROVIDER *pProvider;
+ bool bMissingKey = false;
+
+ DBVARIANT dbv = { 0 };
+ dbv.type = DBVT_BLOB;
+ if (GetContactSetting(NULL, "CryptoEngine", "Provider", &dbv)) {
+ LBL_CreateProvider:
+ CRYPTO_PROVIDER **ppProvs;
+ int iNumProvs;
+ Crypto_EnumProviders(&iNumProvs, &ppProvs);
+ if (iNumProvs == 0)
+ return 1;
+
+ pProvider = ppProvs[0]; //!!!!!!!!!!!!!!!!!!
+
+ DBCONTACTWRITESETTING dbcws = { "CryptoEngine", "Provider" };
+ dbcws.value.type = DBVT_BLOB;
+ dbcws.value.pbVal = (PBYTE)pProvider->pszName;
+ dbcws.value.cpbVal = (int)strlen(pProvider->pszName) + 1;
+ WriteContactSetting(NULL, &dbcws);
+ }
+ else {
+ if (dbv.type != DBVT_BLOB) { // old version, clean it up
+ bMissingKey = true;
+ goto LBL_CreateProvider;
+ }
+
+ pProvider = Crypto_GetProvider(LPCSTR(dbv.pbVal));
+ FreeVariant(&dbv);
+ if (pProvider == NULL)
+ goto LBL_CreateProvider;
+ }
+
+ if ((m_crypto = pProvider->pFactory()) == NULL)
+ return 3;
+
+ dbv.type = DBVT_BLOB;
+ if (GetContactSetting(NULL, "CryptoEngine", "StoredKey", &dbv)) {
+ bMissingKey = true;
+
+ LBL_SetNewKey:
+ m_crypto->generateKey(); // unencrypted key
+ StoreKey();
+ }
+ else {
+ size_t iKeyLength = m_crypto->getKeyLength();
+ if (dbv.cpbVal != (WORD)iKeyLength)
+ goto LBL_SetNewKey;
+
+ if (!m_crypto->setKey(dbv.pbVal, iKeyLength))
+ if (!EnterPassword(dbv.pbVal, iKeyLength)) // password protected?
+ return 4;
+
+ FreeVariant(&dbv);
+ }
+
+ if (bMissingKey)
+ EnumModuleNames(sttModuleEnum, this);
+
+ dbv.type = DBVT_BYTE;
+ if (!GetContactSetting(NULL, "CryptoEngine", "DatabaseEncryption", &dbv))
+ m_bEncrypted = dbv.bVal != 0;
+
+ InitDialogs();
+ return 0;
+}
+
+void CDbxKV::StoreKey()
+{
+ size_t iKeyLength = m_crypto->getKeyLength();
+ BYTE *pKey = (BYTE*)_alloca(iKeyLength);
+ m_crypto->getKey(pKey, iKeyLength);
+
+ DBCONTACTWRITESETTING dbcws = { "CryptoEngine", "StoredKey" };
+ dbcws.value.type = DBVT_BLOB;
+ dbcws.value.cpbVal = (WORD)iKeyLength;
+ dbcws.value.pbVal = pKey;
+ WriteContactSetting(NULL, &dbcws);
+
+ SecureZeroMemory(pKey, iKeyLength);
+}
+
+void CDbxKV::SetPassword(LPCTSTR ptszPassword)
+{
+ if (ptszPassword == NULL || *ptszPassword == 0) {
+ m_bUsesPassword = false;
+ m_crypto->setPassword(NULL);
+ }
+ else {
+ m_bUsesPassword = true;
+ m_crypto->setPassword(ptrA(mir_utf8encodeT(ptszPassword)));
+ }
+ UpdateMenuItem();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+void CDbxKV::ToggleEncryption()
+{
+ HANDLE hSave1 = hSettingChangeEvent; hSettingChangeEvent = NULL;
+ HANDLE hSave2 = hEventAddedEvent; hEventAddedEvent = NULL;
+ HANDLE hSave3 = hEventDeletedEvent; hEventDeletedEvent = NULL;
+ HANDLE hSave4 = hEventFilterAddedEvent; hEventFilterAddedEvent = NULL;
+
+ mir_cslock lck(m_csDbAccess);
+ ToggleSettingsEncryption(NULL);
+ ToggleEventsEncryption(NULL);
+
+ for (MCONTACT contactID = FindFirstContact(); contactID; contactID = FindNextContact(contactID)) {
+ ToggleSettingsEncryption(contactID);
+ ToggleEventsEncryption(contactID);
+ }
+
+ m_bEncrypted = !m_bEncrypted;
+
+ DBCONTACTWRITESETTING dbcws = { "CryptoEngine", "DatabaseEncryption" };
+ dbcws.value.type = DBVT_BYTE;
+ dbcws.value.bVal = m_bEncrypted;
+ WriteContactSetting(NULL, &dbcws);
+
+ hSettingChangeEvent = hSave1;
+ hEventAddedEvent = hSave2;
+ hEventDeletedEvent = hSave3;
+ hEventFilterAddedEvent = hSave4;
+}
+
+void CDbxKV::ToggleSettingsEncryption(MCONTACT contactID)
+{
+}
+
+void CDbxKV::ToggleEventsEncryption(MCONTACT contactID)
+{
+}
diff --git a/plugins/Dbx_kyoto/src/dbevents.cpp b/plugins/Dbx_kyoto/src/dbevents.cpp new file mode 100644 index 0000000000..ec4c562751 --- /dev/null +++ b/plugins/Dbx_kyoto/src/dbevents.cpp @@ -0,0 +1,365 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+STDMETHODIMP_(LONG) CDbxKV::GetEventCount(MCONTACT contactID)
+{
+ DBCachedContact *cc = m_cache->GetCachedContact(contactID);
+ return (cc == NULL) ? 0 : cc->dbc.dwEventCount;
+}
+
+STDMETHODIMP_(MEVENT) CDbxKV::AddEvent(MCONTACT contactID, DBEVENTINFO *dbei)
+{
+ if (dbei == NULL || dbei->cbSize != sizeof(DBEVENTINFO)) return 0;
+ if (dbei->timestamp == 0) return 0;
+
+ DBEvent dbe;
+ dbe.dwSignature = DBEVENT_SIGNATURE;
+ dbe.contactID = contactID; // store native or subcontact's id
+ dbe.ofsModuleName = GetModuleNameOfs(dbei->szModule);
+ dbe.timestamp = dbei->timestamp;
+ dbe.flags = dbei->flags;
+ dbe.wEventType = dbei->eventType;
+ dbe.cbBlob = dbei->cbBlob;
+ BYTE *pBlob = dbei->pBlob;
+
+ MCONTACT contactNotifyID = contactID;
+ DBCachedContact *cc, *ccSub = NULL;
+ if ((cc = m_cache->GetCachedContact(contactID)) == NULL)
+ return 0;
+
+ if (cc->IsSub()) {
+ ccSub = cc;
+ if ((cc = m_cache->GetCachedContact(cc->parentID)) == NULL)
+ return 0;
+
+ // set default sub to the event's source
+ if (!(dbei->flags & DBEF_SENT))
+ db_mc_setDefault(cc->contactID, contactID, false);
+ contactID = cc->contactID; // and add an event to a metahistory
+ if (db_mc_isEnabled())
+ contactNotifyID = contactID;
+ }
+
+ if (m_safetyMode)
+ if (NotifyEventHooks(hEventFilterAddedEvent, contactNotifyID, (LPARAM)dbei))
+ return NULL;
+
+ mir_ptr<BYTE> pCryptBlob;
+ if (m_bEncrypted) {
+ size_t len;
+ BYTE *pResult = m_crypto->encodeBuffer(pBlob, dbe.cbBlob, &len);
+ if (pResult != NULL) {
+ pCryptBlob = pBlob = pResult;
+ dbe.cbBlob = (DWORD)len;
+ dbe.flags |= DBEF_ENCRYPTED;
+ }
+ }
+
+ DWORD dwEventId = ++m_dwMaxEventId;
+
+ BYTE *pDest = (BYTE*)_alloca(sizeof(DBEvent) + dbe.cbBlob);
+ memcpy(pDest, &dbe, sizeof(DBEvent));
+ memcpy(pDest + sizeof(DBEvent), pBlob, dbe.cbBlob);
+ m_dbEvents.set((LPCSTR)&dwEventId, sizeof(int), (LPCSTR)pDest, sizeof(DBEvent) + dbe.cbBlob);
+
+ // add a sorting key
+ DBEventSortingKey key2 = { contactID, dbe.timestamp, dwEventId };
+ m_dbEventsSort.set((LPCSTR)&key2, sizeof(key2), "", 1);
+
+ cc->Advance(dwEventId, dbe);
+ m_dbContacts.set((LPCSTR)&contactID, sizeof(int), (LPCSTR)&cc->dbc, sizeof(DBContact));
+
+ // insert an event into a sub's history too
+ if (ccSub != NULL) {
+ key2.dwContactId = ccSub->contactID;
+ m_dbEventsSort.set((LPCSTR)&key2, sizeof(key2), "", 1);
+
+ ccSub->Advance(dwEventId, dbe);
+ m_dbContacts.set((LPCSTR)&ccSub->contactID, sizeof(int), (LPCSTR)&ccSub->dbc, sizeof(DBContact));
+ }
+
+ // Notify only in safe mode or on really new events
+ if (m_safetyMode)
+ NotifyEventHooks(hEventAddedEvent, contactNotifyID, dwEventId);
+
+ return dwEventId;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::DeleteEvent(MCONTACT contactID, MEVENT hDbEvent)
+{
+ if (hDbEvent == 0) return INVALID_CONTACT_ID;
+
+ DBCachedContact *cc = m_cache->GetCachedContact(contactID);
+ if (cc == NULL || cc->dbc.dwEventCount == 0)
+ return 1;
+
+ DBEvent dbe;
+ if (-1 == m_dbEvents.get((LPCSTR)&hDbEvent, sizeof(MEVENT), (LPSTR)&dbe, sizeof(dbe)))
+ return 1;
+
+ DWORD dwSavedContact = dbe.contactID;
+ DBEventSortingKey key2 = { contactID, dbe.timestamp, hDbEvent };
+ m_dbEvents.remove((LPCSTR)&hDbEvent, sizeof(MEVENT));
+
+ // remove a sorting key
+ m_dbEventsSort.remove((LPCSTR)&key2, sizeof(key2));
+
+ // remove a sub's history entry too
+ if (contactID != dwSavedContact) {
+ key2.dwContactId = dwSavedContact;
+ m_dbEventsSort.remove((LPCSTR)&key2, sizeof(key2));
+ }
+
+ // update a contact
+ cc->dbc.dwEventCount--;
+ if (cc->dbc.dwFirstUnread == hDbEvent)
+ FindNextUnread(cc, key2);
+
+ // call notifier while outside mutex
+ NotifyEventHooks(hEventDeletedEvent, contactID, hDbEvent);
+ return 0;
+}
+
+STDMETHODIMP_(LONG) CDbxKV::GetBlobSize(MEVENT hDbEvent)
+{
+ DBEvent dbe;
+ if (-1 == m_dbEvents.get((LPCSTR)&hDbEvent, sizeof(MEVENT), (LPSTR)&dbe, sizeof(dbe)))
+ return -1;
+
+ return (dbe.dwSignature == DBEVENT_SIGNATURE) ? dbe.cbBlob : 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::GetEvent(MEVENT hDbEvent, DBEVENTINFO *dbei)
+{
+ if (hDbEvent == 0 || dbei == NULL || dbei->cbSize != sizeof(DBEVENTINFO)) return 1;
+ if (dbei->cbBlob > 0 && dbei->pBlob == NULL) {
+ dbei->cbBlob = 0;
+ return 1;
+ }
+
+ char rec[65536];
+ if (-1 == m_dbEvents.get((LPCSTR)&hDbEvent, sizeof(MEVENT), rec, sizeof(rec)))
+ return 1;
+
+ DBEvent *dbe = (DBEvent*)rec;
+ if (dbe->dwSignature != DBEVENT_SIGNATURE)
+ return 1;
+
+ dbei->szModule = GetModuleNameByOfs(dbe->ofsModuleName);
+ dbei->timestamp = dbe->timestamp;
+ dbei->flags = dbe->flags;
+ dbei->eventType = dbe->wEventType;
+ int bytesToCopy = (dbei->cbBlob < dbe->cbBlob) ? dbei->cbBlob : dbe->cbBlob;
+ dbei->cbBlob = dbe->cbBlob;
+ if (bytesToCopy && dbei->pBlob) {
+ BYTE *pSrc = (BYTE*)rec + sizeof(DBEvent);
+ if (dbe->flags & DBEF_ENCRYPTED) {
+ dbei->flags &= ~DBEF_ENCRYPTED;
+ size_t len;
+ BYTE* pBlob = (BYTE*)m_crypto->decodeBuffer(pSrc, dbe->cbBlob, &len);
+ if (pBlob == NULL)
+ return 1;
+
+ memcpy(dbei->pBlob, pBlob, bytesToCopy);
+ if (bytesToCopy > (int)len)
+ memset(dbei->pBlob + len, 0, bytesToCopy - len);
+ mir_free(pBlob);
+ }
+ else memcpy(dbei->pBlob, pSrc, bytesToCopy);
+ }
+ return 0;
+}
+
+void CDbxKV::FindNextUnread(DBCachedContact *cc, DBEventSortingKey &key2)
+{
+ key2.dwEventId++;
+
+ cursor_ptr cursor(m_dbEventsSort);
+ cursor->jump((LPCSTR)&key2, sizeof(key2));
+ while(cursor->step()) {
+ size_t size;
+ const char *pRec;
+ delete[] cursor->get(&size, &pRec, &size);
+ DBEvent *dbe = (DBEvent*)pRec;
+ if (!dbe->markedRead()) {
+ cc->dbc.dwFirstUnread = key2.dwEventId;
+ cc->dbc.tsFirstUnread = key2.ts;
+ return;
+ }
+ }
+
+ cc->dbc.dwFirstUnread = cc->dbc.tsFirstUnread = 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::MarkEventRead(MCONTACT contactID, MEVENT hDbEvent)
+{
+ if (hDbEvent == 0) return -1;
+
+ DBCachedContact *cc = m_cache->GetCachedContact(contactID);
+ if (cc == NULL)
+ return -1;
+
+ char rec[65536];
+ int32_t recLen = m_dbEvents.get((LPCSTR)&hDbEvent, sizeof(MEVENT), rec, sizeof(rec));
+ if (recLen == -1)
+ return -1;
+
+ DBEvent *dbe = (DBEvent*)rec;
+ if (dbe->dwSignature != DBEVENT_SIGNATURE)
+ return -1;
+
+ if (dbe->markedRead())
+ return dbe->flags;
+
+ DBEventSortingKey key2 = { contactID, dbe->timestamp, hDbEvent };
+
+ dbe->flags |= DBEF_READ;
+ m_dbEvents.set((LPCSTR)&hDbEvent, sizeof(MEVENT), rec, recLen);
+
+ FindNextUnread(cc, key2);
+ m_dbContacts.set((LPCSTR)&contactID, sizeof(int), (LPCSTR)&cc->dbc, sizeof(cc->dbc));
+
+ NotifyEventHooks(hEventMarkedRead, contactID, (LPARAM)hDbEvent);
+ return dbe->flags;
+}
+
+STDMETHODIMP_(MCONTACT) CDbxKV::GetEventContact(MEVENT hDbEvent)
+{
+ if (hDbEvent == 0) return INVALID_CONTACT_ID;
+
+ char rec[65536];
+ if (-1 == m_dbEvents.get((LPCSTR)&hDbEvent, sizeof(MEVENT), rec, sizeof(rec)))
+ return 1;
+
+ DBEvent *dbe = (DBEvent*)rec;
+ return (dbe->dwSignature == DBEVENT_SIGNATURE) ? dbe->contactID : INVALID_CONTACT_ID;
+}
+
+STDMETHODIMP_(MEVENT) CDbxKV::FindFirstEvent(MCONTACT contactID)
+{
+ DBEventSortingKey keyVal = { contactID, 0, 0 };
+ cursor_ptr cursor(m_dbEventsSort);
+ cursor->jump((LPCSTR)&keyVal, sizeof(keyVal));
+
+ size_t size;
+ DBEventSortingKey *pKey = (DBEventSortingKey*)cursor->get_key(&size);
+ if (pKey == NULL)
+ return m_evLast = 0;
+
+ m_tsLast = pKey->ts;
+ m_evLast = (pKey->dwContactId == contactID) ? pKey->dwEventId : 0;
+ delete[] pKey;
+ return m_evLast;
+}
+
+STDMETHODIMP_(MEVENT) CDbxKV::FindFirstUnreadEvent(MCONTACT contactID)
+{
+ DBCachedContact *cc = m_cache->GetCachedContact(contactID);
+ return (cc == NULL) ? 0 : cc->dbc.dwFirstUnread;
+}
+
+STDMETHODIMP_(MEVENT) CDbxKV::FindLastEvent(MCONTACT contactID)
+{
+ DBEventSortingKey keyVal = { contactID, 0xFFFFFFFF, 0xFFFFFFFF };
+ cursor_ptr cursor(m_dbEventsSort);
+ cursor->jump_back((LPCSTR)&keyVal, sizeof(keyVal));
+
+ size_t size;
+ DBEventSortingKey *pKey = (DBEventSortingKey*)cursor->get_key(&size);
+ if (pKey == NULL)
+ return m_evLast = 0;
+
+ m_tsLast = pKey->ts;
+ m_evLast = (pKey->dwContactId == contactID) ? pKey->dwEventId : 0;
+ delete[] pKey;
+ return m_evLast;
+}
+
+STDMETHODIMP_(MEVENT) CDbxKV::FindNextEvent(MCONTACT contactID, MEVENT hDbEvent)
+{
+ if (hDbEvent == 0) return m_evLast = 0;
+
+ DWORD ts;
+
+ if (m_evLast != hDbEvent) {
+ DBEvent dbe;
+ if (-1 == m_dbEvents.get((LPCSTR)&hDbEvent, sizeof(MEVENT), (LPSTR)&dbe, sizeof(dbe)))
+ return 0;
+ m_tsLast = ts = dbe.timestamp;
+ }
+ else ts = m_tsLast;
+
+ DBEventSortingKey keyVal = { contactID, ts, hDbEvent+1 };
+ cursor_ptr cursor(m_dbEventsSort);
+ cursor->jump((LPCSTR)&keyVal, sizeof(keyVal));
+
+ size_t size;
+ DBEventSortingKey *pKey = (DBEventSortingKey*)cursor->get_key(&size);
+ if (pKey == NULL)
+ return m_evLast = 0;
+
+ m_tsLast = pKey->ts;
+ m_evLast = (pKey->dwContactId == contactID) ? pKey->dwEventId : 0;
+ delete[] pKey;
+ return m_evLast;
+}
+
+STDMETHODIMP_(MEVENT) CDbxKV::FindPrevEvent(MCONTACT contactID, MEVENT hDbEvent)
+{
+ if (hDbEvent == 0) return m_evLast = 0;
+
+ DWORD ts;
+
+ if (m_evLast != hDbEvent) {
+ DBEvent dbe;
+ if (-1 == m_dbEvents.get((LPCSTR)&hDbEvent, sizeof(MEVENT), (LPSTR)&dbe, sizeof(dbe)))
+ return 0;
+ m_tsLast = ts = dbe.timestamp;
+ }
+ else ts = m_tsLast;
+
+ DBEventSortingKey keyVal = { contactID, ts, hDbEvent-1 };
+ cursor_ptr cursor(m_dbEventsSort);
+ cursor->jump_back((LPCSTR)&keyVal, sizeof(keyVal));
+
+ size_t size;
+ DBEventSortingKey *pKey = (DBEventSortingKey*)cursor->get_key(&size);
+ if (pKey == NULL)
+ return m_evLast = 0;
+
+ m_tsLast = pKey->ts;
+ m_evLast = (pKey->dwContactId == contactID) ? pKey->dwEventId : 0;
+ delete[] pKey;
+ return m_evLast;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+// low-level history cleaner
+
+int CDbxKV::WipeContactHistory(DBContact*)
+{
+ // drop subContact's history if any
+ return 0;
+}
diff --git a/plugins/Dbx_kyoto/src/dbintf.cpp b/plugins/Dbx_kyoto/src/dbintf.cpp new file mode 100644 index 0000000000..cbed2eb648 --- /dev/null +++ b/plugins/Dbx_kyoto/src/dbintf.cpp @@ -0,0 +1,249 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+static int ModCompare(const ModuleName *mn1, const ModuleName *mn2)
+{
+ return strcmp(mn1->name, mn2->name);
+}
+
+static int OfsCompare(const ModuleName *mn1, const ModuleName *mn2)
+{
+ return (mn1->ofs - mn2->ofs);
+}
+
+static int stringCompare2(const char *p1, const char *p2)
+{
+ return strcmp(p1, p2);
+}
+
+CDbxKV::CDbxKV(const TCHAR *tszFileName, int iMode) :
+ m_safetyMode(true),
+ m_bReadOnly((iMode & DBMODE_READONLY) != 0),
+ m_bShared((iMode & DBMODE_SHARED) != 0),
+ m_dwMaxContactId(1),
+ m_lMods(50, ModCompare),
+ m_lOfs(50, OfsCompare),
+ m_lResidentSettings(50, stringCompare2)
+{
+ m_tszProfileName = mir_tstrdup(tszFileName);
+ InitDbInstance(this);
+
+ m_codePage = CallService(MS_LANGPACK_GETCODEPAGE, 0, 0);
+ m_hModHeap = HeapCreate(0, 0, 0);
+}
+
+CDbxKV::~CDbxKV()
+{
+ // destroy modules
+ HeapDestroy(m_hModHeap);
+
+ // automatically closes all tables
+ DestroyServiceFunction(hService);
+ UnhookEvent(hHook);
+
+ if (m_crypto)
+ m_crypto->destroy();
+
+ DestroyHookableEvent(hContactDeletedEvent);
+ DestroyHookableEvent(hContactAddedEvent);
+ DestroyHookableEvent(hSettingChangeEvent);
+ DestroyHookableEvent(hEventMarkedRead);
+
+ DestroyHookableEvent(hEventAddedEvent);
+ DestroyHookableEvent(hEventDeletedEvent);
+ DestroyHookableEvent(hEventFilterAddedEvent);
+
+ DestroyDbInstance(this);
+ mir_free(m_tszProfileName);
+}
+
+int CDbxKV::Load(bool bSkipInit)
+{
+ if (!bSkipInit) {
+ int iFlags = TreeDB::OREADER | TreeDB::ONOREPAIR;
+ if (!m_bReadOnly)
+ iFlags |= TreeDB::OWRITER;
+
+ std::string szFilename((char*)_T2A(m_tszProfileName));
+ if (!m_dbGlobal.open(szFilename, iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbContacts.open(szFilename + ".cnt", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbModules.open(szFilename + ".mod", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbEvents.open(szFilename + ".evt", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbEventsSort.open(szFilename + ".evs", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbSettings.open(szFilename + ".set", iFlags)) return EGROKPRF_DAMAGED;
+
+ DWORD keyVal = 1;
+ if (-1 != m_dbGlobal.get((LPCSTR)&keyVal, sizeof(keyVal), (LPSTR)&m_header, sizeof(m_header))) {
+ if (m_header.dwSignature != DBHEADER_SIGNATURE)
+ DatabaseCorruption(NULL);
+ }
+ else {
+ m_header.dwSignature = DBHEADER_SIGNATURE;
+ m_header.dwVersion = 1;
+ m_dbGlobal.set((LPCSTR)&keyVal, sizeof(keyVal), (LPCSTR)&m_header, sizeof(m_header));
+
+ keyVal = 0;
+ DBContact dbc = { DBCONTACT_SIGNATURE, 0, 0, 0 };
+ m_dbContacts.set((LPCSTR)&keyVal, sizeof(keyVal), (LPCSTR)&dbc, sizeof(dbc));
+ }
+
+ if (InitModuleNames()) return EGROKPRF_CANTREAD;
+ if (InitCrypt()) return EGROKPRF_CANTREAD;
+
+ // everything is ok, go on
+ if (!m_bReadOnly) {
+ // we don't need events in the service mode
+ if (ServiceExists(MS_DB_SETSAFETYMODE)) {
+ hContactDeletedEvent = CreateHookableEvent(ME_DB_CONTACT_DELETED);
+ hContactAddedEvent = CreateHookableEvent(ME_DB_CONTACT_ADDED);
+ hSettingChangeEvent = CreateHookableEvent(ME_DB_CONTACT_SETTINGCHANGED);
+ hEventMarkedRead = CreateHookableEvent(ME_DB_EVENT_MARKED_READ);
+
+ hEventAddedEvent = CreateHookableEvent(ME_DB_EVENT_ADDED);
+ hEventDeletedEvent = CreateHookableEvent(ME_DB_EVENT_DELETED);
+ hEventFilterAddedEvent = CreateHookableEvent(ME_DB_EVENT_FILTER_ADD);
+ }
+ }
+
+ FillContacts();
+ }
+
+ return ERROR_SUCCESS;
+}
+
+int CDbxKV::Create(void)
+{
+ int iFlags = TreeDB::OREADER | TreeDB::OCREATE;
+ if (!m_bReadOnly)
+ iFlags |= TreeDB::OWRITER;
+
+ std::string szFilename((char*)_T2A(m_tszProfileName));
+ if (!m_dbGlobal.open(szFilename, iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbContacts.open(szFilename + ".cnt", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbModules.open(szFilename + ".mod", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbEvents.open(szFilename + ".evt", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbEventsSort.open(szFilename + ".evs", iFlags)) return EGROKPRF_DAMAGED;
+ if (!m_dbSettings.open(szFilename + ".set", iFlags)) return EGROKPRF_DAMAGED;
+ return 0;
+}
+
+int CDbxKV::Check(void)
+{
+ HANDLE hFile = CreateFile(m_tszProfileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
+ if (hFile == INVALID_HANDLE_VALUE)
+ return EGROKPRF_CANTREAD;
+
+ DWORD dummy = 0;
+ char buf[32];
+ if (!ReadFile(hFile, buf, sizeof(buf), &dummy, NULL)) {
+ CloseHandle(hFile);
+ return EGROKPRF_CANTREAD;
+ }
+
+ CloseHandle(hFile);
+ return (memcmp(buf, "\x4B\x43\x0A\x00", 4)) ? EGROKPRF_UNKHEADER : 0;
+}
+
+int CDbxKV::PrepareCheck(int*)
+{
+ InitModuleNames();
+ return InitCrypt();
+}
+
+STDMETHODIMP_(void) CDbxKV::SetCacheSafetyMode(BOOL bIsSet)
+{
+ mir_cslock lck(m_csDbAccess);
+ m_safetyMode = bIsSet != 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+static DWORD DatabaseCorrupted = 0;
+static const TCHAR *msg = NULL;
+static DWORD dwErr = 0;
+static TCHAR tszPanic[] = LPGENT("Miranda has detected corruption in your database. This corruption may be fixed by DbChecker plugin. Please download it from http://miranda-ng.org/p/DbChecker/. Miranda will now shut down.");
+
+void __cdecl dbpanic(void *)
+{
+ if (msg) {
+ if (dwErr == ERROR_DISK_FULL)
+ msg = TranslateT("Disk is full. Miranda will now shut down.");
+
+ TCHAR err[256];
+ mir_sntprintf(err, SIZEOF(err), msg, TranslateT("Database failure. Miranda will now shut down."), dwErr);
+
+ MessageBox(0, err, TranslateT("Database Error"), MB_SETFOREGROUND | MB_TOPMOST | MB_APPLMODAL | MB_ICONWARNING | MB_OK);
+ }
+ else MessageBox(0, TranslateTS(tszPanic), TranslateT("Database Panic"), MB_SETFOREGROUND | MB_TOPMOST | MB_APPLMODAL | MB_ICONWARNING | MB_OK);
+ TerminateProcess(GetCurrentProcess(), 255);
+}
+
+void CDbxKV::DatabaseCorruption(const TCHAR *text)
+{
+ int kill = 0;
+
+ mir_cslockfull lck(m_csDbAccess);
+ if (DatabaseCorrupted == 0) {
+ DatabaseCorrupted++;
+ kill++;
+ msg = text;
+ dwErr = GetLastError();
+ }
+ else {
+ /* db is already corrupted, someone else is dealing with it, wait here
+ so that we don't do any more damage */
+ Sleep(INFINITE);
+ return;
+ }
+ lck.unlock();
+
+ if (kill) {
+ _beginthread(dbpanic, 0, NULL);
+ Sleep(INFINITE);
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// MIDatabaseChecker
+
+typedef int (CDbxKV::*CheckWorker)(int);
+
+int CDbxKV::Start(DBCHeckCallback *callback)
+{
+ cb = callback;
+ return ERROR_SUCCESS;
+}
+
+int CDbxKV::CheckDb(int, int)
+{
+ return ERROR_OUT_OF_PAPER;
+
+ // return (this->*Workers[phase])(firstTime);
+}
+
+void CDbxKV::Destroy()
+{
+ delete this;
+}
diff --git a/plugins/Dbx_kyoto/src/dbintf.h b/plugins/Dbx_kyoto/src/dbintf.h new file mode 100644 index 0000000000..cfde9f33b4 --- /dev/null +++ b/plugins/Dbx_kyoto/src/dbintf.h @@ -0,0 +1,304 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include <kchashdb.h> +using namespace kyotocabinet;
+
+/* tree diagram
+
+DBHeader
+|-->end of file (plain offset)
+|-->first contact (DBContact)
+| |-->next contact (DBContact)
+| | \--> ...
+| |-->first settings (DBContactSettings)
+| | |-->next settings (DBContactSettings)
+| | | \--> ...
+| | \-->module name (DBModuleName)
+| \-->first/last/firstunread event
+|-->user contact (DBContact)
+| |-->next contact = NULL
+| |-->first settings as above
+| \-->first/last/firstunread event as above
+\-->first module name (DBModuleName)
+\-->next module name (DBModuleName)
+\--> ...
+*/
+
+#define DBMODE_SHARED 0x0001
+#define DBMODE_READONLY 0x0002
+
+#define DBVT_ENCRYPTED 250
+#define DBVT_UNENCRYPTED 251
+
+#define MARKED_READ (DBEF_READ | DBEF_SENT)
+
+struct ModuleName
+{
+ char *name;
+ DWORD ofs;
+};
+
+#include <pshpack1.h>
+
+#define DBHEADER_SIGNATURE 0x40DECADEu
+struct DBHeader
+{
+ DWORD dwSignature;
+ DWORD dwVersion; // database format version
+};
+
+#define DBCONTACT_SIGNATURE 0x43DECADEu
+struct DBContact
+{
+ DWORD dwSignature;
+ DWORD dwEventCount; // number of events in the chain for this contact
+ DWORD tsFirstUnread;
+ DWORD dwFirstUnread;
+};
+
+#define DBMODULENAME_SIGNATURE 0x4DDECADEu
+struct DBModuleName
+{
+ DWORD dwSignature;
+ BYTE cbName; // number of characters in this module name
+ char name[1]; // name, no nul terminator
+};
+
+#define DBEVENT_SIGNATURE 0x45DECADEu
+struct DBEvent
+{
+ DWORD dwSignature;
+ MCONTACT contactID; // a contact this event belongs to
+ DWORD ofsModuleName; // offset to a DBModuleName struct of the name of
+ DWORD timestamp; // seconds since 00:00:00 01/01/1970
+ DWORD flags; // see m_database.h, db/event/add
+ WORD wEventType; // module-defined event type
+ WORD cbBlob; // number of bytes in the blob
+
+ bool __forceinline markedRead() const
+ {
+ return (flags & MARKED_READ) != 0;
+ }
+};
+
+#include <poppack.h>
+
+struct DBEventSortingKey
+{
+ DWORD dwContactId, ts, dwEventId;
+};
+
+struct DBSettingSortingKey
+{
+ DWORD dwContactID;
+ DWORD dwOfsModule;
+ char szSettingName[100];
+};
+
+struct DBCachedContact : public DBCachedContactBase
+{
+ void Advance(DWORD id, DBEvent &dbe);
+
+ DBContact dbc;
+};
+
+struct CDbxKV : public MIDatabase, public MIDatabaseChecker, public MZeroedObject
+{
+ CDbxKV(const TCHAR *tszFileName, int mode);
+ ~CDbxKV();
+
+ int Load(bool bSkipInit);
+ int Create(void);
+ int Check(void);
+
+ void DatabaseCorruption(const TCHAR *ptszText);
+
+ void ToggleEncryption(void);
+ void StoreKey(void);
+ void SetPassword(const TCHAR *ptszPassword);
+ void UpdateMenuItem(void);
+
+ int PrepareCheck(int*);
+
+ __forceinline LPSTR GetMenuTitle() const { return m_bUsesPassword ? LPGEN("Change/remove password") : LPGEN("Set password"); }
+
+ __forceinline bool isEncrypted() const { return m_bEncrypted; }
+ __forceinline bool usesPassword() const { return m_bUsesPassword; }
+
+public:
+ STDMETHODIMP_(void) SetCacheSafetyMode(BOOL);
+
+ STDMETHODIMP_(LONG) GetContactCount(void);
+ STDMETHODIMP_(MCONTACT) FindFirstContact(const char *szProto = NULL);
+ STDMETHODIMP_(MCONTACT) FindNextContact(MCONTACT contactID, const char *szProto = NULL);
+ STDMETHODIMP_(LONG) DeleteContact(MCONTACT contactID);
+ STDMETHODIMP_(MCONTACT) AddContact(void);
+ STDMETHODIMP_(BOOL) IsDbContact(MCONTACT contactID);
+ STDMETHODIMP_(LONG) GetContactSize(void);
+
+ STDMETHODIMP_(LONG) GetEventCount(MCONTACT contactID);
+ STDMETHODIMP_(MEVENT) AddEvent(MCONTACT contactID, DBEVENTINFO *dbe);
+ STDMETHODIMP_(BOOL) DeleteEvent(MCONTACT contactID, MEVENT hDbEvent);
+ STDMETHODIMP_(LONG) GetBlobSize(MEVENT hDbEvent);
+ STDMETHODIMP_(BOOL) GetEvent(MEVENT hDbEvent, DBEVENTINFO *dbe);
+ STDMETHODIMP_(BOOL) MarkEventRead(MCONTACT contactID, MEVENT hDbEvent);
+ STDMETHODIMP_(MCONTACT) GetEventContact(MEVENT hDbEvent);
+ STDMETHODIMP_(MEVENT) FindFirstEvent(MCONTACT contactID);
+ STDMETHODIMP_(MEVENT) FindFirstUnreadEvent(MCONTACT contactID);
+ STDMETHODIMP_(MEVENT) FindLastEvent(MCONTACT contactID);
+ STDMETHODIMP_(MEVENT) FindNextEvent(MCONTACT contactID, MEVENT hDbEvent);
+ STDMETHODIMP_(MEVENT) FindPrevEvent(MCONTACT contactID, MEVENT hDbEvent);
+
+ STDMETHODIMP_(BOOL) EnumModuleNames(DBMODULEENUMPROC pFunc, void *pParam);
+
+ STDMETHODIMP_(BOOL) GetContactSetting(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv);
+ STDMETHODIMP_(BOOL) GetContactSettingStr(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv);
+ STDMETHODIMP_(BOOL) GetContactSettingStatic(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv);
+ STDMETHODIMP_(BOOL) FreeVariant(DBVARIANT *dbv);
+ STDMETHODIMP_(BOOL) WriteContactSetting(MCONTACT contactID, DBCONTACTWRITESETTING *dbcws);
+ STDMETHODIMP_(BOOL) DeleteContactSetting(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting);
+ STDMETHODIMP_(BOOL) EnumContactSettings(MCONTACT contactID, DBCONTACTENUMSETTINGS *dbces);
+ STDMETHODIMP_(BOOL) SetSettingResident(BOOL bIsResident, const char *pszSettingName);
+ STDMETHODIMP_(BOOL) EnumResidentSettings(DBMODULEENUMPROC pFunc, void *pParam);
+ STDMETHODIMP_(BOOL) IsSettingEncrypted(LPCSTR szModule, LPCSTR szSetting);
+
+ STDMETHODIMP_(BOOL) MetaDetouchSub(DBCachedContact *cc, int nSub);
+ STDMETHODIMP_(BOOL) MetaSetDefault(DBCachedContact *cc);
+ STDMETHODIMP_(BOOL) MetaMergeHistory(DBCachedContact *ccMeta, DBCachedContact *ccSub);
+ STDMETHODIMP_(BOOL) MetaSplitHistory(DBCachedContact *ccMeta, DBCachedContact *ccSub);
+
+protected:
+ STDMETHODIMP_(BOOL) Start(DBCHeckCallback *callback);
+ STDMETHODIMP_(BOOL) CheckDb(int phase, int firstTime);
+ STDMETHODIMP_(VOID) Destroy();
+
+protected:
+ void InvalidateSettingsGroupOfsCacheEntry(DWORD) {}
+ int WorkInitialCheckHeaders(void);
+
+ void FillContacts(void);
+
+public: // Check functions
+ int WorkInitialChecks(int);
+ int WorkModuleChain(int);
+ int WorkUser(int);
+ int WorkContactChain(int);
+ int WorkAggressive(int);
+ int WorkFinalTasks(int);
+
+protected:
+ TCHAR* m_tszProfileName;
+ bool m_safetyMode, m_bReadOnly, m_bShared, m_bEncrypted, m_bUsesPassword;
+
+ ////////////////////////////////////////////////////////////////////////////
+ // database stuff
+public:
+ MICryptoEngine *m_crypto;
+
+protected:
+ TreeDB m_dbGlobal;
+ DBHeader m_header;
+
+ HANDLE hSettingChangeEvent, hContactDeletedEvent, hContactAddedEvent, hEventMarkedRead;
+
+ mir_cs m_csDbAccess;
+
+ int CheckProto(DBCachedContact *cc, const char *proto);
+
+ ////////////////////////////////////////////////////////////////////////////
+ // settings
+
+ TreeDB m_dbSettings;
+ int m_codePage;
+ HANDLE hService, hHook;
+
+ ////////////////////////////////////////////////////////////////////////////
+ // contacts
+
+ TreeDB m_dbContacts;
+ int m_contactCount, m_dwMaxContactId;
+
+ int WipeContactHistory(DBContact *dbc);
+
+ ////////////////////////////////////////////////////////////////////////////
+ // events
+
+ TreeDB m_dbEvents, m_dbEventsSort;
+ DWORD m_dwMaxEventId, m_tsLast;
+ MEVENT m_evLast;
+
+ void FindNextUnread(DBCachedContact *cc, DBEventSortingKey &key2);
+
+ ////////////////////////////////////////////////////////////////////////////
+ // modules
+
+ TreeDB m_dbModules;
+ HANDLE m_hModHeap;
+ LIST<ModuleName> m_lMods, m_lOfs;
+ LIST<char> m_lResidentSettings;
+ HANDLE hEventAddedEvent, hEventDeletedEvent, hEventFilterAddedEvent;
+ MCONTACT m_hLastCachedContact;
+ int m_maxModuleID;
+
+ void AddToList(char *name, DWORD ofs);
+ DWORD FindExistingModuleNameOfs(const char *szName);
+ int InitModuleNames(void);
+ DWORD GetModuleNameOfs(const char *szName);
+ char* GetModuleNameByOfs(DWORD ofs);
+
+ ////////////////////////////////////////////////////////////////////////////
+ // checker
+
+ int PeekSegment(DWORD ofs, PVOID buf, int cbBytes);
+ int ReadSegment(DWORD ofs, PVOID buf, int cbBytes);
+ int ReadWrittenSegment(DWORD ofs, PVOID buf, int cbBytes);
+ int SignatureValid(DWORD ofs, DWORD dwSignature);
+ void FreeModuleChain();
+
+ DWORD ConvertModuleNameOfs(DWORD ofsOld);
+ void ConvertOldEvent(DBEvent*& dbei);
+
+ int GetContactSettingWorker(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv, int isStatic);
+ int WorkSettingsChain(DBContact *dbc, int firstTime);
+ int WorkEventChain(DWORD ofsContact, DBContact *dbc, int firstTime);
+
+ DWORD WriteSegment(DWORD ofs, PVOID buf, int cbBytes);
+ DWORD WriteEvent(DBEvent *dbe);
+ DWORD PeekEvent(DWORD ofs, DWORD dwContactID, DBEvent &dbe);
+ void WriteOfsNextToPrevious(DWORD ofsPrev, DBContact *dbc, DWORD ofsNext);
+ void FinishUp(DWORD ofsLast, DBContact *dbc);
+
+ DBCHeckCallback *cb;
+ DWORD sourceFileSize, ofsAggrCur;
+
+ ////////////////////////////////////////////////////////////////////////////
+ // encryption
+
+ int InitCrypt(void);
+ void ToggleEventsEncryption(MCONTACT contactID);
+ void ToggleSettingsEncryption(MCONTACT contactID);
+
+ void InitDialogs();
+ bool EnterPassword(const BYTE *pKey, const size_t keyLen);
+};
diff --git a/plugins/Dbx_kyoto/src/dbmodulechain.cpp b/plugins/Dbx_kyoto/src/dbmodulechain.cpp new file mode 100644 index 0000000000..9416f5305d --- /dev/null +++ b/plugins/Dbx_kyoto/src/dbmodulechain.cpp @@ -0,0 +1,127 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+void CDbxKV::AddToList(char *name, DWORD ofs)
+{
+ ModuleName *mn = (ModuleName*)HeapAlloc(m_hModHeap, 0, sizeof(ModuleName));
+ mn->name = name;
+ mn->ofs = ofs;
+
+ if (m_lMods.getIndex(mn) != -1)
+ DatabaseCorruption(_T("%s (Module Name not unique)"));
+ m_lMods.insert(mn);
+
+ if (m_lOfs.getIndex(mn) != -1)
+ DatabaseCorruption(_T("%s (Module Offset not unique)"));
+ m_lOfs.insert(mn);
+}
+
+int CDbxKV::InitModuleNames(void)
+{
+ m_maxModuleID = 0;
+
+ cursor_ptr cursor(m_dbModules);
+ for (cursor->jump(); cursor->step();) {
+ size_t size;
+ const char *pRec, *pKey = cursor->get(&size, &pRec, &size);
+ DBModuleName *pmod = (DBModuleName*)pRec;
+ if (pmod->dwSignature != DBMODULENAME_SIGNATURE)
+ DatabaseCorruption(NULL);
+
+ char *pVal = (char*)HeapAlloc(m_hModHeap, 0, pmod->cbName+1);
+ memcpy(pVal, pmod->name, pmod->cbName);
+ pVal[pmod->cbName] = 0;
+
+ int moduleId = *(int*)pKey;
+ AddToList(pVal, moduleId);
+
+ if (moduleId > m_maxModuleID)
+ m_maxModuleID = moduleId;
+ delete[] pKey;
+ }
+
+ return 0;
+}
+
+DWORD CDbxKV::FindExistingModuleNameOfs(const char *szName)
+{
+ ModuleName mn = { (char*)szName, 0 };
+
+ int index = m_lMods.getIndex(&mn);
+ if (index != -1)
+ return m_lMods[index]->ofs;
+
+ return 0;
+}
+
+// will create the offset if it needs to
+DWORD CDbxKV::GetModuleNameOfs(const char *szName)
+{
+ DWORD ofsExisting = FindExistingModuleNameOfs(szName);
+ if (ofsExisting)
+ return ofsExisting;
+
+ if (m_bReadOnly)
+ return 0;
+
+ int nameLen = (int)strlen(szName);
+
+ // need to create the module name
+ int newIdx = ++m_maxModuleID;
+ DBModuleName *pmod = (DBModuleName*)_alloca(sizeof(DBModuleName) + nameLen);
+ pmod->dwSignature = DBMODULENAME_SIGNATURE;
+ pmod->cbName = (char)nameLen;
+ strcpy(pmod->name, szName);
+ m_dbModules.set((LPCSTR)&newIdx, sizeof(int), (LPCSTR)pmod, sizeof(DBModuleName) + nameLen);
+
+ // add to cache
+ char *mod = (char*)HeapAlloc(m_hModHeap, 0, nameLen + 1);
+ strcpy(mod, szName);
+ AddToList(mod, newIdx);
+
+ // quit
+ return -1;
+}
+
+char* CDbxKV::GetModuleNameByOfs(DWORD ofs)
+{
+ ModuleName mn = { NULL, ofs };
+ int index = m_lOfs.getIndex(&mn);
+ if (index != -1)
+ return m_lOfs[index]->name;
+
+ return NULL;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::EnumModuleNames(DBMODULEENUMPROC pFunc, void *pParam)
+{
+ for (int i = 0; i < m_lMods.getCount(); i++) {
+ ModuleName *pmn = m_lMods[i];
+ int ret = pFunc(pmn->name, pmn->ofs, (LPARAM)pParam);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
diff --git a/plugins/Dbx_kyoto/src/dbsettings.cpp b/plugins/Dbx_kyoto/src/dbsettings.cpp new file mode 100644 index 0000000000..4f1e0579b1 --- /dev/null +++ b/plugins/Dbx_kyoto/src/dbsettings.cpp @@ -0,0 +1,598 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+#define VLT(n) ((n == DBVT_UTF8 || n == DBVT_ENCRYPTED)?DBVT_ASCIIZ:n)
+
+BOOL CDbxKV::IsSettingEncrypted(LPCSTR szModule, LPCSTR szSetting)
+{
+ if (!_strnicmp(szSetting, "password", 8)) return true;
+ if (!strcmp(szSetting, "NLProxyAuthPassword")) return true;
+ if (!strcmp(szSetting, "LNPassword")) return true;
+ if (!strcmp(szSetting, "FileProxyPassword")) return true;
+ if (!strcmp(szSetting, "TokenSecret")) return true;
+
+ if (!strcmp(szModule, "SecureIM")) {
+ if (!strcmp(szSetting, "pgp")) return true;
+ if (!strcmp(szSetting, "pgpPrivKey")) return true;
+ }
+ return false;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+static bool ValidLookupName(LPCSTR szModule, LPCSTR szSetting)
+{
+ if (!strcmp(szModule, META_PROTO))
+ return strcmp(szSetting, "IsSubcontact") && strcmp(szSetting, "ParentMetaID");
+
+ if (!strcmp(szModule, "Ignore"))
+ return false;
+
+ return true;
+}
+
+int CDbxKV::GetContactSettingWorker(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv, int isStatic)
+{
+ if (szSetting == NULL || szModule == NULL)
+ return 1;
+
+ // the db format can't tolerate more than 255 bytes of space (incl. null) for settings+module name
+ int settingNameLen = (int)strlen(szSetting);
+ int moduleNameLen = (int)strlen(szModule);
+ if (settingNameLen > 0xFE) {
+#ifdef _DEBUG
+ OutputDebugStringA("GetContactSettingWorker() got a > 255 setting name length. \n");
+#endif
+ return 1;
+ }
+ if (moduleNameLen > 0xFE) {
+#ifdef _DEBUG
+ OutputDebugStringA("GetContactSettingWorker() got a > 255 module name length. \n");
+#endif
+ return 1;
+ }
+
+ mir_cslock lck(m_csDbAccess);
+
+LBL_Seek:
+ char *szCachedSettingName = m_cache->GetCachedSetting(szModule, szSetting, moduleNameLen, settingNameLen);
+
+ DBVARIANT *pCachedValue = m_cache->GetCachedValuePtr(contactID, szCachedSettingName, 0);
+ if (pCachedValue != NULL) {
+ if (pCachedValue->type == DBVT_ASCIIZ || pCachedValue->type == DBVT_UTF8) {
+ int cbOrigLen = dbv->cchVal;
+ char *cbOrigPtr = dbv->pszVal;
+ memcpy(dbv, pCachedValue, sizeof(DBVARIANT));
+ if (isStatic) {
+ int cbLen = 0;
+ if (pCachedValue->pszVal != NULL)
+ cbLen = (int)strlen(pCachedValue->pszVal);
+
+ cbOrigLen--;
+ dbv->pszVal = cbOrigPtr;
+ if (cbLen < cbOrigLen)
+ cbOrigLen = cbLen;
+ memcpy(dbv->pszVal, pCachedValue->pszVal, cbOrigLen);
+ dbv->pszVal[cbOrigLen] = 0;
+ dbv->cchVal = cbLen;
+ }
+ else {
+ dbv->pszVal = (char*)mir_alloc(strlen(pCachedValue->pszVal) + 1);
+ strcpy(dbv->pszVal, pCachedValue->pszVal);
+ }
+ }
+ else memcpy(dbv, pCachedValue, sizeof(DBVARIANT));
+
+ return (pCachedValue->type == DBVT_DELETED) ? 1 : 0;
+ }
+
+ // never look db for the resident variable
+ if (szCachedSettingName[-1] != 0)
+ return 1;
+
+ DBCachedContact *cc = (contactID) ? m_cache->GetCachedContact(contactID) : NULL;
+
+ DBSettingSortingKey keySearch;
+ keySearch.dwContactID = contactID;
+ keySearch.dwOfsModule = GetModuleNameOfs(szModule);
+ strncpy_s(keySearch.szSettingName, szSetting, _TRUNCATE);
+
+ char *rec = (char*)_alloca(65536);
+ if (-1 == m_dbSettings.get((LPCSTR)&keySearch, 2 * sizeof(DWORD) + settingNameLen, rec, 65536)) {
+ // try to get the missing mc setting from the active sub
+ if (cc && cc->IsMeta() && ValidLookupName(szModule, szSetting)) {
+ if (contactID = db_mc_getDefault(contactID)) {
+ if (szModule = GetContactProto(contactID)) {
+ moduleNameLen = (int)strlen(szModule);
+ goto LBL_Seek;
+ }
+ }
+ }
+ return 1;
+ }
+
+ BYTE *pBlob = (BYTE*)rec;
+ if (isStatic && (pBlob[0] & DBVTF_VARIABLELENGTH) && VLT(dbv->type) != VLT(pBlob[0]))
+ return 1;
+
+ int varLen;
+ BYTE iType = dbv->type = pBlob[0]; pBlob++;
+ switch (iType) {
+ case DBVT_DELETED: /* this setting is deleted */
+ dbv->type = DBVT_DELETED;
+ return 2;
+
+ case DBVT_BYTE: dbv->bVal = *pBlob; break;
+ case DBVT_WORD: dbv->wVal = *(WORD*)pBlob; break;
+ case DBVT_DWORD: dbv->dVal = *(DWORD*)pBlob; break;
+
+ case DBVT_UTF8:
+ case DBVT_ASCIIZ:
+ varLen = *(WORD*)pBlob;
+ pBlob += 2;
+ if (isStatic) {
+ dbv->cchVal--;
+ if (varLen < dbv->cchVal)
+ dbv->cchVal = varLen;
+ memmove(dbv->pszVal, pBlob, dbv->cchVal); // decode
+ dbv->pszVal[dbv->cchVal] = 0;
+ dbv->cchVal = varLen;
+ }
+ else {
+ dbv->pszVal = (char*)mir_alloc(1 + varLen);
+ memmove(dbv->pszVal, pBlob, varLen);
+ dbv->pszVal[varLen] = 0;
+ }
+ break;
+
+ case DBVT_BLOB:
+ varLen = *(WORD*)pBlob;
+ pBlob += 2;
+ if (isStatic) {
+ if (varLen < dbv->cpbVal)
+ dbv->cpbVal = varLen;
+ memmove(dbv->pbVal, pBlob, dbv->cpbVal);
+ }
+ else {
+ dbv->pbVal = (BYTE *)mir_alloc(varLen);
+ memmove(dbv->pbVal, pBlob, varLen);
+ }
+ dbv->cpbVal = varLen;
+ break;
+
+ case DBVT_ENCRYPTED:
+ if (m_crypto == NULL)
+ return 1;
+
+ varLen = *(WORD*)pBlob;
+ pBlob += 2;
+
+ size_t realLen;
+ ptrA decoded(m_crypto->decodeString(pBlob, varLen, &realLen));
+ if (decoded == NULL)
+ return 1;
+
+ varLen = (WORD)realLen;
+ dbv->type = DBVT_UTF8;
+ if (isStatic) {
+ dbv->cchVal--;
+ if (varLen < dbv->cchVal)
+ dbv->cchVal = varLen;
+ memmove(dbv->pszVal, decoded, dbv->cchVal);
+ dbv->pszVal[dbv->cchVal] = 0;
+ dbv->cchVal = varLen;
+ }
+ else {
+ dbv->pszVal = (char*)mir_alloc(1 + varLen);
+ memmove(dbv->pszVal, decoded, varLen);
+ dbv->pszVal[varLen] = 0;
+ }
+ break;
+ }
+
+ /**** add to cache **********************/
+ if (iType != DBVT_BLOB && iType != DBVT_ENCRYPTED) {
+ DBVARIANT *pCachedValue = m_cache->GetCachedValuePtr(contactID, szCachedSettingName, 1);
+ if (pCachedValue != NULL)
+ m_cache->SetCachedVariant(dbv, pCachedValue);
+ }
+
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::GetContactSetting(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv)
+{
+ dbv->type = 0;
+ if (GetContactSettingWorker(contactID, szModule, szSetting, dbv, 0))
+ return 1;
+
+ if (dbv->type == DBVT_UTF8) {
+ WCHAR *tmp = NULL;
+ char *p = NEWSTR_ALLOCA(dbv->pszVal);
+ if (mir_utf8decode(p, &tmp) != NULL) {
+ BOOL bUsed = FALSE;
+ int result = WideCharToMultiByte(m_codePage, WC_NO_BEST_FIT_CHARS, tmp, -1, NULL, 0, NULL, &bUsed);
+
+ mir_free(dbv->pszVal);
+
+ if (bUsed || result == 0) {
+ dbv->type = DBVT_WCHAR;
+ dbv->pwszVal = tmp;
+ }
+ else {
+ dbv->type = DBVT_ASCIIZ;
+ dbv->pszVal = (char *)mir_alloc(result);
+ WideCharToMultiByte(m_codePage, WC_NO_BEST_FIT_CHARS, tmp, -1, dbv->pszVal, result, NULL, NULL);
+ mir_free(tmp);
+ }
+ }
+ else {
+ dbv->type = DBVT_ASCIIZ;
+ mir_free(tmp);
+ }
+ }
+
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::GetContactSettingStr(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv)
+{
+ int iSaveType = dbv->type;
+
+ if (GetContactSettingWorker(contactID, szModule, szSetting, dbv, 0))
+ return 1;
+
+ if (iSaveType == 0 || iSaveType == dbv->type)
+ return 0;
+
+ if (dbv->type != DBVT_ASCIIZ && dbv->type != DBVT_UTF8)
+ return 1;
+
+ if (iSaveType == DBVT_WCHAR) {
+ if (dbv->type != DBVT_UTF8) {
+ int len = MultiByteToWideChar(CP_ACP, 0, dbv->pszVal, -1, NULL, 0);
+ wchar_t *wszResult = (wchar_t*)mir_alloc((len + 1)*sizeof(wchar_t));
+ if (wszResult == NULL)
+ return 1;
+
+ MultiByteToWideChar(CP_ACP, 0, dbv->pszVal, -1, wszResult, len);
+ wszResult[len] = 0;
+ mir_free(dbv->pszVal);
+ dbv->pwszVal = wszResult;
+ }
+ else {
+ char* savePtr = NEWSTR_ALLOCA(dbv->pszVal);
+ mir_free(dbv->pszVal);
+ if (!mir_utf8decode(savePtr, &dbv->pwszVal))
+ return 1;
+ }
+ }
+ else if (iSaveType == DBVT_UTF8) {
+ char* tmpBuf = mir_utf8encode(dbv->pszVal);
+ if (tmpBuf == NULL)
+ return 1;
+
+ mir_free(dbv->pszVal);
+ dbv->pszVal = tmpBuf;
+ }
+ else if (iSaveType == DBVT_ASCIIZ)
+ mir_utf8decode(dbv->pszVal, NULL);
+
+ dbv->type = iSaveType;
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::GetContactSettingStatic(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting, DBVARIANT *dbv)
+{
+ if (GetContactSettingWorker(contactID, szModule, szSetting, dbv, 1))
+ return 1;
+
+ if (dbv->type == DBVT_UTF8) {
+ mir_utf8decode(dbv->pszVal, NULL);
+ dbv->type = DBVT_ASCIIZ;
+ }
+
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::FreeVariant(DBVARIANT *dbv)
+{
+ if (dbv == 0) return 1;
+
+ switch (dbv->type) {
+ case DBVT_ASCIIZ:
+ case DBVT_UTF8:
+ case DBVT_WCHAR:
+ if (dbv->pszVal) mir_free(dbv->pszVal);
+ dbv->pszVal = 0;
+ break;
+ case DBVT_BLOB:
+ if (dbv->pbVal) mir_free(dbv->pbVal);
+ dbv->pbVal = 0;
+ break;
+ }
+ dbv->type = 0;
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::SetSettingResident(BOOL bIsResident, const char *pszSettingName)
+{
+ char *szSetting = m_cache->GetCachedSetting(NULL, pszSettingName, 0, (int)strlen(pszSettingName));
+ szSetting[-1] = (char)bIsResident;
+
+ mir_cslock lck(m_csDbAccess);
+ int idx = m_lResidentSettings.getIndex(szSetting);
+ if (idx == -1) {
+ if (bIsResident)
+ m_lResidentSettings.insert(szSetting);
+ }
+ else if (!bIsResident)
+ m_lResidentSettings.remove(idx);
+
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::WriteContactSetting(MCONTACT contactID, DBCONTACTWRITESETTING *dbcws)
+{
+ if (dbcws == NULL || dbcws->szSetting == NULL || dbcws->szModule == NULL || m_bReadOnly)
+ return 1;
+
+ // the db format can't tolerate more than 255 bytes of space (incl. null) for settings+module name
+ int settingNameLen = (int)strlen(dbcws->szSetting);
+ int moduleNameLen = (int)strlen(dbcws->szModule);
+ if (settingNameLen > 0xFE) {
+#ifdef _DEBUG
+ OutputDebugStringA("WriteContactSetting() got a > 255 setting name length. \n");
+#endif
+ return 1;
+ }
+ if (moduleNameLen > 0xFE) {
+#ifdef _DEBUG
+ OutputDebugStringA("WriteContactSetting() got a > 255 module name length. \n");
+#endif
+ return 1;
+ }
+
+ // used for notifications
+ DBCONTACTWRITESETTING dbcwNotif = *dbcws;
+ if (dbcwNotif.value.type == DBVT_WCHAR) {
+ if (dbcwNotif.value.pszVal != NULL) {
+ char* val = mir_utf8encodeW(dbcwNotif.value.pwszVal);
+ if (val == NULL)
+ return 1;
+
+ dbcwNotif.value.pszVal = (char*)alloca(strlen(val) + 1);
+ strcpy(dbcwNotif.value.pszVal, val);
+ mir_free(val);
+ dbcwNotif.value.type = DBVT_UTF8;
+ }
+ else return 1;
+ }
+
+ if (dbcwNotif.szModule == NULL || dbcwNotif.szSetting == NULL)
+ return 1;
+
+ DBCONTACTWRITESETTING dbcwWork = dbcwNotif;
+
+ mir_ptr<BYTE> pEncoded(NULL);
+ bool bIsEncrypted = false;
+ switch (dbcwWork.value.type) {
+ case DBVT_BYTE: case DBVT_WORD: case DBVT_DWORD:
+ break;
+
+ case DBVT_ASCIIZ: case DBVT_UTF8:
+ bIsEncrypted = m_bEncrypted || IsSettingEncrypted(dbcws->szModule, dbcws->szSetting);
+ LBL_WriteString:
+ if (dbcwWork.value.pszVal == NULL)
+ return 1;
+ dbcwWork.value.cchVal = (WORD)strlen(dbcwWork.value.pszVal);
+ if (bIsEncrypted) {
+ size_t len;
+ BYTE *pResult = m_crypto->encodeString(dbcwWork.value.pszVal, &len);
+ if (pResult != NULL) {
+ pEncoded = dbcwWork.value.pbVal = pResult;
+ dbcwWork.value.cpbVal = (WORD)len;
+ dbcwWork.value.type = DBVT_ENCRYPTED;
+ }
+ }
+ break;
+
+ case DBVT_UNENCRYPTED:
+ dbcwNotif.value.type = dbcwWork.value.type = DBVT_UTF8;
+ goto LBL_WriteString;
+
+ case DBVT_BLOB: case DBVT_ENCRYPTED:
+ if (dbcwWork.value.pbVal == NULL)
+ return 1;
+ break;
+ default:
+ return 1;
+ }
+
+ mir_cslockfull lck(m_csDbAccess);
+
+ char *szCachedSettingName = m_cache->GetCachedSetting(dbcwWork.szModule, dbcwWork.szSetting, moduleNameLen, settingNameLen);
+
+ // we don't cache blobs and passwords
+ if (dbcwWork.value.type != DBVT_BLOB && dbcwWork.value.type != DBVT_ENCRYPTED && !bIsEncrypted) {
+ DBVARIANT *pCachedValue = m_cache->GetCachedValuePtr(contactID, szCachedSettingName, 1);
+ if (pCachedValue != NULL) {
+ bool bIsIdentical = false;
+ if (pCachedValue->type == dbcwWork.value.type) {
+ switch (dbcwWork.value.type) {
+ case DBVT_BYTE: bIsIdentical = pCachedValue->bVal == dbcwWork.value.bVal; break;
+ case DBVT_WORD: bIsIdentical = pCachedValue->wVal == dbcwWork.value.wVal; break;
+ case DBVT_DWORD: bIsIdentical = pCachedValue->dVal == dbcwWork.value.dVal; break;
+ case DBVT_UTF8:
+ case DBVT_ASCIIZ: bIsIdentical = strcmp(pCachedValue->pszVal, dbcwWork.value.pszVal) == 0; break;
+ }
+ if (bIsIdentical)
+ return 0;
+ }
+ m_cache->SetCachedVariant(&dbcwWork.value, pCachedValue);
+ }
+ if (szCachedSettingName[-1] != 0) {
+ lck.unlock();
+ NotifyEventHooks(hSettingChangeEvent, contactID, (LPARAM)&dbcwWork);
+ return 0;
+ }
+ }
+ else m_cache->GetCachedValuePtr(contactID, szCachedSettingName, -1);
+
+ DBSettingSortingKey keySearch;
+ keySearch.dwContactID = contactID;
+ keySearch.dwOfsModule = GetModuleNameOfs(dbcws->szModule);
+ strncpy_s(keySearch.szSettingName, dbcws->szSetting, _TRUNCATE);
+
+ size_t recSize;
+ switch (dbcwWork.value.type) {
+ case DBVT_BYTE: recSize = 2; break;
+ case DBVT_WORD: recSize = 3; break;
+ case DBVT_DWORD: recSize = 5; break;
+
+ case DBVT_ASCIIZ:
+ case DBVT_UTF8:
+ recSize = 3 + dbcwWork.value.cchVal; break;
+
+ case DBVT_BLOB:
+ case DBVT_ENCRYPTED:
+ recSize = 3 + dbcwWork.value.cpbVal; break;
+
+ default:
+ return 1;
+ }
+
+ char *recData = (char*)_alloca(recSize);
+ BYTE *pBlob = (BYTE*)recData;
+ *pBlob++ = dbcwWork.value.type;
+ switch (dbcwWork.value.type) {
+ case DBVT_BYTE: *pBlob = dbcwWork.value.bVal; break;
+ case DBVT_WORD: *(WORD*)pBlob = dbcwWork.value.wVal; break;
+ case DBVT_DWORD: *(DWORD*)pBlob = dbcwWork.value.dVal; break;
+
+ case DBVT_ASCIIZ:
+ case DBVT_UTF8:
+ *(WORD*)pBlob = dbcwWork.value.cchVal;
+ memcpy(pBlob+2, dbcwWork.value.pszVal, dbcwWork.value.cchVal);
+ break;
+
+ case DBVT_BLOB:
+ case DBVT_ENCRYPTED:
+ *(WORD*)pBlob = dbcwWork.value.cpbVal;
+ memcpy(pBlob+2, dbcwWork.value.pbVal, dbcwWork.value.cpbVal);
+ }
+ m_dbSettings.set((LPCSTR)&keySearch, 2 * sizeof(DWORD) + settingNameLen, recData, recSize);
+ lck.unlock();
+
+ // notify
+ NotifyEventHooks(hSettingChangeEvent, contactID, (LPARAM)&dbcwNotif);
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::DeleteContactSetting(MCONTACT contactID, LPCSTR szModule, LPCSTR szSetting)
+{
+ if (!szModule || !szSetting)
+ return 1;
+
+ // the db format can't tolerate more than 255 bytes of space (incl. null) for settings+module name
+ int settingNameLen = (int)strlen(szSetting);
+ int moduleNameLen = (int)strlen(szModule);
+ if (settingNameLen > 0xFE) {
+#ifdef _DEBUG
+ OutputDebugStringA("DeleteContactSetting() got a > 255 setting name length. \n");
+#endif
+ return 1;
+ }
+ if (moduleNameLen > 0xFE) {
+#ifdef _DEBUG
+ OutputDebugStringA("DeleteContactSetting() got a > 255 module name length. \n");
+#endif
+ return 1;
+ }
+
+ MCONTACT saveContact = contactID;
+ {
+ mir_cslock lck(m_csDbAccess);
+ char *szCachedSettingName = m_cache->GetCachedSetting(szModule, szSetting, moduleNameLen, settingNameLen);
+ if (szCachedSettingName[-1] == 0) { // it's not a resident variable
+ DBSettingSortingKey keySearch;
+ keySearch.dwContactID = contactID;
+ keySearch.dwOfsModule = GetModuleNameOfs(szModule);
+ strncpy_s(keySearch.szSettingName, szSetting, _TRUNCATE);
+ if (!m_dbSettings.remove((LPCSTR)&keySearch, 2 * sizeof(DWORD) + settingNameLen))
+ return 1;
+ }
+
+ m_cache->GetCachedValuePtr(saveContact, szCachedSettingName, -1);
+ }
+
+ // notify
+ DBCONTACTWRITESETTING dbcws = { 0 };
+ dbcws.szModule = szModule;
+ dbcws.szSetting = szSetting;
+ dbcws.value.type = DBVT_DELETED;
+ NotifyEventHooks(hSettingChangeEvent, saveContact, (LPARAM)&dbcws);
+ return 0;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::EnumContactSettings(MCONTACT contactID, DBCONTACTENUMSETTINGS* dbces)
+{
+ if (!dbces->szModule)
+ return -1;
+
+ mir_cslock lck(m_csDbAccess);
+
+ int result = -1;
+
+ DBSettingSortingKey keySearch;
+ keySearch.dwContactID = contactID;
+ keySearch.dwOfsModule = GetModuleNameOfs(dbces->szModule);
+ memset(keySearch.szSettingName, 0, SIZEOF(keySearch.szSettingName));
+
+ cursor_ptr cursor(m_dbSettings);
+ cursor->jump((LPCSTR)&keySearch, sizeof(keySearch));
+ while (cursor->step()) {
+ size_t size, keySize;
+ const char *pRec, *key = cursor->get(&keySize, &pRec, &size);
+ DBSettingSortingKey *pKey = (DBSettingSortingKey*)key;
+ if (pKey->dwContactID != contactID || pKey->dwOfsModule != keySearch.dwOfsModule)
+ break;
+
+ char szSetting[256];
+ strncpy_s(szSetting, pKey->szSettingName, keySize - sizeof(DWORD) * 2);
+ result = (dbces->pfnEnumProc)(szSetting, dbces->lParam);
+ }
+
+ return result;
+}
+
+STDMETHODIMP_(BOOL) CDbxKV::EnumResidentSettings(DBMODULEENUMPROC pFunc, void *pParam)
+{
+ for (int i = 0; i < m_lResidentSettings.getCount(); i++) {
+ int ret = pFunc(m_lResidentSettings[i], 0, (LPARAM)pParam);
+ if (ret) return ret;
+ }
+ return 0;
+}
diff --git a/plugins/Dbx_kyoto/src/init.cpp b/plugins/Dbx_kyoto/src/init.cpp new file mode 100644 index 0000000000..e9e3247fd8 --- /dev/null +++ b/plugins/Dbx_kyoto/src/init.cpp @@ -0,0 +1,134 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+int hLangpack;
+
+static PLUGININFOEX pluginInfo =
+{
+ sizeof(PLUGININFOEX),
+ __PLUGIN_NAME,
+ PLUGIN_MAKE_VERSION(__MAJOR_VERSION, __MINOR_VERSION, __RELEASE_NUM, __BUILD_NUM),
+ __DESCRIPTION,
+ __AUTHOR,
+ __AUTHOREMAIL,
+ __COPYRIGHT,
+ __AUTHORWEB,
+ UNICODE_AWARE | STATIC_PLUGIN,
+ // {7C3D0A33-2646-4001-9107-F35EA299D292}
+ { 0x7c3d0a33, 0x2646, 0x4001, { 0x91, 0x7, 0xf3, 0x5e, 0xa2, 0x99, 0xd2, 0x92 } }
+};
+
+HINSTANCE g_hInst = NULL;
+
+LIST<CDbxKV> g_Dbs(1, HandleKeySortT);
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+// returns 0 if the profile is created, EMKPRF*
+static int makeDatabase(const TCHAR *profile)
+{
+ std::auto_ptr<CDbxKV> db(new CDbxKV(profile, 0));
+ return db->Create();
+}
+
+// returns 0 if the given profile has a valid header
+static int grokHeader(const TCHAR *profile)
+{
+ std::auto_ptr<CDbxKV> db(new CDbxKV(profile, DBMODE_SHARED | DBMODE_READONLY));
+ return db->Check();
+}
+
+// returns 0 if all the APIs are injected otherwise, 1
+static MIDatabase* LoadDatabase(const TCHAR *profile, BOOL bReadOnly)
+{
+ // set the memory, lists & UTF8 manager
+ mir_getLP(&pluginInfo);
+
+ std::auto_ptr<CDbxKV> db(new CDbxKV(profile, (bReadOnly) ? DBMODE_READONLY : 0));
+ if (db->Load(false) != ERROR_SUCCESS)
+ return NULL;
+
+ g_Dbs.insert(db.get());
+ return db.release();
+}
+
+static int UnloadDatabase(MIDatabase *db)
+{
+ g_Dbs.remove((CDbxKV*)db);
+ delete (CDbxKV*)db;
+ return 0;
+}
+
+MIDatabaseChecker* CheckDb(const TCHAR *profile, int *error)
+{
+ std::auto_ptr<CDbxKV> db(new CDbxKV(profile, DBMODE_READONLY));
+ if (db->Load(true) != ERROR_SUCCESS) {
+ *error = ERROR_ACCESS_DENIED;
+ return NULL;
+ }
+
+ if (db->PrepareCheck(error))
+ return NULL;
+
+ return db.release();
+}
+
+static DATABASELINK dblink =
+{
+ sizeof(DATABASELINK),
+ "dbx_kyoto",
+ _T("KyotoCabinet database driver"),
+ makeDatabase,
+ grokHeader,
+ LoadDatabase,
+ UnloadDatabase,
+ CheckDb
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+extern "C" __declspec(dllexport) PLUGININFOEX* MirandaPluginInfoEx(DWORD)
+{
+ return &pluginInfo;
+}
+
+extern "C" __declspec(dllexport) const MUUID MirandaInterfaces[] = { MIID_DATABASE, MIID_LAST };
+
+extern "C" __declspec(dllexport) int Load(void)
+{
+ RegisterDatabasePlugin(&dblink);
+ return 0;
+}
+
+extern "C" __declspec(dllexport) int Unload(void)
+{
+ return 0;
+}
+
+BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD, LPVOID)
+{
+ g_hInst = hInstDLL;
+ return TRUE;
+}
diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/COPYING b/plugins/Dbx_kyoto/src/kyotocabinet/COPYING new file mode 100644 index 0000000000..94a9ed024d --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/ChangeLog b/plugins/Dbx_kyoto/src/kyotocabinet/ChangeLog new file mode 100644 index 0000000000..aec5eebf7e --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/ChangeLog @@ -0,0 +1,1170 @@ +2012-05-24 FAL Labs <info@fallabs.com> + + * kcthread.cc (CondVar::wait): a bug on Win32 was fixed. + + * kcdbext.h (IndexDB::set, IndexDB::replace): a bug of updating existing records was fixed. + + * kcdb.h (DB::check): new function. + + - Release: 1.2.76 + +2012-03-28 FAL Labs <info@fallabs.com> + + * kcpolydb.h (PolyDB::match_similar): supressed warnings on signedness. + + - Release: 1.2.75 + +2012-03-15 FAL Labs <info@fallabs.com> + + * kcpolydb.h (PolyDB::SimilarKey): the type of a member was modified for c++0x mode. + + - Release: 1.2.74 + +2012-03-06 FAL Labs <info@fallabs.com> + + * kcutil.h (memdist, strutfdist, strucsdist): new functions. + + * kcpolydb.h (PolyDB::match_similar): new funcion. + + * kclangc.h (kclevdist, kcdbmatchsimilar): new functions. + + * kcfile.cc (File::open): LOCKFILE_FAIL_IMMEDIATELY support was added for Win32. + + - Release: 1.2.73 + +2011-12-15 FAL Labs <info@fallabs.com> + + * kcfile.h (File::read, File::read_rast): a bug of invalid memory deletion was fixed. + + - Release: 1.2.72 + +2011-12-01 FAL Labs <info@fallabs.com> + + * kclang.cc (kcdbcas): the erroneous order of parameters was rectified. + + - Release: 1.2.71 + +2011-08-01 FAL Labs <info@fallabs.com> + + * kcplantdb.h (PlantDB::jump, PlantDB::step): a bug related to direction switch was fixed. + + - Release: 1.2.70 + +2011-07-16 FAL Labs <info@fallabs.com> + + * kcutil.h (kcstrutflen): new function. + + * kcplantdb (PlantDB::write_key): new function. + + * kctextdb (TextDB::write_key): new function. + + * kctextdb.h (TextDB::scan_parallel_impl): performance was improved. + + * kctextdb.h (read_next): a bug of border condition was fixed. + + - Release: 1.2.69 + +2011-07-14 FAL Labs <info@fallabs.com> + + * kcutil.h (strutftoucs, strucstoutf): performance was improved. + + - Release: 1.2.68 + +2011-07-12 FAL Labs <info@fallabs.com> + + * kcutil.h (memicmp, memmem, memimem, stristr): new functions. + + * kcutil.h (strupper, strlower, strtrim): new functions. + + - Release: 1.2.67 + +2011-07-01 FAL Labs <info@fallabs.com> + + * kcutil.h (strutftoucs, strucstoutf): new functions. + + - Release: 1.2.66 + +2011-06-28 FAL Labs <info@fallabs.com> + + * kcutil.h (urlencode): modified for compatibility with RFC 3986. + + * kctextdb.h, kctextdb.cc: new files. + + - Release: 1.2.65 + +2011-06-15 FAL Labs <info@fallabs.com> + + * kcmap.h (TinyHasyMap::accept_parallel): new function. + + * kcdbext.h (MapReduce::FlushThread): new class. + + * kcdbext.h (MapReduce::cache_flush): parallel mode was added. + + - Release: 1.2.64 + +2011-06-12 FAL Labs <info@fallabs.com> + + * kcthread.cc (TaskQueue::do_start, TaskQueue::do_finish): new functions. + + * kcdb.cc (BasicDB::Curosr::seize, BasicDB::seize): new functions. + + * kcplantdb.h (PlantDB::Cursor::accept_atom): backward stepping is now supported. + + - Release: 1.2.63 + +2011-06-12 FAL Labs <info@fallabs.com> + + * kcthread.cc (CondVar::wait): a bug of race condition on Win32 was fixed. + + - Release: 1.2.62 + +2011-06-09 FAL Labs <info@fallabs.com> + + * kcthread.h (CondMap): new class. + + * kcutiltest.cc (runcond, proccond): new functions. + + - Release: 1.2.61 + +2011-06-07 FAL Labs <info@fallabs.com> + + * kccachedb.h (CacheDB::scan_parallel): a bug related to overloading was fixed. + + - Release: 1.2.60 + +2011-05-30 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::scan_parallel, BasicDB::scan_parallel_impl): new functions. + + * kccachedb.h (CacheDB::switch_rotation): new function. + + * kcdbext.h (MapReduce::execute): parallel options were added. + + - Release: 1.2.59 + +2011-05-19 FAL Labs <info@fallabs.com> + + * kcplantdb.cc (PlantDB::accept): locking mode of transaction was modified. + + - Release: 1.2.58 + +2011-05-18 FAL Labs <info@fallabs.com> + + * kclangc.cc (kcdbincrint, kcdbincrdouble): "orig" parameter was added. + + - Release: 1.2.57 + +2011-05-14 FAL Labs <info@fallabs.com> + + * kcthread.c (AtomicInt64::cas): a bug on 64-bit Windows was fixed. + + * kcdb.h (DB::increment, DB::increment_double): "orig" parameter was added. + + - Release: 1.2.56 + +2011-05-13 FAL Labs <info@fallabs.com> + + * kcplantdb.h (PlantDB::accept): lock promotion was abolished. + + - Release: 1.2.55 + +2011-05-12 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB::accept, HashDB::accept_bulk): promotion was replaced by try locking. + + * kcplantdb.h (PlantDB::recalc_count): validation checking was added. + + - some spin locks in the database classes were replaced by the system lock primitives. + + - Release: 1.2.54 + +2011-05-01 FAL Labs <info@fallabs.com> + + * kcdbext.h (MapReduce::map): MapEmitter was integrated. + + * kcdbext.h (MapReduce::emit): new function. + + * kcdbext.h (MapReduce::execute): a bug of wrong handling of empty databases was fixed. + + - Release: 1.2.53 + +2011-04-10 FAL Labs <info@fallabs.com> + + * kcutil.h (OSNAME, PAGESIZ): renamed for portability to Solaris. + + - Release: 1.2.52 + +2011-03-22 FAL Labs <info@fallabs.com> + + * kclangc.h (kcidxnew, kcidxdel, kcidxopen, kcidxclose): new functions. + + - Release: 1.2.51 + +2011-03-08 FAL Labs <info@fallabs.com> + + * kcplantdb.h (PlantDB::defrag): dirty buffer clean-up was added. + + - Release: 1.2.50 + +2011-03-06 FAL Labs <info@fallabs.com> + + * kcutil.h (hexdecode): a bug of boundary violation was fixed. + + * kcthread.cc (SpinLock::unlock, SlottedSpinLock::unlock): release lock is now used. + + * kcmap.h (TinyArrayList): new class. + + * kcdb.h (BasicDB::log): new function. + + * kcdbext.h (IndexDB): new class. + + * kcutiltest.cc (runtalist, proctalist): new functions. + + * kcpolytest.h (runindex, procindex): new functions. + + * kclangctest.cc (runlist, proclist): new functions. + + - Release: 1.2.49 + +2011-03-01 FAL Labs <info@fallabs.com> + + * kclangc.h (kcmapnew, kcmapdel, kcmapiterator, kcmapsorter): new functions. + + * kclangctest.cc (runmap, procmap): new functions. + + - Release: 1.2.48 + +2011-02-25 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::occupy): new function. + + - Release: 1.2.47 + +2011-02-23 FAL Labs <info@fallabs.com> + + * kcutil.h (SIZEMAX, FLTMAX, DBLMAX): new constants. + + * kcplantdb.h (PlantDB::open): repair mechanism was modified. + + * kcplantdb.h (PlantDB::Cursor::accept_spec): a bug related to transactin was fixed. + + - all database classes were refactored to support some old compilers. + + - Release: 1.2.46 + +2011-02-21 FAL Labs <info@fallabs.com> + + * kcdbext.h (MapReduce::execute): adaptive comparator was added. + + * kcdbext.h (MapReduce::execute_reduce): new function. + + * kcdbext.h (MapReduce::before, MapReduce::after): new functions. + + - Release: 1.2.45 + +2011-02-15 FAL Labs <info@fallabs.com> + + * kcdb.h (DB::Visitor::visit_before, DB::Visitor::visit_after): new functions. + + - configuration files were modified for affinity for binary distributions. + + - all classes were refactored to abolish unnamed namespaces. + + - Release: 1.2.44 + +2011-02-12 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB): the type of record locking objects was changed. + + * kcdirdb.h (DirDB): the type of record locking objects was changed. + + * kcplantdb.h (PlantDB): the type of page locking objects was changed. + + - Release: 1.2.43 + +2011-02-10 FAL Labs <info@fallabs.com> + + * kcdb.h (DB::Cursor::get_key, DB::Cursor::get_value): signature was modified. + + * kcdb.h (DB::Cursor::get_pair, DB::Cursor::get): renamed. + + * kcdb.h (DB::get): signature was modified. + + - Release: 1.2.42 + +2011-02-07 FAL Labs <info@fallabs.com> + + * kccommon.h: macros for integer range were added. + + * kchashdb.h (HashDB::begin_transaction): behavior of locking was modified. + + * kcdirdb.h (DirDB::begin_transaction): behavior of locking was modified. + + - Release: 1.2.41 + +2011-02-05 FAL Labs <info@fallabs.com> + + * kccompare.h (LexicalDescendingComparator, LexicalDescendingComparator): new classes. + + * kccompare.h (LEXICALCOMP, DECIMALCOMP): the type was changed to pointer. + + * kccompress.h (ZLIBRAWCOMP): the type was changed to pointer. + + * kcplantdb.h (PlantDB::load_meta): a bug related to custom comparator was fixed. + + - Release: 1.2.40 + +2011-02-01 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::open): checking type type was added. + + * kcplantdb.h (Plantdb::reorganize_file): recovery condition was modified. + + * kcpolymgr.cc (proccheck): a bug related to path handling was fixed. + + - Release: 1.2.39 + +2011-01-30 FAL Labs <info@fallabs.com> + + * kcthread.cc: threshold of sleep locking was modified. + + * kcplantdb.h (PlantDB::accept): a bug related to race condition was fixed. + + * kcplantdb.h (PlantDB::save_leaf_node): locking mode was modified. + + - Release: 1.2.38 + +2011-01-25 FAL Labs <info@fallabs.com> + + * kcutil.h (strvecdump, strvecload, strmapdump, strmapload): new functions. + + - Release: 1.2.37 + +2011-01-21 FAL Labs <info@fallabs.com> + + * kcutil.cc (getsysinfo): a bug related to memory stats was fixed. + + * kcthread.cc (Thread::chill): new function. + + * kcthread.cc (ScopedMutex, ScopedSpinLock): refactored using primitives. + + * kcthread.cc (ScopedRWLock, ScopedSpinRWLock): refactored using primitives. + + - Release: 1.2.36 + +2011-01-11 FAL Labs <info@fallabs.com> + + * kcutil.cc: the order of header inclusion was modified. + + * kcthread.cc (SpinRWLock): space efficiency was improved. + + - Release: 1.2.35 + +2011-01-09 FAL Labs <info@fallabs.com> + + * kcplantdb.h (PlantDB::accept, PlantDB::accept_bulk): behavior of async was modified. + + - Release: 1.2.34 + +2011-01-05 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::open): handling the WAL file was modified. + + * kchashdb.h (HashDB::clear): handling the open flag was added. + + * kcplantdb.h (PlantDB::clear): handling the recovery flag was modified. + + - Release: 1.2.33 + +2011-01-03 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::read_fast): error check was added. + + * kclangc.cc (kcdbacceptbulk, kcdbsetbulk, kcdbremovebulk, kcdbgetbulk): new functions. + + - Release: 1.2.32 + +2010-12-30 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::accept_bulk): new function. + + * kcdb.h (BasicDB::set_bulk, BasicDB::remove_bulk, BasicDB::get_bulk): new functions. + + * kcthread.h (SlottedMutex::unlock_all): the order became forward. + + * kcpolymgr.cc (procsetbulk, procremovebulk, procgetbulk): new functions. + + - Release: 1.2.31 + +2010-12-19 FAL Labs <info@fallabs.com> + + * kcpolydb.h (PolyDB::set_internal_db): new function. + + - Release: 1.2.30 + +2010-12-11 FAL Labs <info@fallabs.com> + + * kccommon.h (modfl): new function for portability to cygwin. + + * kcthread.cc (Mutex::lock_try): cygwin was added to work around cases. + + * cmdcommon.h (oprintf): new function instead of "iprintf". + + - Release: 1.2.29 + +2010-12-10 FAL Labs <info@fallabs.com> + + * ktutil.h (atoin, atofn): new functions. + + - Release: 1.2.28 + +2010-11-30 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::write, File::read): warnings on 32-bit environment were cleared. + + * kcfile.cc (File::write_file, File::make_directory): modified for Win32 virus checkers. + + - Release: 1.2.27 + +2010-11-28 FAL Labs <info@fallabs.com> + + * ktutil.h (vstrprintf): solved a compilation warning. + + * kcdbext.h (MapReduce::~MapReduce): modified as a virtual function. + + - Release: 1.2.26 + +2010-11-17 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::MetaTrigger): new class. + + * kcdb.h (BasicDB::tune_trigger): new function. + + * kcpolydb.h (PolyDB::StreamMetaTrigger): new class. + + - Release: 1.2.25 + +2010-11-10 FAL Labs <info@fallabs.com> + + * kcstashdb.h, kcstashdb.o: new files. + + - Release: 1.2.24 + +2010-11-06 FAL Labs <info@fallabs.com> + + * kcutil.h (sizevarnum): new function. + + * kcmap.h (TinyHashMap): new class. + + * kcdbext.h (MapReduce::execute): the cache algorithm was modified. + + - Release: 1.2.23 + +2010-11-04 FAL Labs <info@fallabs.com> + + * kcutil.h (strsplit): new function. + + * kcpolydb.h (PolyDB::match_prefix): performance of the prototype tree was improved. + + - Release: 1.2.22 + +2010-11-02 FAL Labs <info@fallabs.com> + + * kcdbext.h (MapReduce::Emitter::emit): a bug on 32-bit environment was fixed. + + * kcdbext.h (MapReduce::execute): the naming convention of databases was modified. + + * kcregex.cc (Regex::compile): a bug related to error handling was fixed. + + - Release: 1.2.21 + +2010-10-29 FAL Labs <info@fallabs.com> + + * kcdbext.h, kcdbext.cc: new files. + + * kcpolytest.cc (runmapred, procmapred): new functions. + + - Release: 1.2.20 + +2010-10-21 FAL Labs <info@fallabs.com> + + * kcpolydb.h (PolyDB::match_prefix, PolyDB::match_regex): new functions. + + * kclangc.h (kcdbmatchprefix, kcdbmatchregex): new functions. + + * kcregex.h: new file. + + - Release: 1.2.19 + +2010-10-15 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB::accept_impl): hash chaining algorithm was modified. + + * kcpolymgr.cc (runclear, procclear): new functions. + + - Release: 1.2.18 + +2010-10-11 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::Error::NOREPOS): new constant instead of "NOENTRY". + + * configure.in: a building problem on Mac OS X was fixed. + + - Release: 1.2.17 + +2010-10-03 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::Error::NOENTRY): new constant instead of "NOFILE". + + * kchashdb. (HashDB::synchronize_impl): open flag handling was modified. + + - Release: 1.2.16 + +2010-10-01 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::open): how to set the recover flag was modified. + + * kcfile.cc (File::rename, File::remove): difference of Win32 was absorbed. + + * kchashdb.h (HashDB::defrag_impl): auto transaction is now supported. + + * kchashdb.h (HashDB::abort_transaction): a bug related to the open flag was fixed. + + - Release: 1.2.15 + +2010-09-23 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::tune_logger): new function. + + * kcdb.h (BasicDB::increment_double): new function. + + * kchashdb.h (HashDB::reorganize_file): copying method was modified. + + * kchashdb.h (HashDB::begin_transaction_impl): performance was improved. + + * kcplantdb.h (PlantDB::reorganize_file): copying method was modified. + + * kcdirdb.h (DirDB::calc_magic): broken files are now removed. + + - Release: 1.2.14 + +2010-09-13 FAL Labs <info@fallabs.com> + + * kcthread.h (TaskQueue): new class. + + * kcdb.h (DB::Cursor::set_value_str): renamed to solve ambiguity. + + - Release: 1.2.13 + +2010-09-08 FAL Labs <info@fallabs.com> + + * kcutil.h (kcurlencode, kcurldecode): new functions. + + * kcutil.h (kcquoteencode, kcquotedecode): new functions. + + * kcutil.h (kcbaseencode, kcbasedecode): new functions. + + * kcutil.h (memdup, strdup, strtoupper, strtolower, strtrim): new functions. + + * kclangc.h (kcmalloc, kcfree): the parameter type was modified. + + - Release: 1.2.12 + +2010-09-04 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::open, walwrite, walapply): mmap was discarded. + + * kcplantdb.h (PlantDB::recalc_count): performance was improved. + + * kcplantdb.h (PlantDB::fix_auto_transaction_leaf): performance was improved. + + - Release: 1.2.11 + +2010-09-01 FAL Labs <info@fallabs.com> + + * kcprotodb.h (ProtoDB::report_valist): new function. + + * kcplantdb.h (PlantDB::report_valist): new function. + + * kchashdb.h (HashDB::read_record_body): magic data checking was added. + + - Release: 1.2.10 + +2010-08-30 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB::replace): new function. + + * kcpolydb.h (PolyDB::merge): "MREPLACE" mode was added. + + * kchashdb.h (HashDB::synchronize_impl): readers became able to call it. + + * kcplantdb.h (PlantDB::synchronize): readers became able to call it. + + * kcpolymgr.cc (runcopy, proccopy): new functions. + + - Release: 1.2.9 + +2010-08-20 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::remove_recursively): new function. + + * kcdb.h (BasicDB::Logger): new class. + + * kcdb.h (BasicDB::ProgressChecker): new class. + + * kcfile.cc (File::remove, File::rename): timeout on Win32 was modified. + + * kcpolydb.h (PolyDB::open): "erstrm" and "ervbs" were replaced by "log" and "logvbs". + + * kcpolydb.h (PolyDB::merge): new function. + + * kcpolymgr.cc (runmerge, procmerge): new functions. + + - Release: 1.2.8 + +2010-08-18 FAL Labs <info@fallabs.com> + + * kcplantdb.h (PlantDB::rcomp): new function. + + * kctreemgr.cc (runlist, proclist): descending order was added. + + - Release: 1.2.7 + +2010-08-17 FAL Labs <info@fallabs.com> + + * kcthread.cc (CondVar): Win32 support was added. + + * kcdb.h (jump_back, step_back): new functions. + + * kcplantdb.h (jump_last, step_back): new functions. + + * kcutiltest.cc (runpara, procpara): parallel tests were added. + + - Release: 1.2.6 + +2010-08-07 FAL Labs <info@fallabs.com> + + * kccachedb.h (CacheDB::clear_slot): a bug related to transaction was fixed. + + * kchashdb.h (HashDB::reorganize_file): a bug of overflow was fixed. + + - Release: 1.2.5 + +2010-08-06 FAL Labs <info@fallabs.com> + + * kcutil.h (hashpath): variance was improved. + + * kccompress.h (LZO::calculate_crc, LZMA::calculate_crc): new functions. + + - Release: 1.2.4 + +2010-08-01 FAL Labs <info@fallabs.com> + + * kcutil.cc (getsysinfo): some information was added for Darwin and Win32. + + * kcutil.cc (setstdiobin): new function. + + * kccompress.h (LZO, LZMA): new classes. + + * kcfile.cc (File::remove, File::rename): modified for Win32 virus checkers. + + * kcpolydb.h (PolyDB::open): "zcomp=lzo" and "zcomp=lzma" were added. + + - Release: 1.2.3 + +2010-07-29 FAL Labs <info@fallabs.com> + + * kcutil.cc (getsysinfo): new function. + + * kccache.h (GrassDB): new class. + + * kccachetest.cc: new file. + + - Release: 1.2.2 + +2010-07-28 FAL Labs <info@fallabs.com> + + * kcompress.h (ArcfourComressor::ArcfourComressor): suppressed a warning. + + * kccompress.h (ArcfourCompressor::set_compressor): new function. + + * kcpolydb.h (PolyDB::open): "zcomp=arcz" was added. + + * Makefile.in: now deals with an internal bug of GCC 4.2.1 on Mac OS X. + + - Release: 1.2.1 + +2010-07-26 FAL Labs <info@fallabs.com> + + * kcdb.h (BasicDB): renamed from FileDB. + + * kcplantdb.h: new file. + + * kctreedb.h: removed. + + * kchashdb.h (TreeDB): re-defined as a type alias. + + * kcdirdb.h (ForestDB): new class as a type alias. + + - Release: 1.2.0 + +2010-07-24 FAL Labs <info@fallabs.com> + + * kcutil.h (arccipher): new function. + + * kccopmress.h (ArcfourCompressor): new class. + + * kchashdb.h (HashDB::open): synchronize came to be called when auto sync mode. + + * kcdirdb.h (DirDB::commit_transaction): synchronize came to be called. + + * kcdirmgr.cc (dbmetaprint): status information became detail. + + * kcpolydb.h (PolyDB::open): "zcomp" and "zkey" were added. + + - Release: 1.1.1 + +2010-07-06 FAL Labs <info@fallabs.com> + + * kcutil.h (hexencode): performance was improved. + + * kcfile.cc (read_file, write_file): new functions. + + * kcdirdb.h, kcdirdb.cc, kcdirtest.cc, kcdirmgr.cc: new files. + + - Release: 1.1.0 + +2010-07-04 FAL Labs <info@fallabs.com> + + * kcdb.h (typestring): refactored. + + - Release: 1.0.4 + +2010-06-19 FAL Labs <info@fallabs.com> + + * kcdb.h: useless virtual attributes of some functions were abolished. + + * kctreedb.h (synchronize): a bug related to a call back parameter was fixed. + + * kcpolytest.cc (procqueue): a bug related to a log message was fixed. + + - Release: 1.0.3 + +2010-06-05 FAL Labs <info@fallabs.com> + + * cmdcommon.h (mysrand): new function. + + * cmdcommon.h (myrand): algorithm was changed. + + - Release: 1.0.2 + +2010-05-25 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::open): error messages were added. + + * kcdb.h (DB::dump_snapshot, DB::load_snapshot): error codes were modified. + + * kchashdb.h (HashDB::open): error codes were modified. + + - Release: 1.0.1 + +2010-05-19 FAL Labs <info@fallabs.com> + + * kccommon.h: built-in macros with confliction were voided. + + * kclangc.cc (kcdbcopy, kccursetvalue): new functions. + + * kclangc.cc (kcatoi, kcatoix, kcatof): new functions. + + * kclang.cc (kcnan, kcinf, kcchknan, kcchkinf): new functions. + + - Release: 1.0.0 + +2010-05-06 FAL Labs <info@fallabs.com> + + * kccachetest.cc (procorder): a bug about record counting was fixed. + + - Release: 0.9.18 + +2010-05-02 FAL Labs <info@fallabs.com> + + * kcdb.h (FileProcessor::process): parameters for meta data was modified. + + - Release: 0.9.17 + +2010-04-28 FAL Labs <info@fallabs.com> + + * kccommon.h (_yield_, _testyield_, _assert_): new macros. + + * kcfile.h (File::get_current_directory, File::set_current_directory): new functions. + + - Release: 0.9.16 + +2010-04-21 FAL Labs <info@fallabs.com> + + * kcutil.h (inf, chknan, chkinf): new functions. + + * kcfile.h (File::rename, File::read_directory): new function. + + * kcdb.h (FileDB::copy): new function. + + * kccommon.h: FreeBSD and Solaris support was added. + + - Release: 0.9.15 + +2010-04-20 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::close): unlocking operation was added. + + * kcdb.h (FileProcessor::process): the offset parameter was abolished. + + - Release: 0.9.14 + +2010-04-15 FAL Labs <info@fallabs.com> + + * kcutil.h (nan): new function. + + * kcdb.h (File::DB::FileProcessor::process): parameters were added. + + * kclangc.cc (kchashmurmur, kchashfnv): new functions. + + * kcthread.cc: Win32 support was added. + + * kcfile.cc: Win32 support was added. + + * VCmakefile: new file. + + - Release: 0.9.13 + +2010-04-13 FAL Labs <info@fallabs.com> + + * kcdb.h (DB::typestring): a miscellaneous type was added. + + * kcpolydb.h (PolyDB::PolyDB): a constructor with a database object was added. + + * kcthread.cc (Mutex::lock_try): Mac OS X support was added. + + - Release: 0.9.12 + +2010-04-06 FAL Labs <info@fallabs.com> + + * kccommon.h: C++0x check was added. + + * kcdb.h (DB::dump_snapshot, DB::load_snapshot): new functions. + + * kcdb.h (DB::Cursor::set_value): new function. + + * kchashmgr.c (procdump, procload): new functions. + + * kctreemgr.c (procdump, procload): new functions. + + * kclangc.cc (kcmalloc, kcdbdumpsnap, kcdbloadsnap): new functions. + + - Release: 0.9.11 + +2010-04-01 FAL Labs <info@fallabs.com> + + * kcdb.h (DB::accept, DB::iterate): default parameters were added. + + * kcdb.h (DB::Cursor::accept, DB::Cursor::get): default parameters were added. + + - Release: 0.9.10 + +2010-03-28 FAL Labs <info@fallabs.com> + + * kcprotodb.h (ProtoDB::iterate): calls of empty visiting were abolished. + + * kchashdb.h (HashDB::iterate): calls of empty visiting were abolished. + + - Release: 0.9.9 + +2010-03-18 FAL Labs <info@fallabs.com> + + * kcdb.h (Error::operator_int32_t): new function. + + * kcdb.h (FileDB::begin_transaction_try): new function. + + * kcpolydb.h (PolyDB::Cursor::db): the type of the return value was modified. + + * kcpolydb.h (PolyDB::open): the erstrm option was added. + + * kcpolydb.h (PolyDB::Cursor::accept): life checking was added. + + * kcpolytest (runmisc, procmisc): new functions. + + - Release: 0.9.8 + +2010-03-14 FAL Labs <info@fallabs.com> + + * kclangc.h, kclangc.cc, kclangctest.c: new files. + + - Release: 0.9.7 + +2010-03-13 FAL Labs <info@fallabs.com> + + * kctree.h (TreeDB::Cursor::accept_atom): validation check was added. + + * kctree.h (TreeDB::escape_cursors): new function. + + - Release: 0.9.6 + +2010-03-09 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::recovered): new function. + + * kchash.h (HashDB::recovered, HashDB::reorganized): new function. + + * kctree.h (TreeDB::Cursor::accept_spec, TreeDB::Cursor::accept_atom): new functions. + + - Release: 0.9.5 + +2010-03-06 FAL Labs <info@fallabs.com> + + * kctree.h (TreeDB::sub_link_tree): new function. + + * kctree.h (TreeDB::reorganize_tree): tree trimming mechanism was added. + + - Release: 0.9.4 + +2010-03-04 FAL Labs <info@fallabs.com> + + * kctree.h (TreeDB::Cursor::accept): a bug related drifted cursor was fixed. + + * kchashtest.cc (runqueue, procqueue): new functions. + + * kctreetest.cc (runqueue, procqueue): new functions. + + - Release: 0.9.3 + +2010-03-03 FAL Labs <info@fallabs.com> + + * kcdb.h (DB::Error::string): abolished. + + * kcdb.h (DB::Error::name, DB::Error::codename): new functions. + + - Release: 0.9.2 + +2010-03-01 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB::Curosr::jump): a bug of boundary checking was fixed. + + * kchashdb.h (HashDB::report): new function. + + * kcpolydb.h, kcpolydb.cc: new files. + + - Release: 0.9.1 + +2010-02-16 FAL Labs <info@fallabs.com> + + * kctreedb.h (TreeDB::accept): auto transaction was added. + + * kctreedb.h (TreeDB::Cursor::accept): auto transaction was added. + + * kctreedb.h (TreeDB::fix_auto_transaction_tree): new function. + + - Release: 0.9.0 + +2010-02-16 FAL Labs <info@fallabs.com> + + * kcthread.h (Thread::hash): new function. + + * kchashdb.h (HashDB::accept): lock mechanism was modified to promotion locking. + + * kchashdb.h (HashDB::adjust_record): a bug of boundary checking was fixed. + + * kchashdb.h (HashDB::read_record): validation checking of size was added. + + * kchashdb.h (HashDB::dfunit): new function. + + - Release: 0.5.19 + +2010-02-13 FAL Labs <info@fallabs.com> + + * kcdb.h (DB::get): new function. + + * kccompress.h (ZlibRawCompressor): new class. + + * kccompare.h, kccompare.cc: new files. + + * kchashdb.h (HashDB::escape_cursors): end sentry escaping was added. + + - Release: 0.5.18 + +2010-02-11 FAL Labs <info@fallabs.com> + + * kcdb.h (Cursor::get, Cursor::get_pair): new functions. + + * kchashmgr.c (runcreate, runinform): "-onr" option was added. + + - Release: 0.5.17 + +2010-02-08 FAL Labs <info@fallabs.com> + + * kcdb.h (typestring): new function. + + * kcfile.c (File::refresh): new function. + + * kctreedb.h, kctreedb.cc, kctreetest.cc, kctreemgr.cc: new files. + + * kchashdb.h (HashDB::synchronize_opaque, HashDB::dump_opaque): new functions. + + * kchashdb.h (HashDB::reorganize_file): new functions. + + - Release: 0.5.16 + +2010-02-07 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB::abort_auto_transaction): new function. + + * kchashdb.h (HashDB::accept_impl): a bug of race condition was fixed. + + - Release: 0.5.15 + +2010-02-04 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::begin_transaction, File::end_transaction): mmap is now used. + + * kcfile.cc (File::write, File::write_fast): refactored. + + * kchashtest.cc (procorder): a bug of integer overflow was fixed. + + - Release: 0.5.14 + +2010-02-03 FAL Labs <info@fallabs.com> + + * kcfile.h (end_transaction, walapply): performance was improved. + + * kchashdb.h (HashDB::commit_auto_transaction, HashDB::dump_auto_meta): new functions. + + * kcutiltest.cc, kcprototest.cc, kccachetest.cc, kchashtest.cc: refactored for strace. + + - Release: 0.5.13 + +2010-02-02 FAL Labs <info@fallabs.com> + + * kcdb.h (CacheDB::open): OAUTOTRAN and OAUTOSYNC flags was added. + + * kchashdb.h (HashDB::open): OAUTOTRAN and OAUTOSYNC flags was added. + + * kchashdb.h (HashDB::calc_checksum): shuffling algo was modified. + + * kcfile.cc (walapply): a bug of numeric overflow was fixed. + + - Release: 0.5.12 + +2010-02-01 FAL Labs <info@fallabs.com> + + * kccachedb.h (CacheDB::accept_impl): removing algo was modified. + + * kccachedb.h (CacheDB::fold_hash): shuffling algo was modified. + + * kchashdb.h (HashDB::cut_chain): removing algo was modified. + + * kchashdb.h (HashDB::fold_hash): shuffling algo was modified. + + * kcthread.h (PromotiveLock): abolished. + + * kcthread.h (SpinRWLock::promote, SpinRWLock::demote): new functions. + + - Release: 0.5.11 + +2010-01-29 FAL Labs <info@fallabs.com> + + * kchashtest.c (runorder, procorder): a transaction parameter was added. + + - Release: 0.5.10 + +2010-01-26 FAL Labs <info@fallabs.com> + + * kcthread.h (SpinRWLock, ScopedSpinRWLock, SlottedSpinRWLock): new classes. + + * kcproto.h (ProtoDB): lock primitives were replaced by spin locks. + + * kccachedb.h (CacheDB): lock primitives were replaced by spin locks. + + * kchashdb.h (HashDB): lock primitives were replaced by spin locks. + + - Release: 0.5.9 + +2010-01-18 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB::Cursor::accept) a bug of iterator increment was fixed. + + * kcprotodb.h (ProtoDB::accept): the structure of the transaction log was fixed. + + * kccachedb.h, kccachedb.cc, kccachetest.cc: new files. + + - Release: 0.5.8 + +2010-01-14 FAL Labs <info@fallabs.com> + + * kcprotodb.h (ProtoDB): rewritten as a template class. + + * kchashdb.h (HashDB::dump_free_blocks): a buf related to addressing was fixed. + + * kcprototest.cc: new file. + + - Release: 0.5.7 + +2010-01-05 FAL Labs <info@fallabs.com> + + * kcfile.cc (File::end_transaction): a bug of memory management vioration was fixed. + + - Release: 0.5.6 + +2010-01-05 FAL Labs <info@fallabs.com> + + * kcfile.cc (walpath, walwrite, walapply, mywrite, myread): new functions. + + * kcfile.cc (File::begin_transaction, File::end_transaction): new functions. + + - Release: 0.5.5 + +2010-01-04 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB::trim_cursors, HashDB::disable_cursors): new functions. + + * kchashdb.h (HashDB::tune_defrag, HashDB::defrag, HashDB::defrag_impl): new functions. + + * kchashmgr.cc (rundefrag, procdefrag): new functions. + + - Release: 0.5.4 + +2010-01-03 FAL Labs <info@fallabs.com> + + * kcutil.h (nearbyprime): performance was improved. + + * kcfile.cc (File::expand): abolished. + + * kcfile.cc (File::write): a bug of race condition was fixed. + + - Release: 0.5.3 + +2010-01-01 FAL Labs <info@fallabs.com> + + * kchashdb.h (HashDB::clear): implemented. + + * cmdcommon.h: new file. + + - Release: 0.5.2 + +2009-12-27 FAL Labs <info@fallabs.com> + + * kcprotodb.h (ProtoDB::accept): a bug of race condition was fixed. + + * kchashdb.h (HashDB::calc_checksum): new function. + + * kchashdb.h (HashDB::dump_meta, HashDB::load_meta): the data format was modified. + + * kchashdb.h (HashDB::accept_impl): a bug of race condition was fixed. + + * kcfile.cc (File::write): a bug of border vioration was fixed. + + - Release: 0.5.1 + +2009-12-25 FAL Labs <info@fallabs.com> + + - The initial version. + + - Release: 0.5.0 + diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/Doxyfile b/plugins/Dbx_kyoto/src/kyotocabinet/Doxyfile new file mode 100644 index 0000000000..1ed5b75feb --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/Doxyfile @@ -0,0 +1,70 @@ +# Doxyfile for Kyoto Cabinet + + +# General configuration options +PROJECT_NAME = "Kyoto Cabinet" +OUTPUT_LANGUAGE = English +EXTRACT_ALL = NO +EXTRACT_PRIVATE = NO +EXTRACT_STATIC = YES +REPEAT_BRIEF = YES +ALWAYS_DETAILED_SEC = YES +SHOW_INCLUDE_FILES = YES +VERBATIM_HEADERS = NO +JAVADOC_AUTOBRIEF = YES +SORT_MEMBER_DOCS = NO +INLINE_INFO = NO +OPTIMIZE_OUTPUT_FOR_C = NO +OPTIMIZE_OUTPUT_JAVA = NO +SHOW_USED_FILES = NO +QUIET = YES +WARNINGS = YES +SEARCHENGINE = NO + + +# Configuration options related to the input files +INPUT = . +FILE_PATTERNS = overview kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h \ + kccompress.h kccompare.h kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \ + kcpolydb.h kcdbext.h kclangc.h +RECURSIVE = NO + + +# Configuration options related to the alphabetical index +ALPHABETICAL_INDEX = YES +COLS_IN_ALPHA_INDEX = 3 + + +# Configuration options related to the HTML output +GENERATE_HTML = YES +HTML_OUTPUT = doc/api +HTML_FILE_EXTENSION = .html +GENERATE_TREEVIEW = NO +TREEVIEW_WIDTH = 250 + + +# Configuration options related to the LaTeX output +GENERATE_LATEX = NO +LATEX_OUTPUT = latex + + +# Configuration options related to the man page output +GENERATE_MAN = NO +MAN_OUTPUT = . +MAN_EXTENSION = .3 + + +# Configuration options related to the dot tool +HAVE_DOT = NO +CLASS_GRAPH = NO +COLLABORATION_GRAPH = NO +INCLUDE_GRAPH = NO +INCLUDED_BY_GRAPH = NO +GRAPHICAL_HIERARCHY = YES +GENERATE_LEGEND = NO +DOT_CLEANUP = YES + + + +# END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/FOSSEXCEPTION b/plugins/Dbx_kyoto/src/kyotocabinet/FOSSEXCEPTION new file mode 100644 index 0000000000..23b2396a6a --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/FOSSEXCEPTION @@ -0,0 +1,78 @@ + Kyoto Products FOSS License Exception + Version 1.0, 9 March 2011 + +This Free and Open Source Software ("FOSS") License Exception allows +developers of FOSS applications to include Kyoto Products with their FOSS +applications. Kyoto Products are licensed pursuant to version 3 (or later) +of the GNU General Public License ("GPL"), but this exception permits +distribution of Kyoto Products with a developer's FOSS applications licensed +under the terms of another FOSS license listed below, even though such other +FOSS license may be incompatible with the GPL. + +The following terms and conditions describe the circumstances under which +this FOSS License Exception applies. + + FOSS License Exception Terms and Conditions + +1. Definitions + + "Derivative Work" means a derivative work, as defined under applicable + copyright law, formed entirely from the Program and one or more FOSS + Applications. + + "FOSS Application" means a free and open source software application + distributed subject to a license listed in the section below titled + "FOSS License List." + + "FOSS Notice" means a notice placed by FAL Labs in a copy of the Kyoto + Products Libraries stating that such copy of the Kyoto Products + Libraries may be distributed under FAL Lab's or Kyoto Products' FOSS + License Exception. + + "Independent Work" means portions of the Derivative Work that are not + derived from the Program and can reasonably be considered independent + and separate works. + + "Program" means a copy of FAL Labs' Kyoto Products Libraries that + contains a FOSS Notice. + +2. A FOSS application developer ("you" or "your") may distribute a Derivative + Work provided that you and the Derivative Work meet all of the following + conditions: + + a. You obey the GPL in all respects for the Program and all portions + (including modifications) of the Program included in the Derivative + Work (provided that this condition does not apply to Independent Works); + + b. You distribute Independent Works subject to a license listed in the + section below titled "FOSS License List"; + + c. You distribute Independent Works in object code or executable form + with the complete corresponding machine-readable source code on the + same medium and under the same FOSS license applying to the object + code or executable forms; + + d. All works that are aggregated with the Program or the Derivative Work + on a medium or volume of storage are not derivative works of the + Program, Derivative Work or FOSS Application, and must reasonably be + considered independent and separate works. + +3. FAL Labs reserves all rights not expressly granted in these terms and + onditions. If all of the above conditions are not met, then this FOSS + License Exception does not apply to you or your Derivative Work. + + FOSS License List + + License Name Versions/Copyright Date + + Apache Software License 1.0 / 1.1 / 2.0 + Artistic license From Perl 5.8.0 + BSD License "July 22 1999" + GNU General Public License (GPL) 2 + GNU Lesser General Public License (LGPL) 2.1 / 3.0 + Mozilla Public License (MPL) 1.0 / 1.1 + OpenSSL license (with original SSLeay license) "2003" ("1998") + PHP License 3.0/3.01 + Python Software Foundation License 2.1.1 + X11 License "2001" + Zlib/libpng License - diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/LINKEXCEPTION b/plugins/Dbx_kyoto/src/kyotocabinet/LINKEXCEPTION new file mode 100644 index 0000000000..ea30129154 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/LINKEXCEPTION @@ -0,0 +1,57 @@ + Kyoto Products Specific FOSS Library Linking Exception + Version 1.0, 10 June 2011 + +This Specific FOSS (Free and Open Source Software) Library Linking Exception +allows applications of some specific FOSS libraries to link to Kyoto Products +without the "copyleft" duty defined by the GNU General Public License ("GPL"). +Kyoto Products are licensed pursuant to version 3 (or later) of the GPL, but +this exception permits distribution of Kyoto Products with applications of +pecific FOSS libraries listed below, even though their license may be +imcompativle with the GPL. + +The following terms and conditions describe the circumstances under which +this Specific FOSS Library Linking Exception. + + Specific FOSS Library Linking Exception Terms and Conditions + +1. Definitions + + "Derivative Work" means a derivative work, as defined under applicable + copyright law, formed entirely from the Program and the Specific Library + and the Application. + + "Application" means an application under an arbitrary license, which may + or may not be a FOSS license, linking to the Specific Library. + + "Program" means a copy of FAL Labs' Kyoto Products Libraries. + + "Specific Library" means one of the libraries listed in the section below + titled "Specific Library List". + +2. A developer ("you" or "your") may distribute a Derivative Work under an + arbitrary license, provided that you and the Derivative Work and the + Specific Library linked to by the Derivative Work meet all of the + following conditions: + + a. You don't modify the Program at all. + + b. You use the Program as a backend of the Specific Library and you don't + use any interface of the Program for other purposes except for + utilizing the Specific Library. + + c. The Specific Library is a FOSS product whose license complies to + the Open Source Definition defined by the Open Source Initiative. + + d. The Specific Library does not allow applications to use interfaces of + the Product as a general database functions. + +3. FAL Labs reserves all rights not expressly granted in these terms and + conditions. If all of the above conditions are not met, then this + Specific FOSS Library Linking Exception does not apply to you or your + Derivative Work. + + Specific Library List + + Name: GraphLab + Description: a parallel framework for machine learning + URL: http://www.graphlab.ml.cmu.edu/ diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/Makefile.in b/plugins/Dbx_kyoto/src/kyotocabinet/Makefile.in new file mode 100644 index 0000000000..be4bdbb46b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/Makefile.in @@ -0,0 +1,1252 @@ +# Makefile for Kyoto Cabinet + + + +#================================================================ +# Setting Variables +#================================================================ + + +# Generic settings +SHELL = @SHELL@ + +# Package information +PACKAGE = @PACKAGE_NAME@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +VERSION = @PACKAGE_VERSION@ +PACKAGEDIR = $(PACKAGE)-$(VERSION) +PACKAGETGZ = $(PACKAGE)-$(VERSION).tar.gz +LIBVER = @MYLIBVER@ +LIBREV = @MYLIBREV@ +FORMATVER = @MYFORMATVER@ + +# Targets +HEADERFILES = @MYHEADERFILES@ +LIBRARYFILES = @MYLIBRARYFILES@ +LIBOBJFILES = @MYLIBOBJFILES@ +COMMANDFILES = @MYCOMMANDFILES@ +MAN1FILES = @MYMAN1FILES@ +DOCUMENTFILES = @MYDOCUMENTFILES@ +PCFILES = @MYPCFILES@ + +# Install destinations +prefix = @prefix@ +exec_prefix = @exec_prefix@ +datarootdir = @datarootdir@ +INCLUDEDIR = @includedir@ +LIBDIR = @libdir@ +BINDIR = @bindir@ +LIBEXECDIR = @libexecdir@ +DATADIR = @datadir@/$(PACKAGE) +MAN1DIR = @mandir@/man1 +DOCDIR = @docdir@ +PCDIR = @libdir@/pkgconfig +DESTDIR = + +# Building configuration +CC = @CC@ +CXX = @CXX@ +CPPFLAGS = @MYCPPFLAGS@ \ + -D_KC_PREFIX="\"$(prefix)\"" -D_KC_INCLUDEDIR="\"$(INCLUDEDIR)\"" \ + -D_KC_LIBDIR="\"$(LIBDIR)\"" -D_KC_BINDIR="\"$(BINDIR)\"" -D_KC_LIBEXECDIR="\"$(LIBEXECDIR)\"" \ + -D_KC_APPINC="\"-I$(INCLUDEDIR)\"" -D_KC_APPLIBS="\"-L$(LIBDIR) -lkyotocabinet @LIBS@\"" +CFLAGS = @MYCFLAGS@ +CXXFLAGS = @MYCXXFLAGS@ +LDFLAGS = @MYLDFLAGS@ +CMDLDFLAGS = @MYCMDLDFLAGS@ +CMDLIBS = @MYCMDLIBS@ +LIBS = @LIBS@ +RUNENV = @MYLDLIBPATHENV@=@MYLDLIBPATH@ +POSTCMD = @MYPOSTCMD@ + + + +#================================================================ +# Suffix rules +#================================================================ + + +.SUFFIXES : +.SUFFIXES : .c .cc .o + +.c.o : + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< + +.cc.o : + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< + + + +#================================================================ +# Actions +#================================================================ + + +all : $(LIBRARYFILES) $(COMMANDFILES) + @$(POSTCMD) + @printf '\n' + @printf '#================================================================\n' + @printf '# Ready to install.\n' + @printf '#================================================================\n' + + +clean : + rm -rf $(LIBRARYFILES) $(LIBOBJFILES) $(COMMANDFILES) $(CGIFILES) \ + *.o *.gch a.out check.in check.out gmon.out *.log *.vlog words.tsv \ + casket* *.kch *.kct *.kcd *.kcf *.wal *.tmpkc* *.kcss *~ hoge moge tako ika + + +version : + sed -e 's/_KC_VERSION.*/_KC_VERSION "$(VERSION)"/' \ + -e "s/_KC_LIBVER.*/_KC_LIBVER $(LIBVER)/" \ + -e "s/_KC_LIBREV.*/_KC_LIBREV $(LIBREV)/" \ + -e 's/_KC_FMTVER.*/_KC_FMTVER $(FORMATVER)/' myconf.h > myconf.h~ + [ -f myconf.h~ ] && mv -f myconf.h~ myconf.h + + +untabify : + ls *.cc *.h *.idl | while read name ; \ + do \ + sed -e 's/\t/ /g' -e 's/ *$$//' $$name > $$name~; \ + [ -f $$name~ ] && mv -f $$name~ $$name ; \ + done + + +install : + mkdir -p $(DESTDIR)$(INCLUDEDIR) + cp -Rf $(HEADERFILES) $(DESTDIR)$(INCLUDEDIR) + mkdir -p $(DESTDIR)$(LIBDIR) + cp -Rf $(LIBRARYFILES) $(DESTDIR)$(LIBDIR) + mkdir -p $(DESTDIR)$(BINDIR) + cp -Rf $(COMMANDFILES) $(DESTDIR)$(BINDIR) + mkdir -p $(DESTDIR)$(MAN1DIR) + cd man && cp -Rf $(MAN1FILES) $(DESTDIR)$(MAN1DIR) + mkdir -p $(DESTDIR)$(DOCDIR) + cp -Rf $(DOCUMENTFILES) $(DESTDIR)$(DOCDIR) + mkdir -p $(DESTDIR)$(PCDIR) + cp -Rf $(PCFILES) $(DESTDIR)$(PCDIR) + @printf '\n' + @printf '#================================================================\n' + @printf '# Thanks for using Kyoto Cabinet.\n' + @printf '#================================================================\n' + + +install-strip : + $(MAKE) DESTDIR=$(DESTDIR) install + cd $(DESTDIR)$(BINDIR) && strip $(COMMANDFILES) + + +uninstall : + -cd $(DESTDIR)$(INCLUDEDIR) && rm -f $(HEADERFILES) + -cd $(DESTDIR)$(LIBDIR) && rm -f $(LIBRARYFILES) + -cd $(DESTDIR)$(BINDIR) && rm -f $(COMMANDFILES) + -cd $(DESTDIR)$(MAN1DIR) && rm -f $(MAN1FILES) + -cd $(DESTDIR)$(DOCDIR) && rm -rf $(DOCUMENTFILES) && rmdir $(DOCDIR) + -cd $(DESTDIR)$(PCDIR) && rm -f $(PCFILES) + + +dist : + $(MAKE) version + $(MAKE) untabify + $(MAKE) distclean + cd .. && tar cvf - $(PACKAGEDIR) | gzip -c > $(PACKAGETGZ) + sync ; sync + + +distclean : clean + cd example && $(MAKE) clean + rm -rf Makefile kyotocabinet.pc \ + config.cache config.log config.status config.tmp autom4te.cache + + +check : + $(MAKE) check-util + $(MAKE) check-proto + $(MAKE) check-stash + $(MAKE) check-cache + $(MAKE) check-grass + $(MAKE) check-hash + $(MAKE) check-tree + $(MAKE) check-dir + $(MAKE) check-forest + $(MAKE) check-poly + $(MAKE) check-langc + rm -rf casket* + @printf '\n' + @printf '#================================================================\n' + @printf '# Checking completed.\n' + @printf '#================================================================\n' + + +check-util : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcutilmgr version + $(RUNENV) $(RUNCMD) ./kcutilmgr hex Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr hex -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr enc Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr enc -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr enc -hex Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr enc -hex -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr enc -url Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr enc -url -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr enc -quote Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr enc -quote -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr ciph -key "hoge" Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr ciph -key "hoge" check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr comp -gz Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr comp -gz -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr comp -lzo Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr comp -lzo -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr comp -lzma Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr comp -lzma -d check.in > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr hash Makefile > check.in + $(RUNENV) $(RUNCMD) ./kcutilmgr hash -fnv Makefile > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr hash -path Makefile > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr regex mikio Makefile > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr regex -alt "hirarin" mikio Makefile > check.out + $(RUNENV) $(RUNCMD) ./kcutilmgr conf + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcutiltest mutex -th 4 -iv -1 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest cond -th 4 -iv -1 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest para -th 4 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest para -th 4 -iv -1 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest file -th 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest file -th 4 -rnd -msiz 1m casket 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest lhmap -bnum 1000 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest lhmap -rnd -bnum 1000 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest thmap -bnum 1000 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest thmap -rnd -bnum 1000 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest talist 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest talist -rnd 10000 + $(RUNENV) $(RUNCMD) ./kcutiltest misc 10000 + + +check-proto : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcprototest order -etc 10000 + $(RUNENV) $(RUNCMD) ./kcprototest order -th 4 10000 + $(RUNENV) $(RUNCMD) ./kcprototest order -th 4 -rnd -etc 10000 + $(RUNENV) $(RUNCMD) ./kcprototest order -th 4 -rnd -etc -tran 10000 + $(RUNENV) $(RUNCMD) ./kcprototest wicked 10000 + $(RUNENV) $(RUNCMD) ./kcprototest wicked -th 4 -it 4 10000 + $(RUNENV) $(RUNCMD) ./kcprototest tran 10000 + $(RUNENV) $(RUNCMD) ./kcprototest tran -th 2 -it 4 10000 + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcprototest order -tree -etc 10000 + $(RUNENV) $(RUNCMD) ./kcprototest order -tree -th 4 10000 + $(RUNENV) $(RUNCMD) ./kcprototest order -tree -th 4 -rnd -etc 10000 + $(RUNENV) $(RUNCMD) ./kcprototest order -tree -th 4 -rnd -etc -tran 10000 + $(RUNENV) $(RUNCMD) ./kcprototest wicked -tree 10000 + $(RUNENV) $(RUNCMD) ./kcprototest wicked -tree -th 4 -it 4 10000 + $(RUNENV) $(RUNCMD) ./kcprototest tran -tree 10000 + $(RUNENV) $(RUNCMD) ./kcprototest tran -tree -th 2 -it 4 10000 + + +check-stash : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcstashtest order -etc -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest order -th 4 -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest order -th 4 -rnd -etc -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest order -th 4 -rnd -etc -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest order -th 4 -rnd -etc -tran \ + -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest wicked -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest wicked -th 4 -it 4 -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest tran -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcstashtest tran -th 2 -it 4 -bnum 5000 10000 + + +check-cache : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kccachetest order -etc -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest order -th 4 -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest order -th 4 -rnd -etc -bnum 5000 -capcnt 10000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest order -th 4 -rnd -etc -bnum 5000 -capsiz 10000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest order -th 4 -rnd -etc -tran \ + -tc -bnum 5000 -capcnt 10000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest wicked -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest wicked -th 4 -it 4 -tc -bnum 5000 -capcnt 10000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest tran -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kccachetest tran -th 2 -it 4 -tc -bnum 5000 10000 + + +check-grass : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcgrasstest order -etc -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcgrasstest order -th 4 -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcgrasstest order -th 4 -rnd -etc -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcgrasstest order -th 4 -rnd -etc -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcgrasstest order -th 4 -rnd -etc -tran \ + -tc -bnum 5000 -pccap 10k -rcd 500 + $(RUNENV) $(RUNCMD) ./kcgrasstest wicked -bnum 5000 10000 + $(RUNENV) $(RUNCMD) ./kcgrasstest wicked -th 4 -it 4 -tc -bnum 5000 -pccap 10k -rcd 1000 + $(RUNENV) $(RUNCMD) ./kcgrasstest tran -bnum 500 10000 + $(RUNENV) $(RUNCMD) ./kcgrasstest tran -th 2 -it 4 -tc -bnum 5000 -pccap 10k -rcd 5000 + + +check-hash : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kchashmgr create -otr -apow 1 -fpow 2 -bnum 3 casket + $(RUNENV) $(RUNCMD) ./kchashmgr inform -st casket + $(RUNENV) $(RUNCMD) ./kchashmgr set -add casket duffy 1231 + $(RUNENV) $(RUNCMD) ./kchashmgr set -add casket micky 0101 + $(RUNENV) $(RUNCMD) ./kchashmgr set casket fal 1007 + $(RUNENV) $(RUNCMD) ./kchashmgr set casket mikio 0211 + $(RUNENV) $(RUNCMD) ./kchashmgr set casket natsuki 0810 + $(RUNENV) $(RUNCMD) ./kchashmgr set casket micky "" + $(RUNENV) $(RUNCMD) ./kchashmgr set -app casket duffy kukuku + $(RUNENV) $(RUNCMD) ./kchashmgr remove casket micky + $(RUNENV) $(RUNCMD) ./kchashmgr list -pv casket > check.out + $(RUNENV) $(RUNCMD) ./kchashmgr set casket ryu 1 + $(RUNENV) $(RUNCMD) ./kchashmgr set casket ken 2 + $(RUNENV) $(RUNCMD) ./kchashmgr remove casket duffy + $(RUNENV) $(RUNCMD) ./kchashmgr set casket ryu syo-ryu-ken + $(RUNENV) $(RUNCMD) ./kchashmgr set casket ken tatsumaki-senpu-kyaku + $(RUNENV) $(RUNCMD) ./kchashmgr set -inci casket int 1234 + $(RUNENV) $(RUNCMD) ./kchashmgr set -inci casket int 5678 + $(RUNENV) $(RUNCMD) ./kchashmgr set -incd casket double 1234.5678 + $(RUNENV) $(RUNCMD) ./kchashmgr set -incd casket double 8765.4321 + $(RUNENV) $(RUNCMD) ./kchashmgr get casket mikio + $(RUNENV) $(RUNCMD) ./kchashmgr get casket ryu + $(RUNENV) $(RUNCMD) ./kchashmgr import casket lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kchashmgr list -pv -px casket > check.out + $(RUNENV) $(RUNCMD) ./kchashmgr copy casket casket-para + $(RUNENV) $(RUNCMD) ./kchashmgr dump casket check.out + $(RUNENV) $(RUNCMD) ./kchashmgr load -otr casket check.out + $(RUNENV) $(RUNCMD) ./kchashmgr defrag -onl casket + $(RUNENV) $(RUNCMD) ./kchashmgr setbulk casket aa aaa bb bbb cc ccc dd ddd + $(RUNENV) $(RUNCMD) ./kchashmgr removebulk casket aa bb zz + $(RUNENV) $(RUNCMD) ./kchashmgr getbulk casket aa bb cc dd + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashmgr inform -st casket + $(RUNENV) $(RUNCMD) ./kchashmgr create -otr -otl -onr -apow 1 -fpow 3 \ + -ts -tl -tc -bnum 1 casket + $(RUNENV) $(RUNCMD) ./kchashmgr import casket < lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kchashmgr set casket mikio kyotocabinet + $(RUNENV) $(RUNCMD) ./kchashmgr set -app casket tako ikaunini + $(RUNENV) $(RUNCMD) ./kchashmgr set -app casket mikio kyototyrant + $(RUNENV) $(RUNCMD) ./kchashmgr set -app casket mikio kyotodystopia + $(RUNENV) $(RUNCMD) ./kchashmgr get -px casket mikio > check.out + $(RUNENV) $(RUNCMD) ./kchashmgr list casket > check.out + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashmgr clear casket + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kchashtest order -set -bnum 5000 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest order -get -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest order -getw -msiz 5000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest order -rem -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest order -bnum 5000 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest order -etc \ + -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest order -th 4 \ + -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest order -th 4 -rnd -etc \ + -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest order -th 4 -rnd -etc -tran \ + -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest order -th 4 -rnd -etc -oat \ + -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest order -th 4 -rnd -etc \ + -apow 2 -fpow 3 -ts -tl -tc -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest queue \ + -bnum 5000 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest queue -rnd \ + -bnum 5000 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest queue -th 4 -it 4 \ + -bnum 5000 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest queue -th 4 -it 4 -rnd \ + -bnum 5000 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest wicked -bnum 5000 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest wicked -th 4 -it 4 \ + -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest wicked -th 4 -it 4 -oat \ + -bnum 5000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest wicked -th 4 -it 4 \ + -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kchashtest tran casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest tran -th 2 -it 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kchashtest tran -th 2 -it 4 \ + -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 casket 10000 + + +check-tree : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kctreemgr create -otr -apow 1 -fpow 2 -bnum 3 casket + $(RUNENV) $(RUNCMD) ./kctreemgr inform -st casket + $(RUNENV) $(RUNCMD) ./kctreemgr set -add casket duffy 1231 + $(RUNENV) $(RUNCMD) ./kctreemgr set -add casket micky 0101 + $(RUNENV) $(RUNCMD) ./kctreemgr set casket fal 1007 + $(RUNENV) $(RUNCMD) ./kctreemgr set casket mikio 0211 + $(RUNENV) $(RUNCMD) ./kctreemgr set casket natsuki 0810 + $(RUNENV) $(RUNCMD) ./kctreemgr set casket micky "" + $(RUNENV) $(RUNCMD) ./kctreemgr set -app casket duffy kukuku + $(RUNENV) $(RUNCMD) ./kctreemgr remove casket micky + $(RUNENV) $(RUNCMD) ./kctreemgr list -pv casket > check.out + $(RUNENV) $(RUNCMD) ./kctreemgr set casket ryu 1 + $(RUNENV) $(RUNCMD) ./kctreemgr set casket ken 2 + $(RUNENV) $(RUNCMD) ./kctreemgr remove casket duffy + $(RUNENV) $(RUNCMD) ./kctreemgr set casket ryu syo-ryu-ken + $(RUNENV) $(RUNCMD) ./kctreemgr set casket ken tatsumaki-senpu-kyaku + $(RUNENV) $(RUNCMD) ./kctreemgr set -inci casket int 1234 + $(RUNENV) $(RUNCMD) ./kctreemgr set -inci casket int 5678 + $(RUNENV) $(RUNCMD) ./kctreemgr set -incd casket double 1234.5678 + $(RUNENV) $(RUNCMD) ./kctreemgr set -incd casket double 8765.4321 + $(RUNENV) $(RUNCMD) ./kctreemgr get casket mikio + $(RUNENV) $(RUNCMD) ./kctreemgr get casket ryu + $(RUNENV) $(RUNCMD) ./kctreemgr import casket lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kctreemgr list -des -pv -px casket > check.out + $(RUNENV) $(RUNCMD) ./kctreemgr copy casket casket-para + $(RUNENV) $(RUNCMD) ./kctreemgr dump casket check.out + $(RUNENV) $(RUNCMD) ./kctreemgr load -otr casket check.out + $(RUNENV) $(RUNCMD) ./kctreemgr defrag -onl casket + $(RUNENV) $(RUNCMD) ./kctreemgr setbulk casket aa aaa bb bbb cc ccc dd ddd + $(RUNENV) $(RUNCMD) ./kctreemgr removebulk casket aa bb zz + $(RUNENV) $(RUNCMD) ./kctreemgr getbulk casket aa bb cc dd + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreemgr inform -st casket + $(RUNENV) $(RUNCMD) ./kctreemgr create -otr -otl -onr -apow 1 -fpow 3 \ + -ts -tl -tc -bnum 1 casket + $(RUNENV) $(RUNCMD) ./kctreemgr import casket < lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kctreemgr set casket mikio kyotocabinet + $(RUNENV) $(RUNCMD) ./kctreemgr set -app casket tako ikaunini + $(RUNENV) $(RUNCMD) ./kctreemgr set -app casket mikio kyototyrant + $(RUNENV) $(RUNCMD) ./kctreemgr set -app casket mikio kyotodystopia + $(RUNENV) $(RUNCMD) ./kctreemgr get -px casket mikio > check.out + $(RUNENV) $(RUNCMD) ./kctreemgr list casket > check.out + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreemgr clear casket + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kctreetest order -set \ + -psiz 100 -bnum 5000 -msiz 50000 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest order -get \ + -msiz 50000 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest order -getw \ + -msiz 5000 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest order -rem \ + -msiz 50000 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest order \ + -bnum 5000 -psiz 100 -msiz 50000 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest order -etc \ + -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest order -th 4 \ + -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest order -th 4 -pccap 100k -rnd -etc \ + -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k -rcd casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest order -th 4 -rnd -etc -tran \ + -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 1000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest order -th 4 -rnd -etc -oat \ + -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 1000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest order -th 4 -rnd -etc \ + -apow 2 -fpow 3 -ts -tl -tc -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest queue \ + -bnum 5000 -psiz 500 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest queue -rnd \ + -bnum 5000 -psiz 500 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest queue -th 4 -it 4 \ + -bnum 5000 -psiz 500 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest queue -th 4 -it 4 -rnd \ + -bnum 5000 -psiz 500 -msiz 50000 casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest wicked \ + -bnum 5000 -psiz 1000 -msiz 50000 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest wicked -th 4 -it 4 \ + -bnum 5000 -msiz 50000 -dfunit 4 -pccap 100k -rcd casket 10000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest wicked -th 4 -it 4 -oat \ + -bnum 5000 -msiz 50000 -dfunit 4 -pccap 100k casket 1000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest wicked -th 4 -it 4 \ + -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 casket 1000 + $(RUNENV) $(RUNCMD) ./kctreemgr check -onr casket + $(RUNENV) $(RUNCMD) ./kctreetest tran casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest tran -th 2 -it 4 -pccap 100k casket 10000 + $(RUNENV) $(RUNCMD) ./kctreetest tran -th 2 -it 4 \ + -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 -rcd casket 10000 + + +check-dir : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcdirmgr create -otr casket + $(RUNENV) $(RUNCMD) ./kcdirmgr inform -st casket + $(RUNENV) $(RUNCMD) ./kcdirmgr set -add casket duffy 1231 + $(RUNENV) $(RUNCMD) ./kcdirmgr set -add casket micky 0101 + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket fal 1007 + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket mikio 0211 + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket natsuki 0810 + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket micky "" + $(RUNENV) $(RUNCMD) ./kcdirmgr set -app casket duffy kukuku + $(RUNENV) $(RUNCMD) ./kcdirmgr remove casket micky + $(RUNENV) $(RUNCMD) ./kcdirmgr list -pv casket > check.out + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket ryu 1 + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket ken 2 + $(RUNENV) $(RUNCMD) ./kcdirmgr remove casket duffy + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket ryu syo-ryu-ken + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket ken tatsumaki-senpu-kyaku + $(RUNENV) $(RUNCMD) ./kcdirmgr set -inci casket int 1234 + $(RUNENV) $(RUNCMD) ./kcdirmgr set -inci casket int 5678 + $(RUNENV) $(RUNCMD) ./kcdirmgr set -incd casket double 1234.5678 + $(RUNENV) $(RUNCMD) ./kcdirmgr set -incd casket double 8765.4321 + $(RUNENV) $(RUNCMD) ./kcdirmgr get casket mikio + $(RUNENV) $(RUNCMD) ./kcdirmgr get casket ryu + $(RUNENV) $(RUNCMD) ./kcdirmgr import casket lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kcdirmgr list -pv -px casket > check.out + $(RUNENV) $(RUNCMD) ./kcdirmgr copy casket casket-para + $(RUNENV) $(RUNCMD) ./kcdirmgr dump casket check.out + $(RUNENV) $(RUNCMD) ./kcdirmgr load -otr casket check.out + $(RUNENV) $(RUNCMD) ./kcdirmgr setbulk casket aa aaa bb bbb cc ccc dd ddd + $(RUNENV) $(RUNCMD) ./kcdirmgr removebulk casket aa bb zz + $(RUNENV) $(RUNCMD) ./kcdirmgr getbulk casket aa bb cc dd + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirmgr inform -st casket + $(RUNENV) $(RUNCMD) ./kcdirmgr create -otr -otl -onr -tc casket + $(RUNENV) $(RUNCMD) ./kcdirmgr import casket < lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kcdirmgr set casket mikio kyotocabinet + $(RUNENV) $(RUNCMD) ./kcdirmgr set -app casket tako ikaunini + $(RUNENV) $(RUNCMD) ./kcdirmgr set -app casket mikio kyototyrant + $(RUNENV) $(RUNCMD) ./kcdirmgr set -app casket mikio kyotodystopia + $(RUNENV) $(RUNCMD) ./kcdirmgr get -px casket mikio > check.out + $(RUNENV) $(RUNCMD) ./kcdirmgr list casket > check.out + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirmgr clear casket + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcdirtest order -set casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest order -get casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest order -getw casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest order -rem casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest order casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest order -etc casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest order -th 4 casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest order -th 4 -rnd -etc casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest order -th 4 -rnd -etc -tran casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest order -th 4 -rnd -etc -oat casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest order -th 4 -rnd -etc -tc casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest queue casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest queue -rnd casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest queue -th 4 -it 4 casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest queue -th 4 -it 4 -rnd casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest wicked casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest wicked -th 4 -it 4 casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest wicked -th 4 -it 4 -oat casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest wicked -th 4 -it 4 -tc casket 500 + $(RUNENV) $(RUNCMD) ./kcdirmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcdirtest tran casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest tran -th 2 -it 4 casket 500 + $(RUNENV) $(RUNCMD) ./kcdirtest tran -th 2 -it 4 -tc casket 500 + + +check-forest : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcforestmgr create -otr -bnum 3 casket + $(RUNENV) $(RUNCMD) ./kcforestmgr inform -st casket + $(RUNENV) $(RUNCMD) ./kcforestmgr set -add casket duffy 1231 + $(RUNENV) $(RUNCMD) ./kcforestmgr set -add casket micky 0101 + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket fal 1007 + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket mikio 0211 + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket natsuki 0810 + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket micky "" + $(RUNENV) $(RUNCMD) ./kcforestmgr set -app casket duffy kukuku + $(RUNENV) $(RUNCMD) ./kcforestmgr remove casket micky + $(RUNENV) $(RUNCMD) ./kcforestmgr list -pv casket > check.out + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket ryu 1 + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket ken 2 + $(RUNENV) $(RUNCMD) ./kcforestmgr remove casket duffy + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket ryu syo-ryu-ken + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket ken tatsumaki-senpu-kyaku + $(RUNENV) $(RUNCMD) ./kcforestmgr set -inci casket int 1234 + $(RUNENV) $(RUNCMD) ./kcforestmgr set -inci casket int 5678 + $(RUNENV) $(RUNCMD) ./kcforestmgr set -incd casket double 1234.5678 + $(RUNENV) $(RUNCMD) ./kcforestmgr set -incd casket double 8765.4321 + $(RUNENV) $(RUNCMD) ./kcforestmgr get casket mikio + $(RUNENV) $(RUNCMD) ./kcforestmgr get casket ryu + $(RUNENV) $(RUNCMD) ./kcforestmgr import casket lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kcforestmgr list -des -pv -px casket > check.out + $(RUNENV) $(RUNCMD) ./kcforestmgr copy casket casket-para + $(RUNENV) $(RUNCMD) ./kcforestmgr dump casket check.out + $(RUNENV) $(RUNCMD) ./kcforestmgr load -otr casket check.out + $(RUNENV) $(RUNCMD) ./kcforestmgr setbulk casket aa aaa bb bbb cc ccc dd ddd + $(RUNENV) $(RUNCMD) ./kcforestmgr removebulk casket aa bb zz + $(RUNENV) $(RUNCMD) ./kcforestmgr getbulk casket aa bb cc dd + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforestmgr inform -st casket + $(RUNENV) $(RUNCMD) ./kcforestmgr create -otr -otl -onr \ + -tc -bnum 1 casket + $(RUNENV) $(RUNCMD) ./kcforestmgr import casket < lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kcforestmgr set casket mikio kyotocabinet + $(RUNENV) $(RUNCMD) ./kcforestmgr set -app casket tako ikaunini + $(RUNENV) $(RUNCMD) ./kcforestmgr set -app casket mikio kyototyrant + $(RUNENV) $(RUNCMD) ./kcforestmgr set -app casket mikio kyotodystopia + $(RUNENV) $(RUNCMD) ./kcforestmgr get -px casket mikio > check.out + $(RUNENV) $(RUNCMD) ./kcforestmgr list casket > check.out + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforestmgr clear casket + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcforesttest order -set \ + -psiz 100 -bnum 5000 -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest order -get \ + -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest order -getw \ + -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest order -rem \ + -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest order \ + -bnum 5000 -psiz 100 -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest order -etc \ + -bnum 5000 -psiz 1000 -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest order -th 4 \ + -bnum 5000 -psiz 1000 -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest order -th 4 -pccap 100k -rnd -etc \ + -bnum 5000 -psiz 1000 -pccap 100k -rcd casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest order -th 4 -rnd -etc -tran \ + -bnum 500 -psiz 1000 -pccap 100k casket 500 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest order -th 4 -rnd -etc -oat \ + -bnum 500 -psiz 1000 -pccap 100k casket 500 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest order -th 4 -rnd -etc \ + -tc -bnum 5000 -psiz 1000 casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest queue \ + -bnum 5000 -psiz 500 casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest queue -rnd \ + -bnum 5000 -psiz 500 casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest queue -th 4 -it 4 \ + -bnum 5000 -psiz 500 casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest queue -th 4 -it 4 -rnd \ + -bnum 5000 -psiz 500 casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest wicked \ + -bnum 5000 -psiz 1000 -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest wicked -th 4 -it 4 \ + -bnum 5000 -pccap 100k -rcd casket 5000 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest wicked -th 4 -it 4 -oat \ + -bnum 500 -pccap 100k casket 500 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest wicked -th 4 -it 4 \ + -tc -bnum 500 casket 500 + $(RUNENV) $(RUNCMD) ./kcforestmgr check -onr casket + $(RUNENV) $(RUNCMD) ./kcforesttest tran casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest tran -th 2 -it 4 -pccap 100k casket 5000 + $(RUNENV) $(RUNCMD) ./kcforesttest tran -th 2 -it 4 \ + -tc -bnum 5000 -rcd casket 5000 + + +check-poly : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolymgr create -otr "casket.kch#apow=1#fpow=2#bnum=3" + $(RUNENV) $(RUNCMD) ./kcpolymgr inform -st casket.kch + $(RUNENV) $(RUNCMD) ./kcpolymgr set -add casket.kch duffy 1231 + $(RUNENV) $(RUNCMD) ./kcpolymgr set -add casket.kch micky 0101 + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch fal 1007 + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch mikio 0211 + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch natsuki 0810 + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch micky "" + $(RUNENV) $(RUNCMD) ./kcpolymgr set -app casket.kch duffy kukuku + $(RUNENV) $(RUNCMD) ./kcpolymgr remove casket.kch micky + $(RUNENV) $(RUNCMD) ./kcpolymgr list -pv casket.kch > check.out + $(RUNENV) $(RUNCMD) ./kcpolymgr copy casket.kch casket-para + $(RUNENV) $(RUNCMD) ./kcpolymgr dump casket.kch check.out + $(RUNENV) $(RUNCMD) ./kcpolymgr load -otr casket.kch check.out + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch ryu 1 + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch ken 2 + $(RUNENV) $(RUNCMD) ./kcpolymgr remove casket.kch duffy + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch ryu syo-ryu-ken + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kch ken tatsumaki-senpu-kyaku + $(RUNENV) $(RUNCMD) ./kcpolymgr set -inci casket.kch int 1234 + $(RUNENV) $(RUNCMD) ./kcpolymgr set -inci casket.kch int 5678 + $(RUNENV) $(RUNCMD) ./kcpolymgr set -incd casket.kch double 1234.5678 + $(RUNENV) $(RUNCMD) ./kcpolymgr set -incd casket.kch double 8765.4321 + $(RUNENV) $(RUNCMD) ./kcpolymgr get "casket.kch" mikio + $(RUNENV) $(RUNCMD) ./kcpolymgr get "casket.kch" ryu + $(RUNENV) $(RUNCMD) ./kcpolymgr import casket.kch lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kcpolymgr list -pv -px "casket.kch#mode=r" > check.out + $(RUNENV) $(RUNCMD) ./kcpolymgr setbulk casket.kch aa aaa bb bbb cc ccc dd ddd + $(RUNENV) $(RUNCMD) ./kcpolymgr removebulk casket.kch aa bb zz + $(RUNENV) $(RUNCMD) ./kcpolymgr getbulk casket.kch aa bb cc dd + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kch + $(RUNENV) $(RUNCMD) ./kcpolymgr inform -st casket.kch + $(RUNENV) $(RUNCMD) ./kcpolymgr create -otr -otl -onr \ + "casket.kct#apow=1#fpow=3#opts=slc#bnum=1" + $(RUNENV) $(RUNCMD) ./kcpolymgr import casket.kct < lab/numbers.tsv + $(RUNENV) $(RUNCMD) ./kcpolymgr set casket.kct mikio kyotocabinet + $(RUNENV) $(RUNCMD) ./kcpolymgr set -app casket.kct tako ikaunini + $(RUNENV) $(RUNCMD) ./kcpolymgr set -app casket.kct mikio kyototyrant + $(RUNENV) $(RUNCMD) ./kcpolymgr set -app casket.kct mikio kyotodystopia + $(RUNENV) $(RUNCMD) ./kcpolymgr get -px casket.kct mikio > check.out + $(RUNENV) $(RUNCMD) ./kcpolymgr list casket.kct > check.out + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolymgr clear casket.kct + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest order -set "casket.kct#bnum=5000#msiz=50000" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -get "casket.kct#msiz=50000" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -getw "casket.kct#msiz=5000" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -rem "casket.kct#msiz=50000" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order "casket.kct#bnum=5000#msiz=50000" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -etc \ + "casket.kct#bnum=5000#msiz=50000#dfunit=4" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -th 4 \ + "casket.kct#bnum=5000#msiz=50000#dfunit=4" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -th 4 -rnd -etc \ + "casket.kct#bnum=5000#msiz=0#dfunit=1" 1000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest order -th 4 -rnd -etc -tran \ + "casket.kct#bnum=5000#msiz=0#dfunit=2" 1000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest order -th 4 -rnd -etc -oat \ + "casket.kct#bnum=5000#msiz=0#dfunit=3" 1000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest order -th 4 -rnd -etc \ + "casket.kct#apow=2#fpow=3#opts=slc#bnum=5000#msiz=0#dfunit=4" 1000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest queue \ + "casket.kct#bnum=5000#msiz=0" 10000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest queue -rnd \ + "casket.kct#bnum=5000#msiz=0" 10000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest queue -th 4 -it 4 \ + "casket.kct#bnum=5000#msiz=0" 10000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest queue -th 4 -it 4 -rnd \ + "casket.kct#bnum=5000#msiz=0" 10000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest wicked "casket.kct#bnum=5000#msiz=0" 1000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest wicked -th 4 -it 4 \ + "casket.kct#bnum=5000#msiz=0#dfunit=1" 1000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest wicked -th 4 -it 4 -oat \ + "casket.kct#bnum=5000#msiz=0#dfunit=1" 1000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest wicked -th 4 -it 4 \ + "casket.kct#apow=2#fpow=3#opts=slc#bnum=10000#msiz=0#dfunit=1" 10000 + $(RUNENV) $(RUNCMD) ./kcpolymgr check -onr casket.kct + $(RUNENV) $(RUNCMD) ./kcpolytest tran casket.kct 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest tran -th 2 -it 4 casket.kct 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest tran -th 2 -it 4 \ + "casket.kct#apow=2#fpow=3#opts=slc#bnum=10000#msiz=0#dfunit=1" 1000 + $(RUNENV) $(RUNCMD) ./kcpolytest mapred -dbnum 2 -clim 10k casket.kct 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest mapred -tmp . -dbnum 2 -clim 10k -xnl -xnc \ + casket.kct 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest mapred -tmp . -dbnum 2 -clim 10k -xpm -xpr -xpf -xnc \ + casket.kct 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest mapred -rnd -dbnum 2 -clim 10k casket.kct 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -set "casket.kct#idxclim=32k" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -get "casket.kct" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -rem "casket.kct" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -etc "casket.kct#idxclim=32k" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -th 4 -rnd -set \ + "casket.kct#idxclim=32k#idxdbnum=4" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -th 4 -rnd -get "casket.kct" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -th 4 -rnd -rem "casket.kct" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest index -th 4 -rnd -etc \ + "casket.kct#idxclim=32k#idxdbnum=4" 10000 + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest order -rnd "casket.kcx" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -th 4 -rnd "casket.kcx" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest wicked "casket.kcx" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest wicked -th 4 "casket.kcx" 10000 + $(RUNENV) $(RUNCMD) ./kcpolymgr list -pv "casket.kcx" > check.out + $(RUNENV) $(RUNCMD) ./kcpolymgr list -max 1000 -pv "casket.kcx" > check.out + $(RUNENV) $(RUNCMD) ./kcpolytest mapred "casket.kcx" 10000 + $(RUNENV) $(RUNCMD) ./kcpolytest mapred -xpm -xpr -xpf "casket.kcx" 10000 + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest order -rnd "casket.kch#opts=s#bnum=256" 1000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -rnd "casket.kct#opts=l#psiz=256" 1000 + $(RUNENV) $(RUNCMD) ./kcpolytest order -rnd "casket.kcd#opts=c#bnum=256" 500 + $(RUNENV) $(RUNCMD) ./kcpolytest order -rnd "casket.kcf#opts=c#psiz=256" 500 + $(RUNENV) $(RUNCMD) ./kcpolytest order -rnd "casket.kcx" 500 + $(RUNENV) $(RUNCMD) ./kcpolymgr merge -add "casket#type=kct" \ + casket.kch casket.kct casket.kcd casket.kcf casket.kcx + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest misc "casket#type=-" + $(RUNENV) $(RUNCMD) ./kcpolytest misc "casket#type=+" + $(RUNENV) $(RUNCMD) ./kcpolytest misc "casket#type=:" + $(RUNENV) $(RUNCMD) ./kcpolytest misc "casket#type=*#zcomp=def" + $(RUNENV) $(RUNCMD) ./kcpolytest misc "casket#type=%#zcomp=gz" + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest misc \ + "casket#type=kch#log=-#logkinds=debug#mtrg=-#zcomp=lzocrc" + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest misc \ + "casket#type=kct#log=-#logkinds=debug#mtrg=-#zcomp=lzmacrc" + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest misc \ + "casket#type=kcd#zcomp=arc#zkey=mikio" + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kcpolytest misc \ + "casket#type=kcf#zcomp=arc#zkey=mikio" + + +check-langc : + rm -rf casket* + $(RUNENV) $(RUNCMD) ./kclangctest order "casket.kch#bnum=5000#msiz=50000" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest order -etc \ + "casket.kch#bnum=5000#msiz=50000#dfunit=2" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest order -rnd -etc \ + "casket.kch#bnum=5000#msiz=50000#dfunit=2" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest order -rnd -etc -oat -tran \ + "casket.kch#bnum=5000#msiz=50000#dfunit=2#zcomp=arcz" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest index "casket.kct#bnum=5000#msiz=50000" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest index -etc \ + "casket.kct#bnum=5000#msiz=50000#dfunit=2" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest index -rnd -etc \ + "casket.kct#bnum=5000#msiz=50000#dfunit=2" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest index -rnd -etc -oat \ + "casket.kct#bnum=5000#msiz=50000#dfunit=2#zcomp=arcz" 10000 + $(RUNENV) $(RUNCMD) ./kclangctest map 10000 + $(RUNENV) $(RUNCMD) ./kclangctest map -etc -bnum 1000 10000 + $(RUNENV) $(RUNCMD) ./kclangctest map -etc -rnd -bnum 1000 10000 + $(RUNENV) $(RUNCMD) ./kclangctest list 10000 + $(RUNENV) $(RUNCMD) ./kclangctest list -etc 10000 + $(RUNENV) $(RUNCMD) ./kclangctest list -etc -rnd 10000 + + +check-valgrind : + $(MAKE) RUNCMD="valgrind --tool=memcheck --log-file=%p.vlog" check + grep ERROR *.vlog | grep -v ' 0 errors' ; true + grep 'at exit' *.vlog | grep -v ' 0 bytes' ; true + + +check-heavy : + $(MAKE) check-hash-heavy + $(MAKE) check-tree-heavy + + +check-hash-heavy : + $(RUNENV) ./kchashtest order -th 4 \ + -apow 2 -fpow 2 -bnum 500000 -msiz 50m -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest order -th 4 -rnd \ + -apow 2 -fpow 2 -bnum 500000 -msiz 50m -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest order -th 4 -etc \ + -apow 2 -fpow 2 -bnum 500000 -msiz 50m -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest order -th 4 -etc -rnd \ + -apow 2 -fpow 2 -bnum 500000 -msiz 50m -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest order -th 4 -etc -rnd \ + -ts -tl -tc -dfunit 2 casket 25000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest queue -th 4 -it 10 \ + -bnum 1000000 -apow 4 -fpow 12 -msiz 100m -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest queue -th 4 -it 5 -rnd \ + -ts -tl -tc -dfunit 2 casket 25000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest queue -th 4 -it 2 -oat -rnd \ + -bnum 1000 -dfunit 8 casket 25000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest queue -th 4 -it 2 -oas -rnd \ + -bnum 1000 -dfunit 8 casket 2500 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest wicked -th 4 -it 10 \ + -bnum 1000000 -apow 4 -fpow 12 -msiz 100m -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest wicked -th 4 -it 5 \ + -ts -tl -tc -dfunit 2 casket 25000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest wicked -th 4 -it 2 -oat \ + -bnum 1000 -dfunit 8 casket 25000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest wicked -th 4 -it 2 -oas \ + -bnum 1000 -dfunit 8 casket 2500 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest tran -th 4 -it 10 \ + -apow 2 -fpow 2 -bnum 500000 -msiz 50m -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + $(RUNENV) ./kchashtest tran -th 4 -it 5 \ + -ts -tl -tc -dfunit 2 casket 250000 + $(RUNENV) ./kchashmgr check -onr casket + + +check-tree-heavy : + $(RUNENV) ./kctreetest order -th 4 \ + -apow 2 -fpow 2 -bnum 50000 -psiz 1000 -msiz 50m -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest order -th 4 -rnd \ + -apow 2 -fpow 2 -bnum 50000 -psiz 1000 -msiz 50m -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest order -th 4 -etc \ + -apow 2 -fpow 2 -bnum 50000 -psiz 1000 -msiz 50m -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest order -th 4 -etc -rnd \ + -apow 2 -fpow 2 -bnum 50000 -psiz 1000 -msiz 50m -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest order -th 4 -etc -rnd \ + -ts -tl -tc -dfunit 2 casket 25000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest queue -th 4 -it 10 \ + -bnum 1000000 -apow 4 -fpow 12 -msiz 100m -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest queue -th 4 -it 5 -rnd \ + -ts -tl -tc -dfunit 2 -pccap 10m casket 25000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest queue -th 4 -it 2 -oat -rnd \ + -bnum 1000 -dfunit 8 -pccap 10m casket 25000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest queue -th 4 -it 2 -oas -rnd \ + -bnum 1000 -dfunit 8 -pccap 10m casket 2500 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest wicked -th 4 -it 5 \ + -bnum 100000 -apow 4 -fpow 12 -msiz 100m -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest wicked -th 4 -it 2 \ + -ts -tl -tc -dfunit 2 -pccap 10m casket 25000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest wicked -th 4 -it 2 -oat \ + -bnum 1000 -dfunit 8 -pccap 10m casket 25000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest wicked -th 4 -it 2 -oas \ + -bnum 1000 -dfunit 8 -pccap 10m casket 2500 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest tran -th 4 -it 5 \ + -apow 2 -fpow 2 -bnum 50000 -msiz 50m -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + $(RUNENV) ./kctreetest tran -th 4 -it 2 \ + -ts -tl -tc -dfunit 2 -pccap 10m casket 250000 + $(RUNENV) ./kctreemgr check -onr casket + + +check-segv : + $(RUNENV) ./lab/segvtest hash "0.5" 100 + $(RUNENV) ./lab/segvtest hash "" 10 + $(RUNENV) ./lab/segvtest hash "100" 1 + $(RUNENV) ./lab/segvtest hash -oat "0.5" 100 + $(RUNENV) ./lab/segvtest hash -oat "" 10 + $(RUNENV) ./lab/segvtest hash -oat "100" 1 + $(RUNENV) ./lab/segvtest hash -tran "" 10 + $(RUNENV) ./lab/segvtest hash -wicked "" 10 + $(RUNENV) ./lab/segvtest hash -wicked -oat "" 10 + $(RUNENV) ./lab/segvtest tree "0.5" 100 + $(RUNENV) ./lab/segvtest tree "" 10 + $(RUNENV) ./lab/segvtest tree "100" 1 + $(RUNENV) ./lab/segvtest tree -oat "0.5" 100 + $(RUNENV) ./lab/segvtest tree -oat "" 10 + $(RUNENV) ./lab/segvtest tree -oat "100" 1 + $(RUNENV) ./lab/segvtest tree -tran "" 10 + $(RUNENV) ./lab/segvtest tree -wicked "" 10 + $(RUNENV) ./lab/segvtest tree -wicked -oat "" 10 + $(RUNENV) ./lab/segvtest dir -oat "" 10 + $(RUNENV) ./lab/segvtest dir -oat "0.5" 100 + $(RUNENV) ./lab/segvtest dir -oat "" 10 + $(RUNENV) ./lab/segvtest dir -oat "100" 1 + $(RUNENV) ./lab/segvtest dir -tran "" 10 + $(RUNENV) ./lab/segvtest dir -wicked "" 10 + $(RUNENV) ./lab/segvtest dir -wicked -oat "" 10 + $(RUNENV) ./lab/segvtest forest "" 10 + $(RUNENV) ./lab/segvtest forest -oat "0.5" 100 + $(RUNENV) ./lab/segvtest forest -oat "" 10 + $(RUNENV) ./lab/segvtest forest -oat "100" 1 + $(RUNENV) ./lab/segvtest forest -tran "" 10 + $(RUNENV) ./lab/segvtest forest -wicked "" 10 + $(RUNENV) ./lab/segvtest forest -wicked -oat "" 10 + + +check-forever : + while true ; \ + do \ + $(MAKE) check || break ; \ + $(MAKE) check || break ; \ + $(MAKE) check || break ; \ + $(MAKE) check || break ; \ + $(MAKE) check-heavy || break ; \ + $(MAKE) check-segv || break ; \ + done + + +doc : + $(MAKE) docclean + mkdir -p doc/api + doxygen + + +docclean : + rm -rf doc/api + + +gch : + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) *.h + + +words.tsv : + cat /usr/share/dict/words | \ + tr '\t\r' ' ' | grep -v '^ *$$' | cat -n | sort | \ + LC_ALL=C sed -e 's/^ *//' -e 's/\(^[0-9]*\)\t\(.*\)/\2\t\1/' > words.tsv + + +def : libkyotocabinet.a + ./lab/makevcdef libkyotocabinet.a > kyotocabinet.def + + +.PHONY : all clean install check doc + + + +#================================================================ +# Building binaries +#================================================================ + + +libkyotocabinet.a : $(LIBOBJFILES) + $(AR) $(ARFLAGS) $@ $(LIBOBJFILES) + + +libkyotocabinet.so.$(LIBVER).$(LIBREV).0 : $(LIBOBJFILES) + if uname -a | egrep -i 'SunOS' > /dev/null ; \ + then \ + $(CXX) $(CXXFLAGS) -shared -Wl,-G,-h,libkyotocabinet.so.$(LIBVER) -o $@ \ + $(LIBOBJFILES) $(LDFLAGS) $(LIBS) ; \ + else \ + $(CXX) $(CXXFLAGS) -shared -Wl,-soname,libkyotocabinet.so.$(LIBVER) -o $@ \ + $(LIBOBJFILES) $(LDFLAGS) $(LIBS) ; \ + fi + + +libkyotocabinet.so.$(LIBVER) : libkyotocabinet.so.$(LIBVER).$(LIBREV).0 + ln -f -s libkyotocabinet.so.$(LIBVER).$(LIBREV).0 $@ + + +libkyotocabinet.so : libkyotocabinet.so.$(LIBVER).$(LIBREV).0 + ln -f -s libkyotocabinet.so.$(LIBVER).$(LIBREV).0 $@ + + +libkyotocabinet.$(LIBVER).$(LIBREV).0.dylib : $(LIBOBJFILES) + $(CXX) $(CXXFLAGS) -dynamiclib -o $@ \ + -install_name $(LIBDIR)/libkyotocabinet.$(LIBVER).dylib \ + -current_version $(LIBVER).$(LIBREV).0 -compatibility_version $(LIBVER) \ + $(LIBOBJFILES) $(LDFLAGS) $(LIBS) + + +libkyotocabinet.$(LIBVER).dylib : libkyotocabinet.$(LIBVER).$(LIBREV).0.dylib + ln -f -s libkyotocabinet.$(LIBVER).$(LIBREV).0.dylib $@ + + +libkyotocabinet.dylib : libkyotocabinet.$(LIBVER).$(LIBREV).0.dylib + ln -f -s libkyotocabinet.$(LIBVER).$(LIBREV).0.dylib $@ + + +kcutiltest : kcutiltest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcutilmgr : kcutilmgr.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcprototest : kcprototest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcstashtest : kcstashtest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kccachetest : kccachetest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcgrasstest : kcgrasstest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kchashtest : kchashtest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kchashmgr : kchashmgr.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kctreetest : kctreetest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kctreemgr : kctreemgr.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcdirtest : kcdirtest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcdirmgr : kcdirmgr.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcforesttest : kcforesttest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcforestmgr : kcforestmgr.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcpolytest : kcpolytest.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcpolymgr : kcpolymgr.o $(LIBRARYFILES) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kclangctest : kclangctest.o $(LIBRARYFILES) + $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) $(CMDLDFLAGS) -lkyotocabinet $(CMDLIBS) + + +kcutil.o : kccommon.h kcutil.h myconf.h + +kcthread.o : kccommon.h kcutil.h kcthread.h myconf.h + +kcfile.o : kccommon.h kcutil.h kcthread.h kcfile.h myconf.h + +kccompress.o : kccommon.h kcutil.h kccompress.h myconf.h + +kccompare.o : kccommon.h kcutil.h kccompare.h myconf.h + +kcmap.o : kccommon.h kcutil.h kcmap.h myconf.h + +kcregex.o : kccommon.h kcutil.h kcregex.h myconf.h + +kcdb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h + +kcplantdb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h + +kcprotodb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h + +kcstashdb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcstashdb.h + +kccachedb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kccachedb.h + +kchashdb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kchashdb.h + +kcdirdb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcdirdb.h + +kctextdb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kctextdb.h + +kcpolydb.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h kcpolydb.h + +kcdbext.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \ + kcpolydb.h kcdbext.h + +kclangc.o : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \ + kcpolydb.h kcdbext.h kclangc.h + +kcutiltest.o kcutilmgr.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + cmdcommon.h + +kcprototest.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h cmdcommon.h + +kcstashtest.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcstashdb.h cmdcommon.h + +kccachetest.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kccachedb.h cmdcommon.h + +kcgrasstest.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kccachedb.h cmdcommon.h + +kchashtest.o kchashmgr.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kchashdb.h cmdcommon.h + +kctreetest.o kctreemgr.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kchashdb.h cmdcommon.h + +kcdirtest.o kcdirmgr.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcdirdb.h cmdcommon.h + +kcforesttest.o kcforestmgr.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcdirdb.h cmdcommon.h + +kcpolytest.o kcpolymgr.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \ + kcpolydb.h kcdbext.h cmdcommon.h + +kclangctest.o : \ + kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \ + kcmap.h kcregex.h \ + kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \ + kcpolydb.h kcdbext.h kclangc.h + + + +# END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/README b/plugins/Dbx_kyoto/src/kyotocabinet/README new file mode 100644 index 0000000000..3d289bc4a5 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/README @@ -0,0 +1,38 @@ +================================================================ + Kyoto Cabinet: a straightforward implementation of DBM + Copyright (C) 2009-2011 FAL Labs +================================================================ + + +Please read the following documents with a WWW browser. +How to install Kyoto Cabinet is explained in the specification. + + README - this file + COPYING - license (GPLv3) + FOSSEXCEPTION - license exception for FOSS + LINKEXCEPTION - license exception for specific libraries + ChangeLog - history of enhancement + doc/index.html - index of documents + + +Contents of the directory tree is below. + + ./ - sources of Kyoto Cabinet + ./doc/ - manuals and specifications + ./man/ - manuals for nroff + ./example/ - sample code for tutorial + ./lab/ - for test and experiment + + +Kyoto Cabinet is released under the terms of the GNU General Public +License version 3. See the file `COPYING' for details. + +Kyoto Cabinet was written by FAL Labs. You can contact the author +by e-mail to `info@fallabs.com'. + + +Thanks. + + + +== END OF FILE == diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/VCmakefile b/plugins/Dbx_kyoto/src/kyotocabinet/VCmakefile new file mode 100644 index 0000000000..1dfc0e2f90 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/VCmakefile @@ -0,0 +1,946 @@ +# Makefile for Kyoto Cabinet for Win32
+
+
+
+#================================================================
+# Setting Variables
+#================================================================
+
+
+# VC++ directory
+VCPATH = C:\Program Files\Microsoft Visual Studio 10.0\VC
+SDKPATH = C:\Program Files\Microsoft SDKs\Windows\v7.0A
+
+
+# Targets
+LIBRARYFILES = kyotocabinet.lib
+LIBOBJFILES = kcutil.obj kcdb.obj kcthread.obj kcfile.obj \
+ kccompress.obj kccompare.obj kcmap.obj kcregex.obj kcplantdb.obj \
+ kcprotodb.obj kcstashdb.obj kccachedb.obj kchashdb.obj kcdirdb.obj kctextdb.obj \
+ kcpolydb.obj kcdbext.obj kclangc.obj
+COMMANDFILES = kcutiltest.exe kcutilmgr.exe kcprototest.exe \
+ kcstashtest.exe kccachetest.exe kcgrasstest.exe \
+ kchashtest.exe kchashmgr.exe kctreetest.exe kctreemgr.exe \
+ kcdirtest.exe kcdirmgr.exe kcforesttest.exe kcforestmgr.exe \
+ kcpolytest.exe kcpolymgr.exe kclangctest.exe
+
+
+# Building configuration
+CL = cl
+LIB = lib
+LINK = link
+CLFLAGS = /nologo \
+ /I "$(VCPATH)\Include" /I "$(VCPATH)\PlatformSDK\Include" /I "$(SDKPATH)\Include" \
+ /I "." \
+ /DNDEBUG /D_CRT_SECURE_NO_WARNINGS \
+ /O2 /EHsc /W3 /wd4244 /wd4351 /wd4800 /MT
+LIBFLAGS = /nologo \
+ /libpath:"$(VCPATH)\lib" /libpath:"$(VCPATH)\PlatformSDK\Lib" /libpath:"$(SDKPATH)\Lib" \
+ /libpath:"."
+LINKFLAGS = /nologo \
+ /libpath:"$(VCPATH)\lib" /libpath:"$(VCPATH)\PlatformSDK\Lib" /libpath:"$(SDKPATH)\Lib" \
+ /libpath:"."
+
+
+
+#================================================================
+# Suffix rules
+#================================================================
+
+
+.SUFFIXES :
+.SUFFIXES : .cc .c .obj .exe
+
+.c.obj :
+ $(CL) /c $(CLFLAGS) $<
+
+.cc.obj :
+ $(CL) /c $(CLFLAGS) $<
+
+
+
+#================================================================
+# Actions
+#================================================================
+
+
+all : $(LIBRARYFILES) $(COMMANDFILES)
+ @echo #
+ @echo #================================================================
+ @echo # Ready to install.
+ @echo #================================================================
+
+
+clean :
+ -del *.obj *.lib *.dll *.exp *.exe /F /Q > NUL: 2>&1
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+
+
+check : check-util check-proto check-stash check-cache check-grass \
+ check-hash check-tree check-dir check-forest check-poly check-langc
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ @echo #
+ @echo #================================================================
+ @echo # Checking completed.
+ @echo #================================================================
+
+
+check-util :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcutilmgr version
+ kcutilmgr hex VCmakefile > check.in
+ kcutilmgr hex -d check.in > check.out
+ kcutilmgr enc VCmakefile > check.in
+ kcutilmgr enc -d check.in > check.out
+ kcutilmgr enc -hex VCmakefile > check.in
+ kcutilmgr enc -hex -d check.in > check.out
+ kcutilmgr enc -url VCmakefile > check.in
+ kcutilmgr enc -url -d check.in > check.out
+ kcutilmgr enc -quote VCmakefile > check.in
+ kcutilmgr enc -quote -d check.in > check.out
+ kcutilmgr ciph -key "hoge" VCmakefile > check.in
+ kcutilmgr ciph -key "hoge" check.in > check.out
+ kcutilmgr comp -gz VCmakefile > check.in
+ kcutilmgr comp -gz -d check.in > check.out
+ kcutilmgr comp -lzo VCmakefile > check.in
+ kcutilmgr comp -lzo -d check.in > check.out
+ kcutilmgr comp -lzma VCmakefile > check.in
+ kcutilmgr comp -lzma -d check.in > check.out
+ kcutilmgr hash VCmakefile > check.in
+ kcutilmgr hash -fnv VCmakefile > check.out
+ kcutilmgr hash -path VCmakefile > check.out
+ kcutilmgr regex mikio VCmakefile > check.out
+ kcutilmgr regex -alt "hirarin" mikio VCmakefile > check.out
+ kcutilmgr conf
+ -del casket* /F /Q > NUL: 2>&1
+ kcutiltest mutex -th 4 -iv -1 10000
+ kcutiltest cond -th 4 -iv -1 10000
+ kcutiltest para -th 4 10000
+ kcutiltest para -th 4 -iv -1 10000
+ kcutiltest file -th 4 casket 10000
+ kcutiltest file -th 4 -rnd -msiz 1m casket 10000
+ kcutiltest lhmap -bnum 1000 10000
+ kcutiltest lhmap -rnd -bnum 1000 10000
+ kcutiltest thmap -bnum 1000 10000
+ kcutiltest thmap -rnd -bnum 1000 10000
+ kcutiltest talist 10000
+ kcutiltest talist -rnd 10000
+ kcutiltest misc 10000
+
+
+check-proto :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcprototest order -etc 10000
+ kcprototest order -th 4 10000
+ kcprototest order -th 4 -rnd -etc 10000
+ kcprototest order -th 4 -rnd -etc -tran 10000
+ kcprototest wicked 10000
+ kcprototest wicked -th 4 -it 4 10000
+ kcprototest tran 10000
+ kcprototest tran -th 2 -it 4 10000
+ -del casket* /F /Q > NUL: 2>&1
+ kcprototest order -tree -etc 10000
+ kcprototest order -tree -th 4 10000
+ kcprototest order -tree -th 4 -rnd -etc 10000
+ kcprototest order -tree -th 4 -rnd -etc -tran 10000
+ kcprototest wicked -tree 10000
+ kcprototest wicked -tree -th 4 -it 4 10000
+ kcprototest tran -tree 10000
+ kcprototest tran -tree -th 2 -it 4 10000
+
+
+check-stash :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcstashtest order -etc -bnum 5000 10000
+ kcstashtest order -th 4 -bnum 5000 10000
+ kcstashtest order -th 4 -rnd -etc -bnum 5000 10000
+ kcstashtest order -th 4 -rnd -etc -bnum 5000 10000
+ kcstashtest order -th 4 -rnd -etc -tran \
+ -bnum 5000 10000
+ kcstashtest wicked -bnum 5000 10000
+ kcstashtest wicked -th 4 -it 4 -bnum 5000 10000
+ kcstashtest tran -bnum 5000 10000
+ kcstashtest tran -th 2 -it 4 -bnum 5000 10000
+
+
+check-cache :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kccachetest order -etc -bnum 5000 10000
+ kccachetest order -th 4 -bnum 5000 10000
+ kccachetest order -th 4 -rnd -etc -bnum 5000 -capcnt 10000 10000
+ kccachetest order -th 4 -rnd -etc -bnum 5000 -capsiz 10000 10000
+ kccachetest order -th 4 -rnd -etc -tran \
+ -tc -bnum 5000 -capcnt 10000 10000
+ kccachetest wicked -bnum 5000 10000
+ kccachetest wicked -th 4 -it 4 -tc -bnum 5000 -capcnt 10000 10000
+ kccachetest tran -bnum 5000 10000
+ kccachetest tran -th 2 -it 4 -tc -bnum 5000 10000
+
+
+check-grass :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ $(RUNENV) $(RUNCMD) kcgrasstest order -etc -bnum 5000 10000
+ $(RUNENV) $(RUNCMD) kcgrasstest order -th 4 -bnum 5000 10000
+ $(RUNENV) $(RUNCMD) kcgrasstest order -th 4 -rnd -etc -bnum 5000 10000
+ $(RUNENV) $(RUNCMD) kcgrasstest order -th 4 -rnd -etc -bnum 5000 10000
+ $(RUNENV) $(RUNCMD) kcgrasstest order -th 4 -rnd -etc -tran \
+ -tc -bnum 5000 -pccap 100k 1000
+ $(RUNENV) $(RUNCMD) kcgrasstest wicked -bnum 5000 10000
+ $(RUNENV) $(RUNCMD) kcgrasstest wicked -th 4 -it 4 -tc -bnum 5000 -pccap 100k 10000
+ $(RUNENV) $(RUNCMD) kcgrasstest tran -bnum 5000 10000
+ $(RUNENV) $(RUNCMD) kcgrasstest tran -th 2 -it 4 -tc -bnum 5000 -pccap 100k 10000
+
+
+check-hash :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kchashmgr create -otr -apow 1 -fpow 2 -bnum 3 casket
+ kchashmgr inform -st casket
+ kchashmgr set -add casket duffy 1231
+ kchashmgr set -add casket micky 0101
+ kchashmgr set casket fal 1007
+ kchashmgr set casket mikio 0211
+ kchashmgr set casket natsuki 0810
+ kchashmgr set casket micky ""
+ kchashmgr set -rep casket duffy 777
+ kchashmgr set -app casket duffy kukuku
+ kchashmgr remove casket micky
+ kchashmgr list -pv casket > check.out
+ kchashmgr set casket ryu 1
+ kchashmgr set casket ken 2
+ kchashmgr remove casket duffy
+ kchashmgr set casket ryu syo-ryu-ken
+ kchashmgr set casket ken tatsumaki-senpu-kyaku
+ kchashmgr set -inci casket int 1234
+ kchashmgr set -inci casket int 5678
+ kchashmgr set -incd casket double 1234.5678
+ kchashmgr set -incd casket double 8765.4321
+ kchashmgr get casket mikio
+ kchashmgr get casket ryu
+ kchashmgr import casket lab/numbers.tsv
+ kchashmgr list -pv -px casket > check.out
+ kchashmgr copy casket casket-para
+ kchashmgr dump casket check.out
+ kchashmgr load -otr casket check.out
+ kchashmgr defrag -onl casket
+ kchashmgr check -onr casket
+ kchashmgr inform -st casket
+ kchashmgr create -otr -otl -onr -apow 1 -fpow 3 \
+ -ts -tl -tc -bnum 1 casket
+ kchashmgr import casket < lab/numbers.tsv
+ kchashmgr set casket mikio kyotocabinet
+ kchashmgr set -app casket tako ikaunini
+ kchashmgr set -app casket mikio kyototyrant
+ kchashmgr set -app casket mikio kyotodystopia
+ kchashmgr get -px casket mikio > check.out
+ kchashmgr list casket > check.out
+ kchashmgr check -onr casket
+ -del casket* /F /Q > NUL: 2>&1
+ kchashtest order -set -bnum 5000 -msiz 50000 casket 10000
+ kchashtest order -get -msiz 50000 casket 10000
+ kchashtest order -getw -msiz 5000 casket 10000
+ kchashtest order -rem -msiz 50000 casket 10000
+ kchashtest order -bnum 5000 -msiz 50000 casket 10000
+ kchashtest order -etc \
+ -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashtest order -th 4 \
+ -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashtest order -th 4 -rnd -etc \
+ -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashmgr check -onr casket
+ kchashtest order -th 4 -rnd -etc -tran \
+ -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashmgr check -onr casket
+ kchashtest order -th 4 -rnd -etc -oat \
+ -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashmgr check -onr casket
+ kchashtest order -th 4 -rnd -etc \
+ -apow 2 -fpow 3 -ts -tl -tc -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashmgr check -onr casket
+ kchashtest queue \
+ -bnum 5000 -msiz 50000 casket 10000
+ kchashmgr check -onr casket
+ kchashtest queue -rnd \
+ -bnum 5000 -msiz 50000 casket 10000
+ kchashmgr check -onr casket
+ kchashtest queue -th 4 -it 4 \
+ -bnum 5000 -msiz 50000 casket 10000
+ kchashmgr check -onr casket
+ kchashtest queue -th 4 -it 4 -rnd \
+ -bnum 5000 -msiz 50000 casket 10000
+ kchashmgr check -onr casket
+ kchashtest wicked -bnum 5000 -msiz 50000 casket 10000
+ kchashmgr check -onr casket
+ kchashtest wicked -th 4 -it 4 \
+ -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashmgr check -onr casket
+ kchashtest wicked -th 4 -it 4 -oat \
+ -bnum 5000 -msiz 50000 -dfunit 4 casket 10000
+ kchashmgr check -onr casket
+ kchashtest wicked -th 4 -it 4 \
+ -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 casket 10000
+ kchashmgr check -onr casket
+ kchashtest tran casket 10000
+ kchashtest tran -th 2 -it 4 casket 10000
+ kchashtest tran -th 2 -it 4 \
+ -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 casket 10000
+
+
+check-tree :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kctreemgr create -otr -apow 1 -fpow 2 -bnum 3 casket
+ kctreemgr inform -st casket
+ kctreemgr set -add casket duffy 1231
+ kctreemgr set -add casket micky 0101
+ kctreemgr set casket fal 1007
+ kctreemgr set casket mikio 0211
+ kctreemgr set casket natsuki 0810
+ kctreemgr set casket micky ""
+ kctreemgr set -rep casket duffy 777
+ kctreemgr set -app casket duffy kukuku
+ kctreemgr remove casket micky
+ kctreemgr list -pv casket > check.out
+ kctreemgr set casket ryu 1
+ kctreemgr set casket ken 2
+ kctreemgr remove casket duffy
+ kctreemgr set casket ryu syo-ryu-ken
+ kctreemgr set casket ken tatsumaki-senpu-kyaku
+ kctreemgr set -inci casket int 1234
+ kctreemgr set -inci casket int 5678
+ kctreemgr set -incd casket double 1234.5678
+ kctreemgr set -incd casket double 8765.4321
+ kctreemgr get casket mikio
+ kctreemgr get casket ryu
+ kctreemgr import casket lab/numbers.tsv
+ kctreemgr list -des -pv -px casket > check.out
+ kctreemgr copy casket casket-para
+ kctreemgr dump casket check.out
+ kctreemgr load -otr casket check.out
+ kctreemgr defrag -onl casket
+ kctreemgr check -onr casket
+ kctreemgr inform -st casket
+ kctreemgr create -otr -otl -onr -apow 1 -fpow 3 \
+ -ts -tl -tc -bnum 1 casket
+ kctreemgr import casket < lab/numbers.tsv
+ kctreemgr set casket mikio kyotocabinet
+ kctreemgr set -app casket tako ikaunini
+ kctreemgr set -app casket mikio kyototyrant
+ kctreemgr set -app casket mikio kyotodystopia
+ kctreemgr get -px casket mikio > check.out
+ kctreemgr list casket > check.out
+ kctreemgr check -onr casket
+ -del casket* /F /Q > NUL: 2>&1
+ kctreetest order -set \
+ -psiz 100 -bnum 5000 -msiz 50000 -pccap 100k casket 10000
+ kctreetest order -get \
+ -msiz 50000 -pccap 100k casket 10000
+ kctreetest order -getw \
+ -msiz 5000 -pccap 100k casket 10000
+ kctreetest order -rem \
+ -msiz 50000 -pccap 100k casket 10000
+ kctreetest order \
+ -bnum 5000 -psiz 100 -msiz 50000 -pccap 100k casket 10000
+ kctreetest order -etc \
+ -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 10000
+ kctreetest order -th 4 \
+ -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 10000
+ kctreetest order -th 4 -pccap 100k -rnd -etc \
+ -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k -rcd casket 10000
+ kctreemgr check -onr casket
+ kctreetest order -th 4 -rnd -etc -tran \
+ -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 1000
+ kctreemgr check -onr casket
+ kctreetest order -th 4 -rnd -etc -oat \
+ -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 -pccap 100k casket 1000
+ kctreemgr check -onr casket
+ kctreetest order -th 4 -rnd -etc \
+ -apow 2 -fpow 3 -ts -tl -tc -bnum 5000 -psiz 1000 -msiz 50000 -dfunit 4 casket 10000
+ kctreemgr check -onr casket
+ kctreetest queue \
+ -bnum 5000 -psiz 500 -msiz 50000 casket 10000
+ kctreemgr check -onr casket
+ kctreetest queue -rnd \
+ -bnum 5000 -psiz 500 -msiz 50000 casket 10000
+ kctreemgr check -onr casket
+ kctreetest queue -th 4 -it 4 \
+ -bnum 5000 -psiz 500 -msiz 50000 casket 10000
+ kctreemgr check -onr casket
+ kctreetest queue -th 4 -it 4 -rnd \
+ -bnum 5000 -psiz 500 -msiz 50000 casket 10000
+ kctreemgr check -onr casket
+ kctreetest wicked \
+ -bnum 5000 -psiz 1000 -msiz 50000 -pccap 100k casket 10000
+ kctreemgr check -onr casket
+ kctreetest wicked -th 4 -it 4 \
+ -bnum 5000 -msiz 50000 -dfunit 4 -pccap 100k -rcd casket 10000
+ kctreemgr check -onr casket
+ kctreetest wicked -th 4 -it 4 -oat \
+ -bnum 5000 -msiz 50000 -dfunit 4 -pccap 100k casket 1000
+ kctreemgr check -onr casket
+ kctreetest wicked -th 4 -it 4 \
+ -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 casket 1000
+ kctreemgr check -onr casket
+ kctreetest tran casket 10000
+ kctreetest tran -th 2 -it 4 -pccap 100k casket 10000
+ kctreetest tran -th 2 -it 4 \
+ -apow 2 -fpow 3 -ts -tl -tc -bnum 10000 -msiz 50000 -dfunit 4 -rcd casket 10000
+
+
+check-dir :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcdirmgr create -otr casket
+ kcdirmgr inform -st casket
+ kcdirmgr set -add casket duffy 1231
+ kcdirmgr set -add casket micky 0101
+ kcdirmgr set casket fal 1007
+ kcdirmgr set casket mikio 0211
+ kcdirmgr set casket natsuki 0810
+ kcdirmgr set casket micky ""
+ kcdirmgr set -rep casket duffy 777
+ kcdirmgr set -app casket duffy kukuku
+ kcdirmgr remove casket micky
+ kcdirmgr list -pv casket > check.out
+ kcdirmgr set casket ryu 1
+ kcdirmgr set casket ken 2
+ kcdirmgr remove casket duffy
+ kcdirmgr set casket ryu syo-ryu-ken
+ kcdirmgr set casket ken tatsumaki-senpu-kyaku
+ kcdirmgr set -inci casket int 1234
+ kcdirmgr set -inci casket int 5678
+ kcdirmgr set -incd casket double 1234.5678
+ kcdirmgr set -incd casket double 8765.4321
+ kcdirmgr get casket mikio
+ kcdirmgr get casket ryu
+ kcdirmgr import casket lab/numbers.tsv
+ kcdirmgr list -pv -px casket > check.out
+ kcdirmgr copy casket casket-para
+ kcdirmgr dump casket check.out
+ kcdirmgr load -otr casket check.out
+ kcdirmgr check -onr casket
+ kcdirmgr inform -st casket
+ kcdirmgr create -otr -otl -onr -tc casket
+ kcdirmgr import casket < lab/numbers.tsv
+ kcdirmgr set casket mikio kyotocabinet
+ kcdirmgr set -app casket tako ikaunini
+ kcdirmgr set -app casket mikio kyototyrant
+ kcdirmgr set -app casket mikio kyotodystopia
+ kcdirmgr get -px casket mikio > check.out
+ kcdirmgr list casket > check.out
+ kcdirmgr check -onr casket
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcdirtest order -set casket 500
+ kcdirtest order -get casket 500
+ kcdirtest order -getw casket 500
+ kcdirtest order -rem casket 500
+ kcdirtest order casket 500
+ kcdirtest order -etc casket 500
+ kcdirtest order -th 4 casket 500
+ kcdirtest order -th 4 -rnd -etc casket 500
+ kcdirmgr check -onr casket
+ kcdirtest order -th 4 -rnd -etc -tran casket 500
+ kcdirmgr check -onr casket
+ kcdirtest order -th 4 -rnd -etc -oat casket 500
+ kcdirmgr check -onr casket
+ kcdirtest order -th 4 -rnd -etc -tc casket 500
+ kcdirmgr check -onr casket
+ kcdirtest queue casket 500
+ kcdirmgr check -onr casket
+ kcdirtest queue -rnd casket 500
+ kcdirmgr check -onr casket
+ kcdirtest queue -th 4 -it 4 casket 500
+ kcdirmgr check -onr casket
+ kcdirtest queue -th 4 -it 4 -rnd casket 500
+ kcdirmgr check -onr casket
+ kcdirtest wicked casket 500
+ kcdirmgr check -onr casket
+ kcdirtest wicked -th 4 -it 4 casket 500
+ kcdirmgr check -onr casket
+ kcdirtest wicked -th 4 -it 4 -oat casket 500
+ kcdirmgr check -onr casket
+ kcdirtest wicked -th 4 -it 4 -tc casket 500
+ kcdirmgr check -onr casket
+ kcdirtest tran casket 500
+ kcdirtest tran -th 2 -it 4 casket 500
+ kcdirtest tran -th 2 -it 4 -tc casket 500
+
+
+check-forest :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcforestmgr create -otr -bnum 3 casket
+ kcforestmgr inform -st casket
+ kcforestmgr set -add casket duffy 1231
+ kcforestmgr set -add casket micky 0101
+ kcforestmgr set casket fal 1007
+ kcforestmgr set casket mikio 0211
+ kcforestmgr set casket natsuki 0810
+ kcforestmgr set casket micky ""
+ kcforestmgr set -rep casket duffy 777
+ kcforestmgr set -app casket duffy kukuku
+ kcforestmgr remove casket micky
+ kcforestmgr list -pv casket > check.out
+ kcforestmgr set casket ryu 1
+ kcforestmgr set casket ken 2
+ kcforestmgr remove casket duffy
+ kcforestmgr set casket ryu syo-ryu-ken
+ kcforestmgr set casket ken tatsumaki-senpu-kyaku
+ kcforestmgr set -inci casket int 1234
+ kcforestmgr set -inci casket int 5678
+ kcforestmgr set -incd casket double 1234.5678
+ kcforestmgr set -incd casket double 8765.4321
+ kcforestmgr get casket mikio
+ kcforestmgr get casket ryu
+ kcforestmgr import casket lab/numbers.tsv
+ kcforestmgr list -des -pv -px casket > check.out
+ kcforestmgr copy casket casket-para
+ kcforestmgr dump casket check.out
+ kcforestmgr load -otr casket check.out
+ kcforestmgr check -onr casket
+ kcforestmgr inform -st casket
+ kcforestmgr create -otr -otl -onr \
+ -tc -bnum 1 casket
+ kcforestmgr import casket < lab/numbers.tsv
+ kcforestmgr set casket mikio kyotocabinet
+ kcforestmgr set -app casket tako ikaunini
+ kcforestmgr set -app casket mikio kyototyrant
+ kcforestmgr set -app casket mikio kyotodystopia
+ kcforestmgr get -px casket mikio > check.out
+ kcforestmgr list casket > check.out
+ kcforestmgr check -onr casket
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcforesttest order -set \
+ -psiz 100 -bnum 5000 -pccap 100k casket 5000
+ kcforesttest order -get \
+ -pccap 100k casket 5000
+ kcforesttest order -getw \
+ -pccap 100k casket 5000
+ kcforesttest order -rem \
+ -pccap 100k casket 5000
+ kcforesttest order \
+ -bnum 5000 -psiz 100 -pccap 100k casket 5000
+ kcforesttest order -etc \
+ -bnum 5000 -psiz 1000 -pccap 100k casket 5000
+ kcforesttest order -th 4 \
+ -bnum 5000 -psiz 1000 -pccap 100k casket 5000
+ kcforesttest order -th 4 -pccap 100k -rnd -etc \
+ -bnum 5000 -psiz 1000 -pccap 100k -rcd casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest order -th 4 -rnd -etc -tran \
+ -bnum 500 -psiz 1000 -pccap 100k casket 500
+ kcforestmgr check -onr casket
+ kcforesttest order -th 4 -rnd -etc -oat \
+ -bnum 500 -psiz 1000 -pccap 100k casket 500
+ kcforestmgr check -onr casket
+ kcforesttest order -th 4 -rnd -etc \
+ -tc -bnum 5000 -psiz 1000 casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest queue \
+ -bnum 5000 -psiz 500 casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest queue -rnd \
+ -bnum 5000 -psiz 500 casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest queue -th 4 -it 4 \
+ -bnum 5000 -psiz 500 casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest queue -th 4 -it 4 -rnd \
+ -bnum 5000 -psiz 500 casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest wicked \
+ -bnum 5000 -psiz 1000 -pccap 100k casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest wicked -th 4 -it 4 \
+ -bnum 5000 -pccap 100k -rcd casket 5000
+ kcforestmgr check -onr casket
+ kcforesttest wicked -th 4 -it 4 -oat \
+ -bnum 5000 -pccap 100k casket 500
+ kcforestmgr check -onr casket
+ kcforesttest wicked -th 4 -it 4 \
+ -tc -bnum 500 casket 500
+ kcforestmgr check -onr casket
+ kcforesttest tran casket 5000
+ kcforesttest tran -th 2 -it 4 -pccap 100k casket 5000
+ kcforesttest tran -th 2 -it 4 \
+ -tc -bnum 5000 -rcd casket 5000
+
+
+check-poly :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolymgr create -otr "casket.kch#apow=1#fpow=2#bnum=3"
+ kcpolymgr inform -st casket.kch
+ kcpolymgr set -add casket.kch duffy 1231
+ kcpolymgr set -add casket.kch micky 0101
+ kcpolymgr set casket.kch fal 1007
+ kcpolymgr set casket.kch mikio 0211
+ kcpolymgr set casket.kch natsuki 0810
+ kcpolymgr set casket.kch micky ""
+ kcpolymgr set -app casket.kch duffy kukuku
+ kcpolymgr remove casket.kch micky
+ kcpolymgr list -pv casket.kch > check.out
+ kcpolymgr copy casket.kch casket-para
+ kcpolymgr dump casket.kch check.out
+ kcpolymgr load -otr casket.kch check.out
+ kcpolymgr set casket.kch ryu 1
+ kcpolymgr set casket.kch ken 2
+ kcpolymgr remove casket.kch duffy
+ kcpolymgr set casket.kch ryu syo-ryu-ken
+ kcpolymgr set casket.kch ken tatsumaki-senpu-kyaku
+ kcpolymgr set -inci casket.kch int 1234
+ kcpolymgr set -inci casket.kch int 5678
+ kcpolymgr set -incd casket.kch double 1234.5678
+ kcpolymgr set -incd casket.kch double 8765.4321
+ kcpolymgr get "casket.kch" mikio
+ kcpolymgr get "casket.kch" ryu
+ kcpolymgr import casket.kch lab/numbers.tsv
+ kcpolymgr list -pv -px "casket.kch#mode=r" > check.out
+ kcpolymgr check -onr casket.kch
+ kcpolymgr inform -st casket.kch
+ kcpolymgr create -otr -otl -onr \
+ "casket.kct#apow=1#fpow=3#opts=slc#bnum=1"
+ kcpolymgr import casket.kct < lab/numbers.tsv
+ kcpolymgr set casket.kct mikio kyotocabinet
+ kcpolymgr set -app casket.kct tako ikaunini
+ kcpolymgr set -app casket.kct mikio kyototyrant
+ kcpolymgr set -app casket.kct mikio kyotodystopia
+ kcpolymgr get -px casket.kct mikio > check.out
+ kcpolymgr list casket.kct > check.out
+ kcpolymgr check -onr casket.kct
+ -del casket* /F /Q > NUL: 2>&1
+ kcpolytest order -set "casket.kct#bnum=5000#msiz=50000" 10000
+ kcpolytest order -get "casket.kct#msiz=50000" 10000
+ kcpolytest order -getw "casket.kct#msiz=5000" 10000
+ kcpolytest order -rem "casket.kct#msiz=50000" 10000
+ kcpolytest order "casket.kct#bnum=5000#msiz=50000" 10000
+ kcpolytest order -etc \
+ "casket.kct#bnum=5000#msiz=50000#dfunit=4" 10000
+ kcpolytest order -th 4 \
+ "casket.kct#bnum=5000#msiz=50000#dfunit=4" 10000
+ kcpolytest order -th 4 -rnd -etc \
+ "casket.kct#bnum=5000#msiz=0#dfunit=1" 1000
+ kcpolymgr check -onr casket.kct
+ kcpolytest order -th 4 -rnd -etc -tran \
+ "casket.kct#bnum=5000#msiz=0#dfunit=2" 1000
+ kcpolymgr check -onr casket.kct
+ kcpolytest order -th 4 -rnd -etc -oat \
+ "casket.kct#bnum=5000#msiz=0#dfunit=3" 1000
+ kcpolymgr check -onr casket.kct
+ kcpolytest order -th 4 -rnd -etc \
+ "casket.kct#apow=2#fpow=3#opts=slc#bnum=5000#msiz=0#dfunit=4" 1000
+ kcpolymgr check -onr casket.kct
+ kcpolytest queue \
+ "casket.kct#bnum=5000#msiz=0" 10000
+ kcpolymgr check -onr casket.kct
+ kcpolytest queue -rnd \
+ "casket.kct#bnum=5000#msiz=0" 10000
+ kcpolymgr check -onr casket.kct
+ kcpolytest queue -th 4 -it 4 \
+ "casket.kct#bnum=5000#msiz=0" 10000
+ kcpolymgr check -onr casket.kct
+ kcpolytest queue -th 4 -it 4 -rnd \
+ "casket.kct#bnum=5000#msiz=0" 10000
+ kcpolymgr check -onr casket.kct
+ kcpolytest wicked "casket.kct#bnum=5000#msiz=0" 1000
+ kcpolymgr check -onr casket.kct
+ kcpolytest wicked -th 4 -it 4 \
+ "casket.kct#bnum=5000#msiz=0#dfunit=1" 1000
+ kcpolymgr check -onr casket.kct
+ kcpolytest wicked -th 4 -it 4 -oat \
+ "casket.kct#bnum=5000#msiz=0#dfunit=1" 1000
+ kcpolymgr check -onr casket.kct
+ kcpolytest wicked -th 4 -it 4 \
+ "casket.kct#apow=2#fpow=3#opts=slc#bnum=10000#msiz=0#dfunit=1" 10000
+ kcpolymgr check -onr casket.kct
+ kcpolytest tran casket.kct 10000
+ kcpolytest tran -th 2 -it 4 casket.kct 10000
+ kcpolytest tran -th 2 -it 4 \
+ "casket.kct#apow=2#fpow=3#opts=slc#bnum=10000#msiz=0#dfunit=1" 1000
+ kcpolytest mapred -dbnum 2 -clim 10k casket.kct 10000
+ kcpolytest mapred -tmp . -dbnum 2 -clim 10k -xnl -xnc \
+ casket.kct 10000
+ kcpolytest mapred -tmp . -dbnum 2 -clim 10k -xpm -xpr -xpf -xnc \
+ casket.kct 10000
+ kcpolytest mapred -rnd -dbnum 2 -clim 10k casket.kct 10000
+ kcpolytest index -set "casket.kct#idxclim=32k" 10000
+ kcpolytest index -get "casket.kct" 10000
+ kcpolytest index -rem "casket.kct" 10000
+ kcpolytest index -etc "casket.kct#idxclim=32k" 10000
+ kcpolytest index -th 4 -rnd -set \
+ "casket.kct#idxclim=32k#idxdbnum=4" 10000
+ kcpolytest index -th 4 -rnd -get "casket.kct" 10000
+ kcpolytest index -th 4 -rnd -rem "casket.kct" 10000
+ kcpolytest index -th 4 -rnd -etc \
+ "casket.kct#idxclim=32k#idxdbnum=4" 10000
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolytest order -rnd "casket.kcx" 10000
+ kcpolytest order -th 4 -rnd "casket.kcx" 10000
+ kcpolytest wicked "casket.kcx" 10000
+ kcpolytest wicked -th 4 "casket.kcx" 10000
+ kcpolymgr list "casket.kcx" > check.in
+ kcpolymgr list -max 1000 "casket.kcx" > check.in
+ kcpolytest mapred "casket.kcx" 10000
+ kcpolytest mapred -xpm -xpr -xpf "casket.kcx" 10000
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolytest order -rnd "casket.kch#opts=s#bnum=256" 1000
+ kcpolytest order -rnd "casket.kct#opts=l#psiz=256" 1000
+ kcpolytest order -rnd "casket.kcd#opts=c#bnum=256" 500
+ kcpolytest order -rnd "casket.kcf#opts=c#psiz=256" 500
+ kcpolytest order -rnd "casket.kcx" 500
+ kcpolymgr merge -add "casket#type=kct" \
+ casket.kch casket.kct casket.kcd casket.kcf casket.kcx
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolytest misc "casket#type=-"
+ kcpolytest misc "casket#type=+"
+ kcpolytest misc "casket#type=:"
+ kcpolytest misc "casket#type=*"
+ kcpolytest misc "casket#type=%"
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolytest misc "casket#type=kch#log=-#logkinds=debug#mtrg=-#zcomp=lzocrc"
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolytest misc "casket#type=kct#log=-#logkinds=debug#mtrg=-#zcomp=lzmacrc"
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolytest misc "casket#type=kcd#zcomp=arc#zkey=mikio"
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kcpolytest misc "casket#type=kcf#zcomp=arc#zkey=mikio"
+
+
+check-langc :
+ -del casket* /F /Q > NUL: 2>&1
+ -rd casket casket.wal casket.tmp casket-para casket.kcd casket.kcf /S /Q > NUL: 2>&1
+ kclangctest order "casket.kch#bnum=5000#msiz=50000" 10000
+ kclangctest order -etc \
+ "casket.kch#bnum=5000#msiz=50000#dfunit=2" 10000
+ kclangctest order -rnd -etc \
+ "casket.kch#bnum=5000#msiz=50000#dfunit=2" 10000
+ kclangctest order -rnd -etc -oat -tran \
+ "casket.kch#bnum=5000#msiz=50000#dfunit=2#zcomp=arcz" 10000
+ kclangctest index "casket.kct#bnum=5000#msiz=50000" 10000
+ kclangctest index -etc \
+ "casket.kct#bnum=5000#msiz=50000#dfunit=2" 10000
+ kclangctest index -rnd -etc \
+ "casket.kct#bnum=5000#msiz=50000#dfunit=2" 10000
+ kclangctest index -rnd -etc -oat \
+ "casket.kct#bnum=5000#msiz=50000#dfunit=2#zcomp=arcz" 10000
+ kclangctest map 10000
+ kclangctest map -etc -bnum 1000 10000
+ kclangctest map -etc -rnd -bnum 1000 10000
+ kclangctest list 10000
+ kclangctest list -etc 10000
+ kclangctest list -etc -rnd 10000
+
+
+check-forever :
+ lab\vcmakecheck
+
+
+binpkg :
+ -rd kcwin32 /S /Q > NUL: 2>&1
+ md kcwin32
+ md kcwin32\include
+ copy *.h kcwin32\include
+ del kcwin32\include\myconf.h
+ del kcwin32\include\cmdcommon.h
+ md kcwin32\lib
+ copy *.lib kcwin32\lib
+ md kcwin32\bin
+ copy *.exe kcwin32\bin
+ xcopy /S /E /I doc kcwin32\doc
+
+
+
+#================================================================
+# Building binaries
+#================================================================
+
+
+kyotocabinet.lib : $(LIBOBJFILES)
+ $(LIB) $(LIBFLAGS) /OUT:$@ $(LIBOBJFILES)
+
+
+kcutiltest.exe : kcutiltest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcutiltest.obj kyotocabinet.lib
+
+
+kcutilmgr.exe : kcutilmgr.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcutilmgr.obj kyotocabinet.lib
+
+
+kcprototest.exe : kcprototest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcprototest.obj kyotocabinet.lib
+
+
+kcstashtest.exe : kcstashtest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcstashtest.obj kyotocabinet.lib
+
+
+kccachetest.exe : kccachetest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kccachetest.obj kyotocabinet.lib
+
+
+kcgrasstest.exe : kcgrasstest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcgrasstest.obj kyotocabinet.lib
+
+
+kchashtest.exe : kchashtest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kchashtest.obj kyotocabinet.lib
+
+
+kchashmgr.exe : kchashmgr.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kchashmgr.obj kyotocabinet.lib
+
+
+kctreetest.exe : kctreetest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kctreetest.obj kyotocabinet.lib
+
+
+kctreemgr.exe : kctreemgr.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kctreemgr.obj kyotocabinet.lib
+
+
+kcdirtest.exe : kcdirtest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcdirtest.obj kyotocabinet.lib
+
+
+kcdirmgr.exe : kcdirmgr.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcdirmgr.obj kyotocabinet.lib
+
+
+kcforesttest.exe : kcforesttest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcforesttest.obj kyotocabinet.lib
+
+
+kcforestmgr.exe : kcforestmgr.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcforestmgr.obj kyotocabinet.lib
+
+
+kcpolytest.exe : kcpolytest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcpolytest.obj kyotocabinet.lib
+
+
+kcpolymgr.exe : kcpolymgr.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kcpolymgr.obj kyotocabinet.lib
+
+
+kclangctest.exe : kclangctest.obj kyotocabinet.lib
+ $(LINK) $(LINKFLAGS) /OUT:$@ kclangctest.obj kyotocabinet.lib
+
+
+kcutil.obj : kccommon.h kcutil.h myconf.h
+
+kcdb.obj : kccommon.h kcutil.h kcdb.h myconf.h
+
+kcthread.obj : kccommon.h kcutil.h kcthread.h myconf.h
+
+kcfile.obj : kccommon.h kcutil.h kcthread.h kcfile.h myconf.h
+
+kccompress.obj : kccommon.h kcutil.h kccompress.h myconf.h
+
+kccompare.obj : kccommon.h kcutil.h kccompare.h myconf.h
+
+kcmap.obj : kccommon.h kcutil.h kcmap.h myconf.h
+
+kcregex.obj : kccommon.h kcutil.h kcregex.h myconf.h
+
+kcplantdb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h
+
+kcprotodb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcprotodb.h
+
+kcstashdb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcstashdb.h
+
+kccachedb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kccachedb.h
+
+kchashdb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kchashdb.h
+
+kcdirdb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcdirdb.h
+
+kctextdb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kctextdb.h
+
+kcpolydb.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h kcpolydb.h
+
+kcdbext.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \
+ kcpolydb.h kcdbext.h
+
+kclangc.obj : kccommon.h kcutil.h kcdb.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \
+ kcpolydb.h kcdbext.h kclangc.h
+
+kcutiltest.obj kcutilmgr.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ cmdcommon.h
+
+kcprototest.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcprotodb.h cmdcommon.h
+
+kcstashtest.obj kcgrasstest.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcstashdb.h cmdcommon.h
+
+kccachetest.obj kcgrasstest.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kccachedb.h cmdcommon.h
+
+kchashtest.obj kchashmgr.obj kctreetest.obj kctreemgr.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kchashdb.h cmdcommon.h
+
+kcdirtest.obj kcdirmgr.obj kcforesttest.obj kcforestmgr.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcdirdb.h cmdcommon.h
+
+kcpolytest.obj kcpolymgr.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \
+ kcpolydb.h kcdbext.h cmdcommon.h
+
+kclangctest.obj : \
+ kccommon.h kcdb.h kcutil.h kcthread.h kcfile.h kccompress.h kccompare.h \
+ kcmap.h kcregex.h \
+ kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h kchashdb.h kcdirdb.h kctextdb.h \
+ kcpolydb.h kcdbext.h kclangc.h
+
+
+
+# END OF FILE
diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/cmdcommon.h b/plugins/Dbx_kyoto/src/kyotocabinet/cmdcommon.h new file mode 100644 index 0000000000..b6fe365c47 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/cmdcommon.h @@ -0,0 +1,322 @@ +/************************************************************************************************* + * Common symbols for command line utilities + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _CMDCOMMON_H // duplication check +#define _CMDCOMMON_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> + +#if !defined(_KC_PREFIX) +#define _KC_PREFIX "*" +#endif +#if !defined(_KC_INCLUDEDIR) +#define _KC_INCLUDEDIR "*" +#endif +#if !defined(_KC_LIBDIR) +#define _KC_LIBDIR "*" +#endif +#if !defined(_KC_BINDIR) +#define _KC_BINDIR "*" +#endif +#if !defined(_KC_LIBEXECDIR) +#define _KC_LIBEXECDIR "*" +#endif +#if !defined(_KC_APPINC) +#define _KC_APPINC "*" +#endif +#if !defined(_KC_APPLIBS) +#define _KC_APPLIBS "*" +#endif + +namespace kc = kyotocabinet; + + +// constants +const int32_t THREADMAX = 64; // maximum number of threads +const size_t RECBUFSIZ = 64; // buffer size for a record +const size_t RECBUFSIZL = 1024; // buffer size for a long record + + +// global variables +uint64_t g_rnd_x = 123456789; +uint64_t g_rnd_y = 362436069; +uint64_t g_rnd_z = 521288629; +uint64_t g_rnd_w = 88675123; + + +// function prototypes +void mysrand(int64_t seed); +int64_t myrand(int64_t range); +int64_t memusage(); +void oprintf(const char* format, ...); +void oputchar(char c); +void eprintf(const char* format, ...); +void printversion(); +void printdata(const char* buf, int32_t size, bool px); +bool mygetline(std::istream* is, std::string* str); +std::string unitnumstr(int64_t num); +std::string unitnumstrbyte(int64_t num); +kc::BasicDB::ProgressChecker* stdchecker(const char* prefix, std::ostream* strm); +kc::BasicDB::Logger* stdlogger(const char* progname, std::ostream* strm); +void printdb(kc::BasicDB* db, bool px = false); + + +// checker to show progress by printing dots +class DotChecker : public kc::BasicDB::ProgressChecker { + public: + explicit DotChecker(std::ostream* strm, int64_t freq) : strm_(strm), freq_(freq), cnt_(0) {} + int64_t count() { + return cnt_; + } + private: + bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) { + if (std::strcmp(message, "processing") || freq_ == 0) return true; + if (freq_ < 0) { + cnt_++; + if (cnt_ % -freq_ == 0) { + oputchar('.'); + if (cnt_ % (-freq_ * 50) == 0) oprintf(" (%lld)\n", (long long)cnt_); + } + } else { + if (curcnt > cnt_) { + cnt_ = curcnt; + if (cnt_ % freq_ == 0) { + oputchar('.'); + if (cnt_ % (freq_ * 50) == 0) oprintf(" (%lld)\n", (long long)cnt_); + } + } + } + return true; + } + std::ostream* strm_; + int64_t freq_; + int64_t cnt_; +}; + + +// get the random seed +inline void mysrand(int64_t seed) { + g_rnd_x = seed; + for (int32_t i = 0; i < 16; i++) { + myrand(1); + } +} + + +// get a random number +inline int64_t myrand(int64_t range) { + uint64_t t = g_rnd_x ^ (g_rnd_x << 11); + g_rnd_x = g_rnd_y; + g_rnd_y = g_rnd_z; + g_rnd_z = g_rnd_w; + g_rnd_w = (g_rnd_w ^ (g_rnd_w >> 19)) ^ (t ^ (t >> 8)); + return (g_rnd_w & kc::INT64MAX) % range; +} + + +// get the current memory usage +inline int64_t memusage() { + std::map<std::string, std::string> info; + kc::getsysinfo(&info); + return kc::atoi(info["mem_rss"].c_str()); +} + + +// print formatted information string and flush the buffer +inline void oprintf(const char* format, ...) { + std::string msg; + va_list ap; + va_start(ap, format); + kc::vstrprintf(&msg, format, ap); + va_end(ap); + std::cout << msg; + std::cout.flush(); +} + + +// print a character and flush the buffer +inline void oputchar(char c) { + std::cout << c; + std::cout.flush(); +} + + +// print formatted error string and flush the buffer +inline void eprintf(const char* format, ...) { + std::string msg; + va_list ap; + va_start(ap, format); + kc::vstrprintf(&msg, format, ap); + va_end(ap); + std::cerr << msg; + std::cerr.flush(); +} + + +// print the versin information +inline void printversion() { + oprintf("Kyoto Cabinet %s (%d.%d:%d) on %s\n", + kc::VERSION, kc::LIBVER, kc::LIBREV, kc::FMTVER, kc::OSNAME); +} + + +// print record data +inline void printdata(const char* buf, int32_t size, bool px) { + size_t cnt = 0; + char numbuf[kc::NUMBUFSIZ]; + while (size-- > 0) { + if (px) { + if (cnt++ > 0) putchar(' '); + std::sprintf(numbuf, "%02X", *(unsigned char*)buf); + std::cout << numbuf; + } else { + std::cout << *buf; + } + buf++; + } +} + + +// read a line from a file descriptor +inline bool mygetline(std::istream* is, std::string* str) { + str->clear(); + bool hit = false; + char c; + while (is->get(c)) { + hit = true; + if (c == '\0' || c == '\r') continue; + if (c == '\n') break; + str->append(1, c); + } + return hit; +} + + +// convert a number into the string with the decimal unit +inline std::string unitnumstr(int64_t num) { + if (num >= std::pow(1000.0, 6)) { + return kc::strprintf("%.3Lf quintillion", (long double)num / std::pow(1000.0, 6)); + } else if (num >= std::pow(1000.0, 5)) { + return kc::strprintf("%.3Lf quadrillion", (long double)num / std::pow(1000.0, 5)); + } else if (num >= std::pow(1000.0, 4)) { + return kc::strprintf("%.3Lf trillion", (long double)num / std::pow(1000.0, 4)); + } else if (num >= std::pow(1000.0, 3)) { + return kc::strprintf("%.3Lf billion", (long double)num / std::pow(1000.0, 3)); + } else if (num >= std::pow(1000.0, 2)) { + return kc::strprintf("%.3Lf million", (long double)num / std::pow(1000.0, 2)); + } else if (num >= std::pow(1000.0, 1)) { + return kc::strprintf("%.3Lf thousand", (long double)num / std::pow(1000.0, 1)); + } + return kc::strprintf("%lld", (long long)num); +} + + +// convert a number into the string with the byte unit +inline std::string unitnumstrbyte(int64_t num) { + if ((unsigned long long)num >= 1ULL << 60) { + return kc::strprintf("%.3Lf EiB", (long double)num / (1ULL << 60)); + } else if ((unsigned long long)num >= 1ULL << 50) { + return kc::strprintf("%.3Lf PiB", (long double)num / (1ULL << 50)); + } else if ((unsigned long long)num >= 1ULL << 40) { + return kc::strprintf("%.3Lf TiB", (long double)num / (1ULL << 40)); + } else if ((unsigned long long)num >= 1ULL << 30) { + return kc::strprintf("%.3Lf GiB", (long double)num / (1ULL << 30)); + } else if ((unsigned long long)num >= 1ULL << 20) { + return kc::strprintf("%.3Lf MiB", (long double)num / (1ULL << 20)); + } else if ((unsigned long long)num >= 1ULL << 10) { + return kc::strprintf("%.3Lf KiB", (long double)num / (1ULL << 10)); + } + return kc::strprintf("%lld B", (long long)num); +} + + +// get the progress checker to print the parameters +inline kc::BasicDB::ProgressChecker* stdchecker(const char* prefix, std::ostream* strm) { + class CheckerImpl : public kc::BasicDB::ProgressChecker { + public: + explicit CheckerImpl(std::ostream* strm, const char* prefix) : + strm_(strm), prefix_(prefix) {} + bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) { + *strm_ << prefix_ << ": " << name << ": " << message << ": " << + curcnt << "/" << allcnt << std::endl; + return true; + } + private: + std::ostream* strm_; + const char* prefix_; + }; + static CheckerImpl checker(strm, prefix); + return &checker; +} + + +// get the logger into the standard stream +inline kc::BasicDB::Logger* stdlogger(const char* prefix, std::ostream* strm) { + class LoggerImpl : public kc::BasicDB::Logger { + public: + explicit LoggerImpl(std::ostream* strm, const char* prefix) : + strm_(strm), prefix_(prefix) {} + void log(const char* file, int32_t line, const char* func, Kind kind, + const char* message) { + const char* kstr = "MISC"; + switch (kind) { + case kc::BasicDB::Logger::DEBUG: kstr = "DEBUG"; break; + case kc::BasicDB::Logger::INFO: kstr = "INFO"; break; + case kc::BasicDB::Logger::WARN: kstr = "WARN"; break; + case kc::BasicDB::Logger::ERROR: kstr = "ERROR"; break; + } + *strm_ << prefix_ << ": [" << kstr << "]: " << + file << ": " << line << ": " << func << ": " << message << std::endl; + } + private: + std::ostream* strm_; + const char* prefix_; + }; + static LoggerImpl logger(strm, prefix); + return &logger; +} + + +// print all record of a database +inline void printdb(kc::BasicDB* db, bool px) { + class Printer : public kc::DB::Visitor { + public: + explicit Printer(bool px) : px_(px) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + printdata(kbuf, ksiz, px_); + oputchar('\t'); + printdata(vbuf, vsiz, px_); + oputchar('\n'); + return NOP; + } + bool px_; + } printer(px); + db->iterate(&printer, false); +} + + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/configure b/plugins/Dbx_kyoto/src/kyotocabinet/configure new file mode 100644 index 0000000000..a07063dd4b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/configure @@ -0,0 +1,5777 @@ +#! /bin/sh +# Guess values for system-dependent variables and create Makefiles. +# Generated by GNU Autoconf 2.68 for kyotocabinet 1.2.76. +# +# +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, +# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software +# Foundation, Inc. +# +# +# This configure script is free software; the Free Software Foundation +# gives unlimited permission to copy, distribute and modify it. +## -------------------- ## +## M4sh Initialization. ## +## -------------------- ## + +# Be more Bourne compatible +DUALCASE=1; export DUALCASE # for MKS sh +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in #( + *posix*) : + set -o posix ;; #( + *) : + ;; +esac +fi + + +as_nl=' +' +export as_nl +# Printing a long string crashes Solaris 7 /usr/bin/printf. +as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo +# Prefer a ksh shell builtin over an external printf program on Solaris, +# but without wasting forks for bash or zsh. +if test -z "$BASH_VERSION$ZSH_VERSION" \ + && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='print -r --' + as_echo_n='print -rn --' +elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='printf %s\n' + as_echo_n='printf %s' +else + if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then + as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' + as_echo_n='/usr/ucb/echo -n' + else + as_echo_body='eval expr "X$1" : "X\\(.*\\)"' + as_echo_n_body='eval + arg=$1; + case $arg in #( + *"$as_nl"*) + expr "X$arg" : "X\\(.*\\)$as_nl"; + arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; + esac; + expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" + ' + export as_echo_n_body + as_echo_n='sh -c $as_echo_n_body as_echo' + fi + export as_echo_body + as_echo='sh -c $as_echo_body as_echo' +fi + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + PATH_SEPARATOR=: + (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { + (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || + PATH_SEPARATOR=';' + } +fi + + +# IFS +# We need space, tab and new line, in precisely that order. Quoting is +# there to prevent editors from complaining about space-tab. +# (If _AS_PATH_WALK were called with IFS unset, it would disable word +# splitting by setting IFS to empty value.) +IFS=" "" $as_nl" + +# Find who we are. Look in the path if we contain no directory separator. +as_myself= +case $0 in #(( + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break + done +IFS=$as_save_IFS + + ;; +esac +# We did not find ourselves, most probably we were run as `sh COMMAND' +# in which case we are not to be found in the path. +if test "x$as_myself" = x; then + as_myself=$0 +fi +if test ! -f "$as_myself"; then + $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + exit 1 +fi + +# Unset variables that we do not need and which cause bugs (e.g. in +# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" +# suppresses any "Segmentation fault" message there. '((' could +# trigger a bug in pdksh 5.2.14. +for as_var in BASH_ENV ENV MAIL MAILPATH +do eval test x\${$as_var+set} = xset \ + && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : +done +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +LC_ALL=C +export LC_ALL +LANGUAGE=C +export LANGUAGE + +# CDPATH. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +if test "x$CONFIG_SHELL" = x; then + as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then : + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which + # is contrary to our usage. Disable this feature. + alias -g '\${1+\"\$@\"}'='\"\$@\"' + setopt NO_GLOB_SUBST +else + case \`(set -o) 2>/dev/null\` in #( + *posix*) : + set -o posix ;; #( + *) : + ;; +esac +fi +" + as_required="as_fn_return () { (exit \$1); } +as_fn_success () { as_fn_return 0; } +as_fn_failure () { as_fn_return 1; } +as_fn_ret_success () { return 0; } +as_fn_ret_failure () { return 1; } + +exitcode=0 +as_fn_success || { exitcode=1; echo as_fn_success failed.; } +as_fn_failure && { exitcode=1; echo as_fn_failure succeeded.; } +as_fn_ret_success || { exitcode=1; echo as_fn_ret_success failed.; } +as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; } +if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then : + +else + exitcode=1; echo positional parameters were not saved. +fi +test x\$exitcode = x0 || exit 1" + as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO + as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO + eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" && + test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1 +test \$(( 1 + 1 )) = 2 || exit 1" + if (eval "$as_required") 2>/dev/null; then : + as_have_required=yes +else + as_have_required=no +fi + if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null; then : + +else + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +as_found=false +for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + as_found=: + case $as_dir in #( + /*) + for as_base in sh bash ksh sh5; do + # Try only shells that exist, to save several forks. + as_shell=$as_dir/$as_base + if { test -f "$as_shell" || test -f "$as_shell.exe"; } && + { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$as_shell"; } 2>/dev/null; then : + CONFIG_SHELL=$as_shell as_have_required=yes + if { $as_echo "$as_bourne_compatible""$as_suggested" | as_run=a "$as_shell"; } 2>/dev/null; then : + break 2 +fi +fi + done;; + esac + as_found=false +done +$as_found || { if { test -f "$SHELL" || test -f "$SHELL.exe"; } && + { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$SHELL"; } 2>/dev/null; then : + CONFIG_SHELL=$SHELL as_have_required=yes +fi; } +IFS=$as_save_IFS + + + if test "x$CONFIG_SHELL" != x; then : + # We cannot yet assume a decent shell, so we have to provide a + # neutralization value for shells without unset; and this also + # works around shells that cannot unset nonexistent variables. + # Preserve -v and -x to the replacement shell. + BASH_ENV=/dev/null + ENV=/dev/null + (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV + export CONFIG_SHELL + case $- in # (((( + *v*x* | *x*v* ) as_opts=-vx ;; + *v* ) as_opts=-v ;; + *x* ) as_opts=-x ;; + * ) as_opts= ;; + esac + exec "$CONFIG_SHELL" $as_opts "$as_myself" ${1+"$@"} +fi + + if test x$as_have_required = xno; then : + $as_echo "$0: This script requires a shell more modern than all" + $as_echo "$0: the shells that I found on your system." + if test x${ZSH_VERSION+set} = xset ; then + $as_echo "$0: In particular, zsh $ZSH_VERSION has bugs and should" + $as_echo "$0: be upgraded to zsh 4.3.4 or later." + else + $as_echo "$0: Please tell bug-autoconf@gnu.org about your system, +$0: including any error possibly output before this +$0: message. Then install a modern shell, or manually run +$0: the script under such a shell if you do have one." + fi + exit 1 +fi +fi +fi +SHELL=${CONFIG_SHELL-/bin/sh} +export SHELL +# Unset more variables known to interfere with behavior of common tools. +CLICOLOR_FORCE= GREP_OPTIONS= +unset CLICOLOR_FORCE GREP_OPTIONS + +## --------------------- ## +## M4sh Shell Functions. ## +## --------------------- ## +# as_fn_unset VAR +# --------------- +# Portably unset VAR. +as_fn_unset () +{ + { eval $1=; unset $1;} +} +as_unset=as_fn_unset + +# as_fn_set_status STATUS +# ----------------------- +# Set $? to STATUS, without forking. +as_fn_set_status () +{ + return $1 +} # as_fn_set_status + +# as_fn_exit STATUS +# ----------------- +# Exit the shell with STATUS, even in a "trap 0" or "set -e" context. +as_fn_exit () +{ + set +e + as_fn_set_status $1 + exit $1 +} # as_fn_exit + +# as_fn_mkdir_p +# ------------- +# Create "$as_dir" as a directory, including parents if necessary. +as_fn_mkdir_p () +{ + + case $as_dir in #( + -*) as_dir=./$as_dir;; + esac + test -d "$as_dir" || eval $as_mkdir_p || { + as_dirs= + while :; do + case $as_dir in #( + *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *) as_qdir=$as_dir;; + esac + as_dirs="'$as_qdir' $as_dirs" + as_dir=`$as_dirname -- "$as_dir" || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + test -d "$as_dir" && break + done + test -z "$as_dirs" || eval "mkdir $as_dirs" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" + + +} # as_fn_mkdir_p +# as_fn_append VAR VALUE +# ---------------------- +# Append the text in VALUE to the end of the definition contained in VAR. Take +# advantage of any shell optimizations that allow amortized linear growth over +# repeated appends, instead of the typical quadratic growth present in naive +# implementations. +if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : + eval 'as_fn_append () + { + eval $1+=\$2 + }' +else + as_fn_append () + { + eval $1=\$$1\$2 + } +fi # as_fn_append + +# as_fn_arith ARG... +# ------------------ +# Perform arithmetic evaluation on the ARGs, and store the result in the +# global $as_val. Take advantage of shells that can avoid forks. The arguments +# must be portable across $(()) and expr. +if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : + eval 'as_fn_arith () + { + as_val=$(( $* )) + }' +else + as_fn_arith () + { + as_val=`expr "$@" || test $? -eq 1` + } +fi # as_fn_arith + + +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- +# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are +# provided, also output the error to LOG_FD, referencing LINENO. Then exit the +# script with STATUS, using 1 if that was 0. +as_fn_error () +{ + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + fi + $as_echo "$as_me: error: $2" >&2 + as_fn_exit $as_status +} # as_fn_error + +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + +if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then + as_dirname=dirname +else + as_dirname=false +fi + +as_me=`$as_basename -- "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ + s//\1/ + q + } + /^X\/\(\/\/\)$/{ + s//\1/ + q + } + /^X\/\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + + + as_lineno_1=$LINENO as_lineno_1a=$LINENO + as_lineno_2=$LINENO as_lineno_2a=$LINENO + eval 'test "x$as_lineno_1'$as_run'" != "x$as_lineno_2'$as_run'" && + test "x`expr $as_lineno_1'$as_run' + 1`" = "x$as_lineno_2'$as_run'"' || { + # Blame Lee E. McMahon (1931-1989) for sed's syntax. :-) + sed -n ' + p + /[$]LINENO/= + ' <$as_myself | + sed ' + s/[$]LINENO.*/&-/ + t lineno + b + :lineno + N + :loop + s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/ + t loop + s/-\n.*// + ' >$as_me.lineno && + chmod +x "$as_me.lineno" || + { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } + + # Don't try to exec as it changes $[0], causing all sort of problems + # (the dirname of $[0] is not the place where we might find the + # original and so on. Autoconf is especially sensitive to this). + . "./$as_me.lineno" + # Exit status is that of the last command. + exit +} + +ECHO_C= ECHO_N= ECHO_T= +case `echo -n x` in #((((( +-n*) + case `echo 'xy\c'` in + *c*) ECHO_T=' ';; # ECHO_T is single tab character. + xy) ECHO_C='\c';; + *) echo `echo ksh88 bug on AIX 6.1` > /dev/null + ECHO_T=' ';; + esac;; +*) + ECHO_N='-n';; +esac + +rm -f conf$$ conf$$.exe conf$$.file +if test -d conf$$.dir; then + rm -f conf$$.dir/conf$$.file +else + rm -f conf$$.dir + mkdir conf$$.dir 2>/dev/null +fi +if (echo >conf$$.file) 2>/dev/null; then + if ln -s conf$$.file conf$$ 2>/dev/null; then + as_ln_s='ln -s' + # ... but there are two gotchas: + # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. + # In both cases, we have to default to `cp -p'. + ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || + as_ln_s='cp -p' + elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln + else + as_ln_s='cp -p' + fi +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file +rmdir conf$$.dir 2>/dev/null + +if mkdir -p . 2>/dev/null; then + as_mkdir_p='mkdir -p "$as_dir"' +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +if test -x / >/dev/null 2>&1; then + as_test_x='test -x' +else + if ls -dL / >/dev/null 2>&1; then + as_ls_L_option=L + else + as_ls_L_option= + fi + as_test_x=' + eval sh -c '\'' + if test -d "$1"; then + test -d "$1/."; + else + case $1 in #( + -*)set "./$1";; + esac; + case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #(( + ???[sx]*):;;*)false;;esac;fi + '\'' sh + ' +fi +as_executable_p=$as_test_x + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + +test -n "$DJDIR" || exec 7<&0 </dev/null +exec 6>&1 + +# Name of the host. +# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, +# so uname gets run too. +ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` + +# +# Initializations. +# +ac_default_prefix=/usr/local +ac_clean_files= +ac_config_libobj_dir=. +LIBOBJS= +cross_compiling=no +subdirs= +MFLAGS= +MAKEFLAGS= + +# Identity of this package. +PACKAGE_NAME='kyotocabinet' +PACKAGE_TARNAME='kyotocabinet' +PACKAGE_VERSION='1.2.76' +PACKAGE_STRING='kyotocabinet 1.2.76' +PACKAGE_BUGREPORT='' +PACKAGE_URL='' + +# Factoring default headers for most tests. +ac_includes_default="\ +#include <stdio.h> +#ifdef HAVE_SYS_TYPES_H +# include <sys/types.h> +#endif +#ifdef HAVE_SYS_STAT_H +# include <sys/stat.h> +#endif +#ifdef STDC_HEADERS +# include <stdlib.h> +# include <stddef.h> +#else +# ifdef HAVE_STDLIB_H +# include <stdlib.h> +# endif +#endif +#ifdef HAVE_STRING_H +# if !defined STDC_HEADERS && defined HAVE_MEMORY_H +# include <memory.h> +# endif +# include <string.h> +#endif +#ifdef HAVE_STRINGS_H +# include <strings.h> +#endif +#ifdef HAVE_INTTYPES_H +# include <inttypes.h> +#endif +#ifdef HAVE_STDINT_H +# include <stdint.h> +#endif +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif" + +ac_subst_vars='LTLIBOBJS +LIBOBJS +MYPOSTCMD +MYLDLIBPATHENV +MYLDLIBPATH +MYCMDLIBS +MYCMDLDFLAGS +MYLDFLAGS +MYCPPFLAGS +MYCXXFLAGS +MYCFLAGS +MYPCFILES +MYDOCUMENTFILES +MYMAN1FILES +MYCOMMANDFILES +MYLIBOBJFILES +MYLIBRARYFILES +MYHEADERFILES +MYFORMATVER +MYLIBREV +MYLIBVER +EGREP +GREP +CXXCPP +ac_ct_CXX +CXXFLAGS +CXX +OBJEXT +EXEEXT +ac_ct_CC +CPPFLAGS +LDFLAGS +CFLAGS +CC +target_alias +host_alias +build_alias +LIBS +ECHO_T +ECHO_N +ECHO_C +DEFS +mandir +localedir +libdir +psdir +pdfdir +dvidir +htmldir +infodir +docdir +oldincludedir +includedir +localstatedir +sharedstatedir +sysconfdir +datadir +datarootdir +libexecdir +sbindir +bindir +program_transform_name +prefix +exec_prefix +PACKAGE_URL +PACKAGE_BUGREPORT +PACKAGE_STRING +PACKAGE_VERSION +PACKAGE_TARNAME +PACKAGE_NAME +PATH_SEPARATOR +SHELL' +ac_subst_files='' +ac_user_opts=' +enable_option_checking +enable_debug +enable_devel +enable_opt +enable_profile +enable_uyield +enable_static +enable_shared +enable_atomic +enable_zlib +enable_lzo +enable_lzma +' + ac_precious_vars='build_alias +host_alias +target_alias +CC +CFLAGS +LDFLAGS +LIBS +CPPFLAGS +CXX +CXXFLAGS +CCC +CXXCPP' + + +# Initialize some variables set by options. +ac_init_help= +ac_init_version=false +ac_unrecognized_opts= +ac_unrecognized_sep= +# The variables have the same names as the options, with +# dashes changed to underlines. +cache_file=/dev/null +exec_prefix=NONE +no_create= +no_recursion= +prefix=NONE +program_prefix=NONE +program_suffix=NONE +program_transform_name=s,x,x, +silent= +site= +srcdir= +verbose= +x_includes=NONE +x_libraries=NONE + +# Installation directory options. +# These are left unexpanded so users can "make install exec_prefix=/foo" +# and all the variables that are supposed to be based on exec_prefix +# by default will actually change. +# Use braces instead of parens because sh, perl, etc. also accept them. +# (The list follows the same order as the GNU Coding Standards.) +bindir='${exec_prefix}/bin' +sbindir='${exec_prefix}/sbin' +libexecdir='${exec_prefix}/libexec' +datarootdir='${prefix}/share' +datadir='${datarootdir}' +sysconfdir='${prefix}/etc' +sharedstatedir='${prefix}/com' +localstatedir='${prefix}/var' +includedir='${prefix}/include' +oldincludedir='/usr/include' +docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' +infodir='${datarootdir}/info' +htmldir='${docdir}' +dvidir='${docdir}' +pdfdir='${docdir}' +psdir='${docdir}' +libdir='${exec_prefix}/lib' +localedir='${datarootdir}/locale' +mandir='${datarootdir}/man' + +ac_prev= +ac_dashdash= +for ac_option +do + # If the previous option needs an argument, assign it. + if test -n "$ac_prev"; then + eval $ac_prev=\$ac_option + ac_prev= + continue + fi + + case $ac_option in + *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; + *=) ac_optarg= ;; + *) ac_optarg=yes ;; + esac + + # Accept the important Cygnus configure options, so we can diagnose typos. + + case $ac_dashdash$ac_option in + --) + ac_dashdash=yes ;; + + -bindir | --bindir | --bindi | --bind | --bin | --bi) + ac_prev=bindir ;; + -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) + bindir=$ac_optarg ;; + + -build | --build | --buil | --bui | --bu) + ac_prev=build_alias ;; + -build=* | --build=* | --buil=* | --bui=* | --bu=*) + build_alias=$ac_optarg ;; + + -cache-file | --cache-file | --cache-fil | --cache-fi \ + | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) + ac_prev=cache_file ;; + -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ + | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) + cache_file=$ac_optarg ;; + + --config-cache | -C) + cache_file=config.cache ;; + + -datadir | --datadir | --datadi | --datad) + ac_prev=datadir ;; + -datadir=* | --datadir=* | --datadi=* | --datad=*) + datadir=$ac_optarg ;; + + -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \ + | --dataroo | --dataro | --datar) + ac_prev=datarootdir ;; + -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \ + | --dataroot=* | --dataroo=* | --dataro=* | --datar=*) + datarootdir=$ac_optarg ;; + + -disable-* | --disable-*) + ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid feature name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"enable_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval enable_$ac_useropt=no ;; + + -docdir | --docdir | --docdi | --doc | --do) + ac_prev=docdir ;; + -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*) + docdir=$ac_optarg ;; + + -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv) + ac_prev=dvidir ;; + -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*) + dvidir=$ac_optarg ;; + + -enable-* | --enable-*) + ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid feature name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"enable_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval enable_$ac_useropt=\$ac_optarg ;; + + -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ + | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ + | --exec | --exe | --ex) + ac_prev=exec_prefix ;; + -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ + | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ + | --exec=* | --exe=* | --ex=*) + exec_prefix=$ac_optarg ;; + + -gas | --gas | --ga | --g) + # Obsolete; use --with-gas. + with_gas=yes ;; + + -help | --help | --hel | --he | -h) + ac_init_help=long ;; + -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) + ac_init_help=recursive ;; + -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) + ac_init_help=short ;; + + -host | --host | --hos | --ho) + ac_prev=host_alias ;; + -host=* | --host=* | --hos=* | --ho=*) + host_alias=$ac_optarg ;; + + -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht) + ac_prev=htmldir ;; + -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \ + | --ht=*) + htmldir=$ac_optarg ;; + + -includedir | --includedir | --includedi | --included | --include \ + | --includ | --inclu | --incl | --inc) + ac_prev=includedir ;; + -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ + | --includ=* | --inclu=* | --incl=* | --inc=*) + includedir=$ac_optarg ;; + + -infodir | --infodir | --infodi | --infod | --info | --inf) + ac_prev=infodir ;; + -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) + infodir=$ac_optarg ;; + + -libdir | --libdir | --libdi | --libd) + ac_prev=libdir ;; + -libdir=* | --libdir=* | --libdi=* | --libd=*) + libdir=$ac_optarg ;; + + -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ + | --libexe | --libex | --libe) + ac_prev=libexecdir ;; + -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ + | --libexe=* | --libex=* | --libe=*) + libexecdir=$ac_optarg ;; + + -localedir | --localedir | --localedi | --localed | --locale) + ac_prev=localedir ;; + -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*) + localedir=$ac_optarg ;; + + -localstatedir | --localstatedir | --localstatedi | --localstated \ + | --localstate | --localstat | --localsta | --localst | --locals) + ac_prev=localstatedir ;; + -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ + | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*) + localstatedir=$ac_optarg ;; + + -mandir | --mandir | --mandi | --mand | --man | --ma | --m) + ac_prev=mandir ;; + -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) + mandir=$ac_optarg ;; + + -nfp | --nfp | --nf) + # Obsolete; use --without-fp. + with_fp=no ;; + + -no-create | --no-create | --no-creat | --no-crea | --no-cre \ + | --no-cr | --no-c | -n) + no_create=yes ;; + + -no-recursion | --no-recursion | --no-recursio | --no-recursi \ + | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) + no_recursion=yes ;; + + -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ + | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ + | --oldin | --oldi | --old | --ol | --o) + ac_prev=oldincludedir ;; + -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ + | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ + | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) + oldincludedir=$ac_optarg ;; + + -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) + ac_prev=prefix ;; + -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) + prefix=$ac_optarg ;; + + -program-prefix | --program-prefix | --program-prefi | --program-pref \ + | --program-pre | --program-pr | --program-p) + ac_prev=program_prefix ;; + -program-prefix=* | --program-prefix=* | --program-prefi=* \ + | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) + program_prefix=$ac_optarg ;; + + -program-suffix | --program-suffix | --program-suffi | --program-suff \ + | --program-suf | --program-su | --program-s) + ac_prev=program_suffix ;; + -program-suffix=* | --program-suffix=* | --program-suffi=* \ + | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) + program_suffix=$ac_optarg ;; + + -program-transform-name | --program-transform-name \ + | --program-transform-nam | --program-transform-na \ + | --program-transform-n | --program-transform- \ + | --program-transform | --program-transfor \ + | --program-transfo | --program-transf \ + | --program-trans | --program-tran \ + | --progr-tra | --program-tr | --program-t) + ac_prev=program_transform_name ;; + -program-transform-name=* | --program-transform-name=* \ + | --program-transform-nam=* | --program-transform-na=* \ + | --program-transform-n=* | --program-transform-=* \ + | --program-transform=* | --program-transfor=* \ + | --program-transfo=* | --program-transf=* \ + | --program-trans=* | --program-tran=* \ + | --progr-tra=* | --program-tr=* | --program-t=*) + program_transform_name=$ac_optarg ;; + + -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd) + ac_prev=pdfdir ;; + -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*) + pdfdir=$ac_optarg ;; + + -psdir | --psdir | --psdi | --psd | --ps) + ac_prev=psdir ;; + -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*) + psdir=$ac_optarg ;; + + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + silent=yes ;; + + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) + ac_prev=sbindir ;; + -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ + | --sbi=* | --sb=*) + sbindir=$ac_optarg ;; + + -sharedstatedir | --sharedstatedir | --sharedstatedi \ + | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ + | --sharedst | --shareds | --shared | --share | --shar \ + | --sha | --sh) + ac_prev=sharedstatedir ;; + -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ + | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ + | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ + | --sha=* | --sh=*) + sharedstatedir=$ac_optarg ;; + + -site | --site | --sit) + ac_prev=site ;; + -site=* | --site=* | --sit=*) + site=$ac_optarg ;; + + -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) + ac_prev=srcdir ;; + -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) + srcdir=$ac_optarg ;; + + -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ + | --syscon | --sysco | --sysc | --sys | --sy) + ac_prev=sysconfdir ;; + -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ + | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) + sysconfdir=$ac_optarg ;; + + -target | --target | --targe | --targ | --tar | --ta | --t) + ac_prev=target_alias ;; + -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) + target_alias=$ac_optarg ;; + + -v | -verbose | --verbose | --verbos | --verbo | --verb) + verbose=yes ;; + + -version | --version | --versio | --versi | --vers | -V) + ac_init_version=: ;; + + -with-* | --with-*) + ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid package name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"with_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval with_$ac_useropt=\$ac_optarg ;; + + -without-* | --without-*) + ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` + # Reject names that are not valid shell variable names. + expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && + as_fn_error $? "invalid package name: $ac_useropt" + ac_useropt_orig=$ac_useropt + ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` + case $ac_user_opts in + *" +"with_$ac_useropt" +"*) ;; + *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig" + ac_unrecognized_sep=', ';; + esac + eval with_$ac_useropt=no ;; + + --x) + # Obsolete; use --with-x. + with_x=yes ;; + + -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ + | --x-incl | --x-inc | --x-in | --x-i) + ac_prev=x_includes ;; + -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ + | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) + x_includes=$ac_optarg ;; + + -x-libraries | --x-libraries | --x-librarie | --x-librari \ + | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) + ac_prev=x_libraries ;; + -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ + | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) + x_libraries=$ac_optarg ;; + + -*) as_fn_error $? "unrecognized option: \`$ac_option' +Try \`$0 --help' for more information" + ;; + + *=*) + ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` + # Reject names that are not valid shell variable names. + case $ac_envvar in #( + '' | [0-9]* | *[!_$as_cr_alnum]* ) + as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; + esac + eval $ac_envvar=\$ac_optarg + export $ac_envvar ;; + + *) + # FIXME: should be removed in autoconf 3.0. + $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 + expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && + $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 + : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" + ;; + + esac +done + +if test -n "$ac_prev"; then + ac_option=--`echo $ac_prev | sed 's/_/-/g'` + as_fn_error $? "missing argument to $ac_option" +fi + +if test -n "$ac_unrecognized_opts"; then + case $enable_option_checking in + no) ;; + fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; + *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; + esac +fi + +# Check all directory arguments for consistency. +for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ + datadir sysconfdir sharedstatedir localstatedir includedir \ + oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ + libdir localedir mandir +do + eval ac_val=\$$ac_var + # Remove trailing slashes. + case $ac_val in + */ ) + ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'` + eval $ac_var=\$ac_val;; + esac + # Be sure to have absolute directory names. + case $ac_val in + [\\/$]* | ?:[\\/]* ) continue;; + NONE | '' ) case $ac_var in *prefix ) continue;; esac;; + esac + as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" +done + +# There might be people who depend on the old broken behavior: `$host' +# used to hold the argument of --host etc. +# FIXME: To remove some day. +build=$build_alias +host=$host_alias +target=$target_alias + +# FIXME: To remove some day. +if test "x$host_alias" != x; then + if test "x$build_alias" = x; then + cross_compiling=maybe + $as_echo "$as_me: WARNING: if you wanted to set the --build type, don't use --host. + If a cross compiler is detected then cross compile mode will be used" >&2 + elif test "x$build_alias" != "x$host_alias"; then + cross_compiling=yes + fi +fi + +ac_tool_prefix= +test -n "$host_alias" && ac_tool_prefix=$host_alias- + +test "$silent" = yes && exec 6>/dev/null + + +ac_pwd=`pwd` && test -n "$ac_pwd" && +ac_ls_di=`ls -di .` && +ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || + as_fn_error $? "working directory cannot be determined" +test "X$ac_ls_di" = "X$ac_pwd_ls_di" || + as_fn_error $? "pwd does not report name of working directory" + + +# Find the source files, if location was not specified. +if test -z "$srcdir"; then + ac_srcdir_defaulted=yes + # Try the directory containing this script, then the parent directory. + ac_confdir=`$as_dirname -- "$as_myself" || +$as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_myself" : 'X\(//\)[^/]' \| \ + X"$as_myself" : 'X\(//\)$' \| \ + X"$as_myself" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_myself" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + srcdir=$ac_confdir + if test ! -r "$srcdir/$ac_unique_file"; then + srcdir=.. + fi +else + ac_srcdir_defaulted=no +fi +if test ! -r "$srcdir/$ac_unique_file"; then + test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." + as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" +fi +ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" +ac_abs_confdir=`( + cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" + pwd)` +# When building in place, set srcdir=. +if test "$ac_abs_confdir" = "$ac_pwd"; then + srcdir=. +fi +# Remove unnecessary trailing slashes from srcdir. +# Double slashes in file names in object file debugging info +# mess up M-x gdb in Emacs. +case $srcdir in +*/) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;; +esac +for ac_var in $ac_precious_vars; do + eval ac_env_${ac_var}_set=\${${ac_var}+set} + eval ac_env_${ac_var}_value=\$${ac_var} + eval ac_cv_env_${ac_var}_set=\${${ac_var}+set} + eval ac_cv_env_${ac_var}_value=\$${ac_var} +done + +# +# Report the --help message. +# +if test "$ac_init_help" = "long"; then + # Omit some internal or obsolete options to make the list less imposing. + # This message is too long to be a string in the A/UX 3.1 sh. + cat <<_ACEOF +\`configure' configures kyotocabinet 1.2.76 to adapt to many kinds of systems. + +Usage: $0 [OPTION]... [VAR=VALUE]... + +To assign environment variables (e.g., CC, CFLAGS...), specify them as +VAR=VALUE. See below for descriptions of some of the useful variables. + +Defaults for the options are specified in brackets. + +Configuration: + -h, --help display this help and exit + --help=short display options specific to this package + --help=recursive display the short help of all the included packages + -V, --version display version information and exit + -q, --quiet, --silent do not print \`checking ...' messages + --cache-file=FILE cache test results in FILE [disabled] + -C, --config-cache alias for \`--cache-file=config.cache' + -n, --no-create do not create output files + --srcdir=DIR find the sources in DIR [configure dir or \`..'] + +Installation directories: + --prefix=PREFIX install architecture-independent files in PREFIX + [$ac_default_prefix] + --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX + [PREFIX] + +By default, \`make install' will install all the files in +\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify +an installation prefix other than \`$ac_default_prefix' using \`--prefix', +for instance \`--prefix=\$HOME'. + +For better control, use the options below. + +Fine tuning of the installation directories: + --bindir=DIR user executables [EPREFIX/bin] + --sbindir=DIR system admin executables [EPREFIX/sbin] + --libexecdir=DIR program executables [EPREFIX/libexec] + --sysconfdir=DIR read-only single-machine data [PREFIX/etc] + --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] + --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --libdir=DIR object code libraries [EPREFIX/lib] + --includedir=DIR C header files [PREFIX/include] + --oldincludedir=DIR C header files for non-gcc [/usr/include] + --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] + --datadir=DIR read-only architecture-independent data [DATAROOTDIR] + --infodir=DIR info documentation [DATAROOTDIR/info] + --localedir=DIR locale-dependent data [DATAROOTDIR/locale] + --mandir=DIR man documentation [DATAROOTDIR/man] + --docdir=DIR documentation root [DATAROOTDIR/doc/kyotocabinet] + --htmldir=DIR html documentation [DOCDIR] + --dvidir=DIR dvi documentation [DOCDIR] + --pdfdir=DIR pdf documentation [DOCDIR] + --psdir=DIR ps documentation [DOCDIR] +_ACEOF + + cat <<\_ACEOF +_ACEOF +fi + +if test -n "$ac_init_help"; then + case $ac_init_help in + short | recursive ) echo "Configuration of kyotocabinet 1.2.76:";; + esac + cat <<\_ACEOF + +Optional Features: + --disable-option-checking ignore unrecognized --enable/--with options + --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) + --enable-FEATURE[=ARG] include FEATURE [ARG=yes] + --enable-debug build for debugging + --enable-devel build for development + --disable-opt build without optimization + --enable-profile build for profiling + --enable-uyield build for detecting race conditions + --enable-static build by static linking + --disable-shared avoid to build shared libraries + --disable-atomic build without atomic operations + --disable-zlib build without ZLIB compression + --enable-lzo build with LZO compression + --enable-lzma build with LZMA compression + +Some influential environment variables: + CC C compiler command + CFLAGS C compiler flags + LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries in a + nonstandard directory <lib dir> + LIBS libraries to pass to the linker, e.g. -l<library> + CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I<include dir> if + you have headers in a nonstandard directory <include dir> + CXX C++ compiler command + CXXFLAGS C++ compiler flags + CXXCPP C++ preprocessor + +Use these variables to override the choices made by `configure' or to help +it to find libraries and programs with nonstandard names/locations. + +Report bugs to the package provider. +_ACEOF +ac_status=$? +fi + +if test "$ac_init_help" = "recursive"; then + # If there are subdirs, report their specific --help. + for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue + test -d "$ac_dir" || + { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } || + continue + ac_builddir=. + +case "$ac_dir" in +.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; +*) + ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` + # A ".." for each directory in $ac_dir_suffix. + ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + case $ac_top_builddir_sub in + "") ac_top_builddir_sub=. ac_top_build_prefix= ;; + *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; + esac ;; +esac +ac_abs_top_builddir=$ac_pwd +ac_abs_builddir=$ac_pwd$ac_dir_suffix +# for backward compatibility: +ac_top_builddir=$ac_top_build_prefix + +case $srcdir in + .) # We are building in place. + ac_srcdir=. + ac_top_srcdir=$ac_top_builddir_sub + ac_abs_top_srcdir=$ac_pwd ;; + [\\/]* | ?:[\\/]* ) # Absolute name. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir + ac_abs_top_srcdir=$srcdir ;; + *) # Relative name. + ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_build_prefix$srcdir + ac_abs_top_srcdir=$ac_pwd/$srcdir ;; +esac +ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix + + cd "$ac_dir" || { ac_status=$?; continue; } + # Check for guested configure. + if test -f "$ac_srcdir/configure.gnu"; then + echo && + $SHELL "$ac_srcdir/configure.gnu" --help=recursive + elif test -f "$ac_srcdir/configure"; then + echo && + $SHELL "$ac_srcdir/configure" --help=recursive + else + $as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 + fi || ac_status=$? + cd "$ac_pwd" || { ac_status=$?; break; } + done +fi + +test -n "$ac_init_help" && exit $ac_status +if $ac_init_version; then + cat <<\_ACEOF +kyotocabinet configure 1.2.76 +generated by GNU Autoconf 2.68 + +Copyright (C) 2010 Free Software Foundation, Inc. +This configure script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it. +_ACEOF + exit +fi + +## ------------------------ ## +## Autoconf initialization. ## +## ------------------------ ## + +# ac_fn_c_try_compile LINENO +# -------------------------- +# Try to compile conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext + if { { ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compile") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_compile + +# ac_fn_cxx_try_compile LINENO +# ---------------------------- +# Try to compile conftest.$ac_ext, and return whether this succeeded. +ac_fn_cxx_try_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext + if { { ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compile") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_compile + +# ac_fn_cxx_try_run LINENO +# ------------------------ +# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes +# that executables *can* be run. +ac_fn_cxx_try_run () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; }; then : + ac_retval=0 +else + $as_echo "$as_me: program exited with status $ac_status" >&5 + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=$ac_status +fi + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_run + +# ac_fn_cxx_try_cpp LINENO +# ------------------------ +# Try to preprocess conftest.$ac_ext, and return whether this succeeded. +ac_fn_cxx_try_cpp () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } > conftest.i && { + test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || + test ! -s conftest.err + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_cpp + +# ac_fn_cxx_check_header_compile LINENO HEADER VAR INCLUDES +# --------------------------------------------------------- +# Tests whether HEADER exists and can be compiled using the include files in +# INCLUDES, setting the cache variable VAR accordingly. +ac_fn_cxx_check_header_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_cxx_check_header_compile + +# ac_fn_cxx_try_link LINENO +# ------------------------- +# Try to link conftest.$ac_ext, and return whether this succeeded. +ac_fn_cxx_try_link () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext conftest$ac_exeext + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information + # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would + # interfere with the next link command; also delete a directory that is + # left behind by Apple's compiler. We do this before executing the actions. + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_link + +# ac_fn_cxx_check_header_mongrel LINENO HEADER VAR INCLUDES +# --------------------------------------------------------- +# Tests whether HEADER exists, giving a warning if it cannot be compiled using +# the include files in INCLUDES and setting the cache variable VAR +# accordingly. +ac_fn_cxx_check_header_mongrel () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if eval \${$3+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +else + # Is the header compilable? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 +$as_echo_n "checking $2 usability... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_header_compiler=yes +else + ac_header_compiler=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 +$as_echo_n "checking $2 presence... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <$2> +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + ac_header_preproc=yes +else + ac_header_preproc=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_cxx_preproc_warn_flag in #(( + yes:no: ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; + no:yes:* ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; +esac + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=\$ac_header_compiler" +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_cxx_check_header_mongrel +cat >config.log <<_ACEOF +This file contains any messages produced by compilers while +running configure, to aid debugging if configure makes a mistake. + +It was created by kyotocabinet $as_me 1.2.76, which was +generated by GNU Autoconf 2.68. Invocation command line was + + $ $0 $@ + +_ACEOF +exec 5>>config.log +{ +cat <<_ASUNAME +## --------- ## +## Platform. ## +## --------- ## + +hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` +uname -m = `(uname -m) 2>/dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` + +/bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` +/usr/bin/hostinfo = `(/usr/bin/hostinfo) 2>/dev/null || echo unknown` +/bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` +/bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` + +_ASUNAME + +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + $as_echo "PATH: $as_dir" + done +IFS=$as_save_IFS + +} >&5 + +cat >&5 <<_ACEOF + + +## ----------- ## +## Core tests. ## +## ----------- ## + +_ACEOF + + +# Keep a trace of the command line. +# Strip out --no-create and --no-recursion so they do not pile up. +# Strip out --silent because we don't want to record it for future runs. +# Also quote any args containing shell meta-characters. +# Make two passes to allow for proper duplicate-argument suppression. +ac_configure_args= +ac_configure_args0= +ac_configure_args1= +ac_must_keep_next=false +for ac_pass in 1 2 +do + for ac_arg + do + case $ac_arg in + -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil) + continue ;; + *\'*) + ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; + esac + case $ac_pass in + 1) as_fn_append ac_configure_args0 " '$ac_arg'" ;; + 2) + as_fn_append ac_configure_args1 " '$ac_arg'" + if test $ac_must_keep_next = true; then + ac_must_keep_next=false # Got value, back to normal. + else + case $ac_arg in + *=* | --config-cache | -C | -disable-* | --disable-* \ + | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ + | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ + | -with-* | --with-* | -without-* | --without-* | --x) + case "$ac_configure_args0 " in + "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; + esac + ;; + -* ) ac_must_keep_next=true ;; + esac + fi + as_fn_append ac_configure_args " '$ac_arg'" + ;; + esac + done +done +{ ac_configure_args0=; unset ac_configure_args0;} +{ ac_configure_args1=; unset ac_configure_args1;} + +# When interrupted or exit'd, cleanup temporary files, and complete +# config.log. We remove comments because anyway the quotes in there +# would cause problems or look ugly. +# WARNING: Use '\'' to represent an apostrophe within the trap. +# WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug. +trap 'exit_status=$? + # Save into config.log some information that might help in debugging. + { + echo + + $as_echo "## ---------------- ## +## Cache variables. ## +## ---------------- ##" + echo + # The following way of writing the cache mishandles newlines in values, +( + for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do + eval ac_val=\$$ac_var + case $ac_val in #( + *${as_nl}*) + case $ac_var in #( + *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 +$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; + esac + case $ac_var in #( + _ | IFS | as_nl) ;; #( + BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( + *) { eval $ac_var=; unset $ac_var;} ;; + esac ;; + esac + done + (set) 2>&1 | + case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #( + *${as_nl}ac_space=\ *) + sed -n \ + "s/'\''/'\''\\\\'\'''\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p" + ;; #( + *) + sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" + ;; + esac | + sort +) + echo + + $as_echo "## ----------------- ## +## Output variables. ## +## ----------------- ##" + echo + for ac_var in $ac_subst_vars + do + eval ac_val=\$$ac_var + case $ac_val in + *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; + esac + $as_echo "$ac_var='\''$ac_val'\''" + done | sort + echo + + if test -n "$ac_subst_files"; then + $as_echo "## ------------------- ## +## File substitutions. ## +## ------------------- ##" + echo + for ac_var in $ac_subst_files + do + eval ac_val=\$$ac_var + case $ac_val in + *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; + esac + $as_echo "$ac_var='\''$ac_val'\''" + done | sort + echo + fi + + if test -s confdefs.h; then + $as_echo "## ----------- ## +## confdefs.h. ## +## ----------- ##" + echo + cat confdefs.h + echo + fi + test "$ac_signal" != 0 && + $as_echo "$as_me: caught signal $ac_signal" + $as_echo "$as_me: exit $exit_status" + } >&5 + rm -f core *.core core.conftest.* && + rm -f -r conftest* confdefs* conf$$* $ac_clean_files && + exit $exit_status +' 0 +for ac_signal in 1 2 13 15; do + trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal +done +ac_signal=0 + +# confdefs.h avoids OS command line length limits that DEFS can exceed. +rm -f -r conftest* confdefs.h + +$as_echo "/* confdefs.h */" > confdefs.h + +# Predefined preprocessor variables. + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_NAME "$PACKAGE_NAME" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_TARNAME "$PACKAGE_TARNAME" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_VERSION "$PACKAGE_VERSION" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_STRING "$PACKAGE_STRING" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" +_ACEOF + +cat >>confdefs.h <<_ACEOF +#define PACKAGE_URL "$PACKAGE_URL" +_ACEOF + + +# Let the site file select an alternate cache file if it wants to. +# Prefer an explicitly selected file to automatically selected ones. +ac_site_file1=NONE +ac_site_file2=NONE +if test -n "$CONFIG_SITE"; then + # We do not want a PATH search for config.site. + case $CONFIG_SITE in #(( + -*) ac_site_file1=./$CONFIG_SITE;; + */*) ac_site_file1=$CONFIG_SITE;; + *) ac_site_file1=./$CONFIG_SITE;; + esac +elif test "x$prefix" != xNONE; then + ac_site_file1=$prefix/share/config.site + ac_site_file2=$prefix/etc/config.site +else + ac_site_file1=$ac_default_prefix/share/config.site + ac_site_file2=$ac_default_prefix/etc/config.site +fi +for ac_site_file in "$ac_site_file1" "$ac_site_file2" +do + test "x$ac_site_file" = xNONE && continue + if test /dev/null != "$ac_site_file" && test -r "$ac_site_file"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 +$as_echo "$as_me: loading site script $ac_site_file" >&6;} + sed 's/^/| /' "$ac_site_file" >&5 + . "$ac_site_file" \ + || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "failed to load site script $ac_site_file +See \`config.log' for more details" "$LINENO" 5; } + fi +done + +if test -r "$cache_file"; then + # Some versions of bash will fail to source /dev/null (special files + # actually), so we avoid doing that. DJGPP emulates it as a regular file. + if test /dev/null != "$cache_file" && test -f "$cache_file"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5 +$as_echo "$as_me: loading cache $cache_file" >&6;} + case $cache_file in + [\\/]* | ?:[\\/]* ) . "$cache_file";; + *) . "./$cache_file";; + esac + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5 +$as_echo "$as_me: creating cache $cache_file" >&6;} + >$cache_file +fi + +# Check that the precious variables saved in the cache have kept the same +# value. +ac_cache_corrupted=false +for ac_var in $ac_precious_vars; do + eval ac_old_set=\$ac_cv_env_${ac_var}_set + eval ac_new_set=\$ac_env_${ac_var}_set + eval ac_old_val=\$ac_cv_env_${ac_var}_value + eval ac_new_val=\$ac_env_${ac_var}_value + case $ac_old_set,$ac_new_set in + set,) + { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 +$as_echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,set) + { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5 +$as_echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} + ac_cache_corrupted=: ;; + ,);; + *) + if test "x$ac_old_val" != "x$ac_new_val"; then + # differences in whitespace do not lead to failure. + ac_old_val_w=`echo x $ac_old_val` + ac_new_val_w=`echo x $ac_new_val` + if test "$ac_old_val_w" != "$ac_new_val_w"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5 +$as_echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} + ac_cache_corrupted=: + else + { $as_echo "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 +$as_echo "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} + eval $ac_var=\$ac_old_val + fi + { $as_echo "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5 +$as_echo "$as_me: former value: \`$ac_old_val'" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5 +$as_echo "$as_me: current value: \`$ac_new_val'" >&2;} + fi;; + esac + # Pass precious variables to config.status. + if test "$ac_new_set" = set; then + case $ac_new_val in + *\'*) ac_arg=$ac_var=`$as_echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; + *) ac_arg=$ac_var=$ac_new_val ;; + esac + case " $ac_configure_args " in + *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. + *) as_fn_append ac_configure_args " '$ac_arg'" ;; + esac + fi +done +if $ac_cache_corrupted; then + { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 +$as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} + as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 +fi +## -------------------- ## +## Main body of script. ## +## -------------------- ## + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + + +# Package information +MYLIBVER=16 +MYLIBREV=13 +MYFORMATVER=5 + +# Targets +MYHEADERFILES="kccommon.h kcutil.h kcthread.h kcfile.h" +MYHEADERFILES="$MYHEADERFILES kccompress.h kccompare.h kcmap.h kcregex.h" +MYHEADERFILES="$MYHEADERFILES kcdb.h kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h" +MYHEADERFILES="$MYHEADERFILES kchashdb.h kcdirdb.h kctextdb.h kcpolydb.h kcdbext.h kclangc.h" +MYLIBRARYFILES="libkyotocabinet.a" +MYLIBOBJFILES="kcutil.o kcthread.o kcfile.o kccompress.o kccompare.o kcmap.o kcregex.o" +MYLIBOBJFILES="$MYLIBOBJFILES kcdb.o kcplantdb.o kcprotodb.o kcstashdb.o kccachedb.o" +MYLIBOBJFILES="$MYLIBOBJFILES kchashdb.o kcdirdb.o kctextdb.o kcpolydb.o kcdbext.o kclangc.o" +MYCOMMANDFILES="kcutiltest kcutilmgr kcprototest kcstashtest kccachetest kcgrasstest" +MYCOMMANDFILES="$MYCOMMANDFILES kchashtest kchashmgr kctreetest kctreemgr" +MYCOMMANDFILES="$MYCOMMANDFILES kcdirtest kcdirmgr kcforesttest kcforestmgr" +MYCOMMANDFILES="$MYCOMMANDFILES kcpolytest kcpolymgr kclangctest" +MYMAN1FILES="kcutiltest.1 kcutilmgr.1 kcprototest.1 kcstashtest.1 kccachetest.1 kcgrasstest.1" +MYMAN1FILES="$MYMAN1FILES kchashtest.1 kchashmgr.1 kctreetest.1 kctreemgr.1" +MYMAN1FILES="$MYMAN1FILES kcdirtest.1 kcdirmgr.1 kcforesttest.1 kcforestmgr.1" +MYMAN1FILES="$MYMAN1FILES kcpolytest.1 kcpolymgr.1 kclangctest.1" +MYDOCUMENTFILES="COPYING FOSSEXCEPTION ChangeLog doc kyotocabinet.idl" +MYPCFILES="kyotocabinet.pc" + +# Building flags +MYCFLAGS="-Wall -ansi -pedantic -fPIC -fsigned-char -g0 -O2" +MYCXXFLAGS="-Wall -fPIC -fsigned-char -g0 -O2" +MYCPPFLAGS="-I. -I\$(INCLUDEDIR) -I/usr/local/include" +MYCPPFLAGS="$MYCPPFLAGS -DNDEBUG -D_GNU_SOURCE=1" +MYCPPFLAGS="$MYCPPFLAGS -D_FILE_OFFSET_BITS=64 -D_REENTRANT -D__EXTENSIONS__" +MYLDFLAGS="-L. -L\$(LIBDIR) -L/usr/local/lib" +MYCMDLDFLAGS="" +MYCMDLIBS="" +MYLDLIBPATH="" +MYLDLIBPATHENV="LD_LIBRARY_PATH" +MYPOSTCMD="true" + +# Building paths +PATH=".:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:$PATH" +CPATH=".:/usr/local/include:$CPATH" +LIBRARY_PATH=".:/usr/local/lib:$LIBRARY_PATH" +LD_LIBRARY_PATH=".:/usr/local/lib:$LD_LIBRARY_PATH" +PKG_CONFIG_PATH=".:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" +export PATH CPATH LIBRARY_PATH LD_LIBRARY_PATH PKG_CONFIG_PATH + + + +#================================================================ +# Options +#================================================================ + +# Internal variables +enables="" +is_static="" + +# Debug mode +# Check whether --enable-debug was given. +if test "${enable_debug+set}" = set; then : + enableval=$enable_debug; +fi + +if test "$enable_debug" = "yes" +then + MYCFLAGS="-Wall -ansi -pedantic -fPIC -fsigned-char -g -O0" + MYCXXFLAGS="-Wall -fPIC -fsigned-char -g -O0" + MYCPPFLAGS="$MYCPPFLAGS -UNDEBUG -D_KCDEBUG" + is_static="yes" + enables="$enables (debug)" +fi + +# Developping mode +# Check whether --enable-devel was given. +if test "${enable_devel+set}" = set; then : + enableval=$enable_devel; +fi + +if test "$enable_devel" = "yes" +then + MYCFLAGS="-Wall -Wextra -Wno-unused-parameter" + MYCFLAGS="$MYCFLAGS -ansi -pedantic -fPIC -fsigned-char -O2 -fno-inline -pipe" + MYCXXFLAGS="-Wall -Wextra -Wno-unused-parameter -Wnon-virtual-dtor" + MYCXXFLAGS="$MYCXXFLAGS -fPIC -fsigned-char -g -O2 -fno-inline -pipe" + MYCPPFLAGS="$MYCPPFLAGS -UNDEBUG -D_KCDEBUG" + enables="$enables (devel)" +fi + +# Disable optimization +# Check whether --enable-opt was given. +if test "${enable_opt+set}" = set; then : + enableval=$enable_opt; +fi + +if test "$enable_opt" = "no" +then + MYCFLAGS="$MYCFLAGS -O0" + MYCXXFLAGS="$MYCXXFLAGS -O0" + enables="$enables (no-opt)" +fi + +# Profiling mode +# Check whether --enable-profile was given. +if test "${enable_profile+set}" = set; then : + enableval=$enable_profile; +fi + +if test "$enable_profile" = "yes" +then + MYCXXFLAGS="-Wall -fPIC -fsigned-char -g -pg -O2 -fno-inline -pipe" + enables="$enables (profile)" +fi + +# Micro yield mode +# Check whether --enable-uyield was given. +if test "${enable_uyield+set}" = set; then : + enableval=$enable_uyield; +fi + +if test "$enable_uyield" = "yes" +then + MYCPPFLAGS="$MYCPPFLAGS -UNDEBUG -D_KCUYIELD" + enables="$enables (uyield)" +fi + +# Static mode +# Check whether --enable-static was given. +if test "${enable_static+set}" = set; then : + enableval=$enable_static; +fi + +if test "$enable_static" = "yes" +then + is_static="yes" + enables="$enables (static)" +fi + +# Disable shared object +# Check whether --enable-shared was given. +if test "${enable_shared+set}" = set; then : + enableval=$enable_shared; +fi + +if test "$enable_shared" = "no" +then + enables="$enables (no-shared)" +fi + +# Disable atomic operations +# Check whether --enable-atomic was given. +if test "${enable_atomic+set}" = set; then : + enableval=$enable_atomic; +fi + +if test "$enable_atomic" = "no" +then + enables="$enables (no-atomic)" +fi + +# Disable ZLIB compression +# Check whether --enable-zlib was given. +if test "${enable_zlib+set}" = set; then : + enableval=$enable_zlib; +fi + +if test "$enable_zlib" = "no" +then + enables="$enables (no-zlib)" +else + MYCPPFLAGS="$MYCPPFLAGS -D_MYZLIB" +fi + +# Enable LZO compression +# Check whether --enable-lzo was given. +if test "${enable_lzo+set}" = set; then : + enableval=$enable_lzo; +fi + +if test "$enable_lzo" = "yes" +then + MYCPPFLAGS="$MYCPPFLAGS -D_MYLZO" + enables="$enables (lzo)" +fi + +# Enable LZMA compression +# Check whether --enable-lzma was given. +if test "${enable_lzma+set}" = set; then : + enableval=$enable_lzma; +fi + +if test "$enable_lzma" = "yes" +then + MYCPPFLAGS="$MYCPPFLAGS -D_MYLZMA" + enables="$enables (lzma)" +fi + +# Messages +printf '#================================================================\n' +printf '# Configuring Kyoto Cabinet version %s%s.\n' "$PACKAGE_VERSION" "$enables" +printf '#================================================================\n' + + + +#================================================================ +# Checking Commands and Libraries +#================================================================ + +# C and C++ compilers +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. +set dummy ${ac_tool_prefix}gcc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="${ac_tool_prefix}gcc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_CC"; then + ac_ct_CC=$CC + # Extract the first word of "gcc", so it can be a program name with args. +set dummy gcc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CC="gcc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 +$as_echo "$ac_ct_CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +else + CC="$ac_cv_prog_CC" +fi + +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. +set dummy ${ac_tool_prefix}cc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="${ac_tool_prefix}cc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + fi +fi +if test -z "$CC"; then + # Extract the first word of "cc", so it can be a program name with args. +set dummy cc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else + ac_prog_rejected=no +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then + ac_prog_rejected=yes + continue + fi + ac_cv_prog_CC="cc" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +if test $ac_prog_rejected = yes; then + # We found a bogon in the path, so make sure we never use it. + set dummy $ac_cv_prog_CC + shift + if test $# != 0; then + # We chose a different compiler from the bogus one. + # However, it has the same basename, so the bogon will be chosen + # first if we set CC to just the basename; use the full file name. + shift + ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" + fi +fi +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + for ac_prog in cl.exe + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CC="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +$as_echo "$CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$CC" && break + done +fi +if test -z "$CC"; then + ac_ct_CC=$CC + for ac_prog in cl.exe +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_CC+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CC="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 +$as_echo "$ac_ct_CC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_CC" && break +done + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +fi + +fi + + +test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no acceptable C compiler found in \$PATH +See \`config.log' for more details" "$LINENO" 5; } + +# Provide some information about the compiler. +$as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +for ac_option in --version -v -V -qversion; do + { { ac_try="$ac_compiler $ac_option >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compiler $ac_option >&5") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + sed '10a\ +... rest of stderr output deleted ... + 10q' conftest.err >conftest.er1 + cat conftest.er1 >&5 + fi + rm -f conftest.er1 conftest.err + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } +done + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out" +# Try to create an executable without -o first, disregard a.out. +# It will help us diagnose broken compilers, and finding out an intuition +# of exeext. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5 +$as_echo_n "checking whether the C compiler works... " >&6; } +ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` + +# The possible output files: +ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*" + +ac_rmfiles= +for ac_file in $ac_files +do + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; + * ) ac_rmfiles="$ac_rmfiles $ac_file";; + esac +done +rm -f $ac_rmfiles + +if { { ac_try="$ac_link_default" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link_default") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then : + # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. +# So ignore a value of `no', otherwise this would lead to `EXEEXT = no' +# in a Makefile. We should not override ac_cv_exeext if it was cached, +# so that the user can short-circuit this test for compilers unknown to +# Autoconf. +for ac_file in $ac_files '' +do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) + ;; + [ab].out ) + # We found the default executable, but exeext='' is most + # certainly right. + break;; + *.* ) + if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no; + then :; else + ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + fi + # We set ac_cv_exeext here because the later test for it is not + # safe: cross compilers may not add the suffix if given an `-o' + # argument, so we may need to know it at that point already. + # Even if this section looks crufty: it has the advantage of + # actually working. + break;; + * ) + break;; + esac +done +test "$ac_cv_exeext" = no && ac_cv_exeext= + +else + ac_file='' +fi +if test -z "$ac_file"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +$as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "C compiler cannot create executables +See \`config.log' for more details" "$LINENO" 5; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5 +$as_echo_n "checking for C compiler default output file name... " >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5 +$as_echo "$ac_file" >&6; } +ac_exeext=$ac_cv_exeext + +rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out +ac_clean_files=$ac_clean_files_save +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5 +$as_echo_n "checking for suffix of executables... " >&6; } +if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then : + # If both `conftest.exe' and `conftest' are `present' (well, observable) +# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will +# work properly (i.e., refer to `conftest.exe'), while it won't with +# `rm'. +for ac_file in conftest.exe conftest conftest.*; do + test -f "$ac_file" || continue + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; + *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` + break;; + * ) break;; + esac +done +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details" "$LINENO" 5; } +fi +rm -f conftest conftest$ac_cv_exeext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 +$as_echo "$ac_cv_exeext" >&6; } + +rm -f conftest.$ac_ext +EXEEXT=$ac_cv_exeext +ac_exeext=$EXEEXT +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <stdio.h> +int +main () +{ +FILE *f = fopen ("conftest.out", "w"); + return ferror (f) || fclose (f) != 0; + + ; + return 0; +} +_ACEOF +ac_clean_files="$ac_clean_files conftest.out" +# Check that the compiler produces executables we can run. If not, either +# the compiler is broken, or we cross compile. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5 +$as_echo_n "checking whether we are cross compiling... " >&6; } +if test "$cross_compiling" != yes; then + { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + if { ac_try='./conftest$ac_cv_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; }; then + cross_compiling=no + else + if test "$cross_compiling" = maybe; then + cross_compiling=yes + else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot run C compiled programs. +If you meant to cross compile, use \`--host'. +See \`config.log' for more details" "$LINENO" 5; } + fi + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 +$as_echo "$cross_compiling" >&6; } + +rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out +ac_clean_files=$ac_clean_files_save +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 +$as_echo_n "checking for suffix of object files... " >&6; } +if ${ac_cv_objext+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +rm -f conftest.o conftest.obj +if { { ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compile") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then : + for ac_file in conftest.o conftest.obj conftest.*; do + test -f "$ac_file" || continue; + case $ac_file in + *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;; + *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` + break;; + esac +done +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot compute suffix of object files: cannot compile +See \`config.log' for more details" "$LINENO" 5; } +fi +rm -f conftest.$ac_cv_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 +$as_echo "$ac_cv_objext" >&6; } +OBJEXT=$ac_cv_objext +ac_objext=$OBJEXT +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 +$as_echo_n "checking whether we are using the GNU C compiler... " >&6; } +if ${ac_cv_c_compiler_gnu+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_compiler_gnu=yes +else + ac_compiler_gnu=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_c_compiler_gnu=$ac_compiler_gnu + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 +$as_echo "$ac_cv_c_compiler_gnu" >&6; } +if test $ac_compiler_gnu = yes; then + GCC=yes +else + GCC= +fi +ac_test_CFLAGS=${CFLAGS+set} +ac_save_CFLAGS=$CFLAGS +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 +$as_echo_n "checking whether $CC accepts -g... " >&6; } +if ${ac_cv_prog_cc_g+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_save_c_werror_flag=$ac_c_werror_flag + ac_c_werror_flag=yes + ac_cv_prog_cc_g=no + CFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_prog_cc_g=yes +else + CFLAGS="" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +else + ac_c_werror_flag=$ac_save_c_werror_flag + CFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_prog_cc_g=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_c_werror_flag=$ac_save_c_werror_flag +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 +$as_echo "$ac_cv_prog_cc_g" >&6; } +if test "$ac_test_CFLAGS" = set; then + CFLAGS=$ac_save_CFLAGS +elif test $ac_cv_prog_cc_g = yes; then + if test "$GCC" = yes; then + CFLAGS="-g -O2" + else + CFLAGS="-g" + fi +else + if test "$GCC" = yes; then + CFLAGS="-O2" + else + CFLAGS= + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 +$as_echo_n "checking for $CC option to accept ISO C89... " >&6; } +if ${ac_cv_prog_cc_c89+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_prog_cc_c89=no +ac_save_CC=$CC +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <stdarg.h> +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ +struct buf { int x; }; +FILE * (*rcsopen) (struct buf *, struct stat *, int); +static char *e (p, i) + char **p; + int i; +{ + return p[i]; +} +static char *f (char * (*g) (char **, int), char **p, ...) +{ + char *s; + va_list v; + va_start (v,p); + s = g (p, va_arg (v,int)); + va_end (v); + return s; +} + +/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has + function prototypes and stuff, but not '\xHH' hex character constants. + These don't provoke an error unfortunately, instead are silently treated + as 'x'. The following induces an error, until -std is added to get + proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an + array size at least. It's necessary to write '\x00'==0 to get something + that's true only with -std. */ +int osf4_cc_array ['\x00' == 0 ? 1 : -1]; + +/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters + inside strings and character constants. */ +#define FOO(x) 'x' +int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1]; + +int test (int i, double x); +struct s1 {int (*f) (int a);}; +struct s2 {int (*f) (double a);}; +int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); +int argc; +char **argv; +int +main () +{ +return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; + ; + return 0; +} +_ACEOF +for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \ + -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" +do + CC="$ac_save_CC $ac_arg" + if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_prog_cc_c89=$ac_arg +fi +rm -f core conftest.err conftest.$ac_objext + test "x$ac_cv_prog_cc_c89" != "xno" && break +done +rm -f conftest.$ac_ext +CC=$ac_save_CC + +fi +# AC_CACHE_VAL +case "x$ac_cv_prog_cc_c89" in + x) + { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 +$as_echo "none needed" >&6; } ;; + xno) + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 +$as_echo "unsupported" >&6; } ;; + *) + CC="$CC $ac_cv_prog_cc_c89" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 +$as_echo "$ac_cv_prog_cc_c89" >&6; } ;; +esac +if test "x$ac_cv_prog_cc_c89" != xno; then : + +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +if test -z "$CXX"; then + if test -n "$CCC"; then + CXX=$CCC + else + if test -n "$ac_tool_prefix"; then + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CXX"; then + ac_cv_prog_CXX="$CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_CXX="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CXX=$ac_cv_prog_CXX +if test -n "$CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +$as_echo "$CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$CXX" && break + done +fi +if test -z "$CXX"; then + ac_ct_CXX=$CXX + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CXX"; then + ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_ac_ct_CXX="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CXX=$ac_cv_prog_ac_ct_CXX +if test -n "$ac_ct_CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CXX" >&5 +$as_echo "$ac_ct_CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_CXX" && break +done + + if test "x$ac_ct_CXX" = x; then + CXX="g++" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CXX=$ac_ct_CXX + fi +fi + + fi +fi +# Provide some information about the compiler. +$as_echo "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +for ac_option in --version -v -V -qversion; do + { { ac_try="$ac_compiler $ac_option >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compiler $ac_option >&5") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + sed '10a\ +... rest of stderr output deleted ... + 10q' conftest.err >conftest.er1 + cat conftest.er1 >&5 + fi + rm -f conftest.er1 conftest.err + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } +done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C++ compiler" >&5 +$as_echo_n "checking whether we are using the GNU C++ compiler... " >&6; } +if ${ac_cv_cxx_compiler_gnu+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_compiler_gnu=yes +else + ac_compiler_gnu=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_cxx_compiler_gnu=$ac_compiler_gnu + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_compiler_gnu" >&5 +$as_echo "$ac_cv_cxx_compiler_gnu" >&6; } +if test $ac_compiler_gnu = yes; then + GXX=yes +else + GXX= +fi +ac_test_CXXFLAGS=${CXXFLAGS+set} +ac_save_CXXFLAGS=$CXXFLAGS +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CXX accepts -g" >&5 +$as_echo_n "checking whether $CXX accepts -g... " >&6; } +if ${ac_cv_prog_cxx_g+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_save_cxx_werror_flag=$ac_cxx_werror_flag + ac_cxx_werror_flag=yes + ac_cv_prog_cxx_g=no + CXXFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_prog_cxx_g=yes +else + CXXFLAGS="" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + +else + ac_cxx_werror_flag=$ac_save_cxx_werror_flag + CXXFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_prog_cxx_g=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_cxx_werror_flag=$ac_save_cxx_werror_flag +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_g" >&5 +$as_echo "$ac_cv_prog_cxx_g" >&6; } +if test "$ac_test_CXXFLAGS" = set; then + CXXFLAGS=$ac_save_CXXFLAGS +elif test $ac_cv_prog_cxx_g = yes; then + if test "$GXX" = yes; then + CXXFLAGS="-g -O2" + else + CXXFLAGS="-g" + fi +else + if test "$GXX" = yes; then + CXXFLAGS="-O2" + else + CXXFLAGS= + fi +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + +# Reset variables +if test "$GCC" != "yes" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: another compiler except for GCC was detected" >&5 +$as_echo "$as_me: WARNING: another compiler except for GCC was detected" >&2;} + MYCFLAGS="" + MYCXXFLAGS="" +fi +test -n "$CFLAGS" && MYCFLAGS="$CFLAGS $MYCFLAGS" +test -n "$CXXFLAGS" && MYCXXFLAGS="$CXXFLAGS $MYCXXFLAGS" +test -n "$CPPFLAGS" && MYCPPFLAGS="$CPPFLAGS $MYCPPFLAGS" +test -n "$LDFLAGS" && MYLDFLAGS="$LDFLAGS $MYLDFLAGS" + +# Byte order + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C++ preprocessor" >&5 +$as_echo_n "checking how to run the C++ preprocessor... " >&6; } +if test -z "$CXXCPP"; then + if ${ac_cv_prog_CXXCPP+:} false; then : + $as_echo_n "(cached) " >&6 +else + # Double quotes because CXXCPP needs to be expanded + for CXXCPP in "$CXX -E" "/lib/cpp" + do + ac_preproc_ok=false +for ac_cxx_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + # <limits.h> exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + Syntax error +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <ac_nonexistent.h> +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + break +fi + + done + ac_cv_prog_CXXCPP=$CXXCPP + +fi + CXXCPP=$ac_cv_prog_CXXCPP +else + ac_cv_prog_CXXCPP=$CXXCPP +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CXXCPP" >&5 +$as_echo "$CXXCPP" >&6; } +ac_preproc_ok=false +for ac_cxx_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since + # <limits.h> exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include <limits.h> +#else +# include <assert.h> +#endif + Syntax error +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <ac_nonexistent.h> +_ACEOF +if ac_fn_cxx_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "C++ preprocessor \"$CXXCPP\" fails sanity check +See \`config.log' for more details" "$LINENO" 5; } +fi + +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 +$as_echo_n "checking for grep that handles long lines and -e... " >&6; } +if ${ac_cv_path_GREP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$GREP"; then + ac_path_GREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in grep ggrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_GREP" && $as_test_x "$ac_path_GREP"; } || continue +# Check for GNU ac_path_GREP and select it if it is found. + # Check for GNU $ac_path_GREP +case `"$ac_path_GREP" --version 2>&1` in +*GNU*) + ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'GREP' >> "conftest.nl" + "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_GREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_GREP="$ac_path_GREP" + ac_path_GREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_GREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_GREP"; then + as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + fi +else + ac_cv_path_GREP=$GREP +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 +$as_echo "$ac_cv_path_GREP" >&6; } + GREP="$ac_cv_path_GREP" + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 +$as_echo_n "checking for egrep... " >&6; } +if ${ac_cv_path_EGREP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 + then ac_cv_path_EGREP="$GREP -E" + else + if test -z "$EGREP"; then + ac_path_EGREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in egrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" + { test -f "$ac_path_EGREP" && $as_test_x "$ac_path_EGREP"; } || continue +# Check for GNU ac_path_EGREP and select it if it is found. + # Check for GNU $ac_path_EGREP +case `"$ac_path_EGREP" --version 2>&1` in +*GNU*) + ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'EGREP' >> "conftest.nl" + "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_EGREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_EGREP="$ac_path_EGREP" + ac_path_EGREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_EGREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_EGREP"; then + as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + fi +else + ac_cv_path_EGREP=$EGREP +fi + + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 +$as_echo "$ac_cv_path_EGREP" >&6; } + EGREP="$ac_cv_path_EGREP" + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 +$as_echo_n "checking for ANSI C header files... " >&6; } +if ${ac_cv_header_stdc+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <stdlib.h> +#include <stdarg.h> +#include <string.h> +#include <float.h> + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_header_stdc=yes +else + ac_cv_header_stdc=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +if test $ac_cv_header_stdc = yes; then + # SunOS 4.x string.h does not declare mem*, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <string.h> + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "memchr" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <stdlib.h> + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "free" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. + if test "$cross_compiling" = yes; then : + : +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <ctype.h> +#include <stdlib.h> +#if ((' ' & 0x0FF) == 0x020) +# define ISLOWER(c) ('a' <= (c) && (c) <= 'z') +# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) +#else +# define ISLOWER(c) \ + (('a' <= (c) && (c) <= 'i') \ + || ('j' <= (c) && (c) <= 'r') \ + || ('s' <= (c) && (c) <= 'z')) +# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) +#endif + +#define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) +int +main () +{ + int i; + for (i = 0; i < 256; i++) + if (XOR (islower (i), ISLOWER (i)) + || toupper (i) != TOUPPER (i)) + return 2; + return 0; +} +_ACEOF +if ac_fn_cxx_try_run "$LINENO"; then : + +else + ac_cv_header_stdc=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5 +$as_echo "$ac_cv_header_stdc" >&6; } +if test $ac_cv_header_stdc = yes; then + +$as_echo "#define STDC_HEADERS 1" >>confdefs.h + +fi + +# On IRIX 5.3, sys/types and inttypes.h are conflicting. +for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ + inttypes.h stdint.h unistd.h +do : + as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` +ac_fn_cxx_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default +" +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + +fi + +done + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether byte ordering is bigendian" >&5 +$as_echo_n "checking whether byte ordering is bigendian... " >&6; } +if ${ac_cv_c_bigendian+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_c_bigendian=unknown + # See if we're dealing with a universal compiler. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifndef __APPLE_CC__ + not a universal capable compiler + #endif + typedef int dummy; + +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + + # Check for potential -arch flags. It is not universal unless + # there are at least two -arch flags with different values. + ac_arch= + ac_prev= + for ac_word in $CC $CFLAGS $CPPFLAGS $LDFLAGS; do + if test -n "$ac_prev"; then + case $ac_word in + i?86 | x86_64 | ppc | ppc64) + if test -z "$ac_arch" || test "$ac_arch" = "$ac_word"; then + ac_arch=$ac_word + else + ac_cv_c_bigendian=universal + break + fi + ;; + esac + ac_prev= + elif test "x$ac_word" = "x-arch"; then + ac_prev=arch + fi + done +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + if test $ac_cv_c_bigendian = unknown; then + # See if sys/param.h defines the BYTE_ORDER macro. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <sys/types.h> + #include <sys/param.h> + +int +main () +{ +#if ! (defined BYTE_ORDER && defined BIG_ENDIAN \ + && defined LITTLE_ENDIAN && BYTE_ORDER && BIG_ENDIAN \ + && LITTLE_ENDIAN) + bogus endian macros + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + # It does; now see whether it defined to BIG_ENDIAN or not. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <sys/types.h> + #include <sys/param.h> + +int +main () +{ +#if BYTE_ORDER != BIG_ENDIAN + not big endian + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_c_bigendian=yes +else + ac_cv_c_bigendian=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + fi + if test $ac_cv_c_bigendian = unknown; then + # See if <limits.h> defines _LITTLE_ENDIAN or _BIG_ENDIAN (e.g., Solaris). + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <limits.h> + +int +main () +{ +#if ! (defined _LITTLE_ENDIAN || defined _BIG_ENDIAN) + bogus endian macros + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + # It does; now see whether it defined to _BIG_ENDIAN or not. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <limits.h> + +int +main () +{ +#ifndef _BIG_ENDIAN + not big endian + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_c_bigendian=yes +else + ac_cv_c_bigendian=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + fi + if test $ac_cv_c_bigendian = unknown; then + # Compile a test program. + if test "$cross_compiling" = yes; then : + # Try to guess by grepping values from an object file. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +short int ascii_mm[] = + { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 }; + short int ascii_ii[] = + { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 }; + int use_ascii (int i) { + return ascii_mm[i] + ascii_ii[i]; + } + short int ebcdic_ii[] = + { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 }; + short int ebcdic_mm[] = + { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 }; + int use_ebcdic (int i) { + return ebcdic_mm[i] + ebcdic_ii[i]; + } + extern int foo; + +int +main () +{ +return use_ascii (foo) == use_ebcdic (foo); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + if grep BIGenDianSyS conftest.$ac_objext >/dev/null; then + ac_cv_c_bigendian=yes + fi + if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then + if test "$ac_cv_c_bigendian" = unknown; then + ac_cv_c_bigendian=no + else + # finding both strings is unlikely to happen, but who knows? + ac_cv_c_bigendian=unknown + fi + fi +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ + + /* Are we little or big endian? From Harbison&Steele. */ + union + { + long int l; + char c[sizeof (long int)]; + } u; + u.l = 1; + return u.c[sizeof (long int) - 1] == 1; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_run "$LINENO"; then : + ac_cv_c_bigendian=no +else + ac_cv_c_bigendian=yes +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_bigendian" >&5 +$as_echo "$ac_cv_c_bigendian" >&6; } + case $ac_cv_c_bigendian in #( + yes) + MYCPPFLAGS="$MYCPPFLAGS -D_MYBIGEND";; #( + no) + ;; #( + universal) + +$as_echo "#define AC_APPLE_UNIVERSAL_BUILD 1" >>confdefs.h + + ;; #( + *) + as_fn_error $? "unknown endianness + presetting ac_cv_c_bigendian=no (or yes) will help" "$LINENO" 5 ;; + esac + + +# System-depending optimization +printf 'checking for 64-bit availability... ' +if printf 'main() {}' | $CC -xc -m64 -o config.tmp - >config.tmp 2>&1 +then + MYCFLAGS="-m64 $MYCFLAGS" + MYCXXFLAGS="-m64 $MYCXXFLAGS" + printf 'yes\n' +else + printf 'no\n' +fi +if test "$enable_opt" != "no" +then + printf 'checking for CPU optimization availability... ' + if printf 'main() {}' | $CC -xc -march=native -o config.tmp - >config.tmp 2>&1 + then + MYCFLAGS="-march=native $MYCFLAGS" + MYCXXFLAGS="-march=native $MYCXXFLAGS" + printf 'yes\n' + else + printf 'no\n' + fi +fi +printf 'checking for useless warnings... ' +if printf 'main() {}' | $CC -xc \ + -Wno-unused-but-set-variable -Wno-unused-but-set-parameter -o config.tmp - >config.tmp 2>&1 +then + MYCFLAGS="$MYCFLAGS -Wno-unused-but-set-variable -Wno-unused-but-set-parameter" + MYCXXFLAGS="$MYCXXFLAGS -Wno-unused-but-set-variable -Wno-unused-but-set-parameter" + printf 'yes\n' +else + printf 'no\n' +fi + +# Atomic operations +if test "$enable_atomic" != "no" +then + printf 'checking for atomic operations... ' + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +__sync_fetch_and_add + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + MYGCCATOMIC=yes +else + MYGCCATOMIC=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + if test "$MYGCCATOMIC" = "yes" + then + MYCPPFLAGS="$MYCPPFLAGS -D_MYGCCATOMIC" + printf 'yes\n' + else + printf 'no\n' + fi +fi + +# Underlying libraries +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lc" >&5 +$as_echo_n "checking for main in -lc... " >&6; } +if ${ac_cv_lib_c_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lc $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_c_main=yes +else + ac_cv_lib_c_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_main" >&5 +$as_echo "$ac_cv_lib_c_main" >&6; } +if test "x$ac_cv_lib_c_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBC 1 +_ACEOF + + LIBS="-lc $LIBS" + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lm" >&5 +$as_echo_n "checking for main in -lm... " >&6; } +if ${ac_cv_lib_m_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lm $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_m_main=yes +else + ac_cv_lib_m_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_m_main" >&5 +$as_echo "$ac_cv_lib_m_main" >&6; } +if test "x$ac_cv_lib_m_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBM 1 +_ACEOF + + LIBS="-lm $LIBS" + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lpthread" >&5 +$as_echo_n "checking for main in -lpthread... " >&6; } +if ${ac_cv_lib_pthread_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lpthread $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_pthread_main=yes +else + ac_cv_lib_pthread_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthread_main" >&5 +$as_echo "$ac_cv_lib_pthread_main" >&6; } +if test "x$ac_cv_lib_pthread_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBPTHREAD 1 +_ACEOF + + LIBS="-lpthread $LIBS" + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lrt" >&5 +$as_echo_n "checking for main in -lrt... " >&6; } +if ${ac_cv_lib_rt_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lrt $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_rt_main=yes +else + ac_cv_lib_rt_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_main" >&5 +$as_echo "$ac_cv_lib_rt_main" >&6; } +if test "x$ac_cv_lib_rt_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBRT 1 +_ACEOF + + LIBS="-lrt $LIBS" + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lstdc++" >&5 +$as_echo_n "checking for main in -lstdc++... " >&6; } +if ${ac_cv_lib_stdcpp_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lstdc++ $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_stdcpp_main=yes +else + ac_cv_lib_stdcpp_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_stdcpp_main" >&5 +$as_echo "$ac_cv_lib_stdcpp_main" >&6; } +if test "x$ac_cv_lib_stdcpp_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSTDC__ 1 +_ACEOF + + LIBS="-lstdc++ $LIBS" + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lregex" >&5 +$as_echo_n "checking for main in -lregex... " >&6; } +if ${ac_cv_lib_regex_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lregex $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_regex_main=yes +else + ac_cv_lib_regex_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_regex_main" >&5 +$as_echo "$ac_cv_lib_regex_main" >&6; } +if test "x$ac_cv_lib_regex_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBREGEX 1 +_ACEOF + + LIBS="-lregex $LIBS" + +fi + +if test "$enable_zlib" != "no" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lz" >&5 +$as_echo_n "checking for main in -lz... " >&6; } +if ${ac_cv_lib_z_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lz $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_z_main=yes +else + ac_cv_lib_z_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_main" >&5 +$as_echo "$ac_cv_lib_z_main" >&6; } +if test "x$ac_cv_lib_z_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBZ 1 +_ACEOF + + LIBS="-lz $LIBS" + +fi + +fi +if test "$enable_lzo" = "yes" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -llzo2" >&5 +$as_echo_n "checking for main in -llzo2... " >&6; } +if ${ac_cv_lib_lzo2_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-llzo2 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_lzo2_main=yes +else + ac_cv_lib_lzo2_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzo2_main" >&5 +$as_echo "$ac_cv_lib_lzo2_main" >&6; } +if test "x$ac_cv_lib_lzo2_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBLZO2 1 +_ACEOF + + LIBS="-llzo2 $LIBS" + +fi + +fi +if test "$enable_lzma" = "yes" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -llzma" >&5 +$as_echo_n "checking for main in -llzma... " >&6; } +if ${ac_cv_lib_lzma_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-llzma $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_lzma_main=yes +else + ac_cv_lib_lzma_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_main" >&5 +$as_echo "$ac_cv_lib_lzma_main" >&6; } +if test "x$ac_cv_lib_lzma_main" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBLZMA 1 +_ACEOF + + LIBS="-llzma $LIBS" + +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lkyotocabinet" >&5 +$as_echo_n "checking for main in -lkyotocabinet... " >&6; } +if ${ac_cv_lib_kyotocabinet_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lkyotocabinet $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + ac_cv_lib_kyotocabinet_main=yes +else + ac_cv_lib_kyotocabinet_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_kyotocabinet_main" >&5 +$as_echo "$ac_cv_lib_kyotocabinet_main" >&6; } +if test "x$ac_cv_lib_kyotocabinet_main" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: old version of Kyoto Cabinet was detected" >&5 +$as_echo "$as_me: WARNING: old version of Kyoto Cabinet was detected" >&2;} +fi + +MYLDLIBPATH="$LD_LIBRARY_PATH" + +# Necessary headers +ac_fn_cxx_check_header_mongrel "$LINENO" "stdlib.h" "ac_cv_header_stdlib_h" "$ac_includes_default" +if test "x$ac_cv_header_stdlib_h" = xyes; then : + true +else + as_fn_error $? "stdlib.h is required" "$LINENO" 5 +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "stdint.h" "ac_cv_header_stdint_h" "$ac_includes_default" +if test "x$ac_cv_header_stdint_h" = xyes; then : + true +else + as_fn_error $? "stdint.h is required" "$LINENO" 5 +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "unistd.h" "ac_cv_header_unistd_h" "$ac_includes_default" +if test "x$ac_cv_header_unistd_h" = xyes; then : + true +else + as_fn_error $? "unistd.h is required" "$LINENO" 5 +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "fcntl.h" "ac_cv_header_fcntl_h" "$ac_includes_default" +if test "x$ac_cv_header_fcntl_h" = xyes; then : + true +else + as_fn_error $? "fcntl.h is required" "$LINENO" 5 +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "dirent.h" "ac_cv_header_dirent_h" "$ac_includes_default" +if test "x$ac_cv_header_dirent_h" = xyes; then : + true +else + as_fn_error $? "dirent.h is required" "$LINENO" 5 +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "pthread.h" "ac_cv_header_pthread_h" "$ac_includes_default" +if test "x$ac_cv_header_pthread_h" = xyes; then : + true +else + as_fn_error $? "pthread.h is required" "$LINENO" 5 +fi + + +ac_fn_cxx_check_header_mongrel "$LINENO" "regex.h" "ac_cv_header_regex_h" "$ac_includes_default" +if test "x$ac_cv_header_regex_h" = xyes; then : + true +else + as_fn_error $? "regex.h is required" "$LINENO" 5 +fi + + +if test "$enable_zlib" != "no" +then + ac_fn_cxx_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" +if test "x$ac_cv_header_zlib_h" = xyes; then : + true +else + as_fn_error $? "zlib.h is required" "$LINENO" 5 +fi + + +fi +if test "$enable_lzo" = "yes" +then + ac_fn_cxx_check_header_mongrel "$LINENO" "lzo/lzo1x.h" "ac_cv_header_lzo_lzo1x_h" "$ac_includes_default" +if test "x$ac_cv_header_lzo_lzo1x_h" = xyes; then : + true +else + as_fn_error $? "lzo/lzo1x.h is required" "$LINENO" 5 +fi + + +fi +if test "$enable_lzma" = "yes" +then + ac_fn_cxx_check_header_mongrel "$LINENO" "lzma.h" "ac_cv_header_lzma_h" "$ac_includes_default" +if test "x$ac_cv_header_lzma_h" = xyes; then : + true +else + as_fn_error $? "lzma.h is required" "$LINENO" 5 +fi + + +fi + +# Static linking +if test "$is_static" = "yes" +then + MYCMDLDFLAGS="$MYCMDLDFLAGS -static" + MYCMDLIBS="$MYCMDLIBS $LIBS" +fi + +# As-needed linking +if uname | grep Linux >config.tmp +then + MYLDFLAGS="$MYLDFLAGS -Wl,-rpath-link,.:/usr/local/lib:$MYLDLIBPATH" + MYLDFLAGS="$MYLDFLAGS -Wl,--as-needed" +else + MYCMDLIBS="$MYCMDLIBS $LIBS" +fi + +# Shared libraries +if test "$enable_shared" != "no" && test "$enable_profile" != "yes" +then + if uname | grep Darwin >config.tmp + then + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.$MYLIBVER.$MYLIBREV.0.dylib" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.$MYLIBVER.dylib" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.dylib" + MYLDLIBPATHENV="DYLD_LIBRARY_PATH" + else + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.so.$MYLIBVER.$MYLIBREV.0" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.so.$MYLIBVER" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.so" + fi +fi + +# Work around of bugs of some environments +if uname | grep Darwin >config.tmp +then + MYCFLAGS="$MYCFLAGS -Os" + MYCXXFLAGS="$MYCXXFLAGS -Os" +fi + + + +#================================================================ +# Generic Settings +#================================================================ + +# Export variables + + + + + + + + + + + + + + + + + + + + +# Targets +ac_config_files="$ac_config_files Makefile kyotocabinet.pc" + +cat >confcache <<\_ACEOF +# This file is a shell script that caches the results of configure +# tests run on this system so they can be shared between configure +# scripts and configure runs, see configure's option --config-cache. +# It is not useful on other systems. If it contains results you don't +# want to keep, you may remove or edit it. +# +# config.status only pays attention to the cache file if you give it +# the --recheck option to rerun configure. +# +# `ac_cv_env_foo' variables (set or unset) will be overridden when +# loading this file, other *unset* `ac_cv_foo' will be assigned the +# following values. + +_ACEOF + +# The following way of writing the cache mishandles newlines in values, +# but we know of no workaround that is simple, portable, and efficient. +# So, we kill variables containing newlines. +# Ultrix sh set writes to stderr and can't be redirected directly, +# and sets the high bit in the cache file unless we assign to the vars. +( + for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do + eval ac_val=\$$ac_var + case $ac_val in #( + *${as_nl}*) + case $ac_var in #( + *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 +$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; + esac + case $ac_var in #( + _ | IFS | as_nl) ;; #( + BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( + *) { eval $ac_var=; unset $ac_var;} ;; + esac ;; + esac + done + + (set) 2>&1 | + case $as_nl`(ac_space=' '; set) 2>&1` in #( + *${as_nl}ac_space=\ *) + # `set' does not quote correctly, so add quotes: double-quote + # substitution turns \\\\ into \\, and sed turns \\ into \. + sed -n \ + "s/'/'\\\\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" + ;; #( + *) + # `set' quotes correctly as required by POSIX, so do not add quotes. + sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" + ;; + esac | + sort +) | + sed ' + /^ac_cv_env_/b end + t clear + :clear + s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ + t end + s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ + :end' >>confcache +if diff "$cache_file" confcache >/dev/null 2>&1; then :; else + if test -w "$cache_file"; then + if test "x$cache_file" != "x/dev/null"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 +$as_echo "$as_me: updating cache $cache_file" >&6;} + if test ! -f "$cache_file" || test -h "$cache_file"; then + cat confcache >"$cache_file" + else + case $cache_file in #( + */* | ?:*) + mv -f confcache "$cache_file"$$ && + mv -f "$cache_file"$$ "$cache_file" ;; #( + *) + mv -f confcache "$cache_file" ;; + esac + fi + fi + else + { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 +$as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} + fi +fi +rm -f confcache + +test "x$prefix" = xNONE && prefix=$ac_default_prefix +# Let make expand exec_prefix. +test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' + +# Transform confdefs.h into DEFS. +# Protect against shell expansion while executing Makefile rules. +# Protect against Makefile macro expansion. +# +# If the first sed substitution is executed (which looks for macros that +# take arguments), then branch to the quote section. Otherwise, +# look for a macro that doesn't take arguments. +ac_script=' +:mline +/\\$/{ + N + s,\\\n,, + b mline +} +t clear +:clear +s/^[ ]*#[ ]*define[ ][ ]*\([^ (][^ (]*([^)]*)\)[ ]*\(.*\)/-D\1=\2/g +t quote +s/^[ ]*#[ ]*define[ ][ ]*\([^ ][^ ]*\)[ ]*\(.*\)/-D\1=\2/g +t quote +b any +:quote +s/[ `~#$^&*(){}\\|;'\''"<>?]/\\&/g +s/\[/\\&/g +s/\]/\\&/g +s/\$/$$/g +H +:any +${ + g + s/^\n// + s/\n/ /g + p +} +' +DEFS=`sed -n "$ac_script" confdefs.h` + + +ac_libobjs= +ac_ltlibobjs= +U= +for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue + # 1. Remove the extension, and $U if already installed. + ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' + ac_i=`$as_echo "$ac_i" | sed "$ac_script"` + # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR + # will be set to the directory where LIBOBJS objects are built. + as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext" + as_fn_append ac_ltlibobjs " \${LIBOBJDIR}$ac_i"'$U.lo' +done +LIBOBJS=$ac_libobjs + +LTLIBOBJS=$ac_ltlibobjs + + + + +: "${CONFIG_STATUS=./config.status}" +ac_write_fail=0 +ac_clean_files_save=$ac_clean_files +ac_clean_files="$ac_clean_files $CONFIG_STATUS" +{ $as_echo "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5 +$as_echo "$as_me: creating $CONFIG_STATUS" >&6;} +as_write_fail=0 +cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1 +#! $SHELL +# Generated by $as_me. +# Run this file to recreate the current configuration. +# Compiler output produced by configure, useful for debugging +# configure, is in config.log if it exists. + +debug=false +ac_cs_recheck=false +ac_cs_silent=false + +SHELL=\${CONFIG_SHELL-$SHELL} +export SHELL +_ASEOF +cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 +## -------------------- ## +## M4sh Initialization. ## +## -------------------- ## + +# Be more Bourne compatible +DUALCASE=1; export DUALCASE # for MKS sh +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : + emulate sh + NULLCMD=: + # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which + # is contrary to our usage. Disable this feature. + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in #( + *posix*) : + set -o posix ;; #( + *) : + ;; +esac +fi + + +as_nl=' +' +export as_nl +# Printing a long string crashes Solaris 7 /usr/bin/printf. +as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo +as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo +# Prefer a ksh shell builtin over an external printf program on Solaris, +# but without wasting forks for bash or zsh. +if test -z "$BASH_VERSION$ZSH_VERSION" \ + && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='print -r --' + as_echo_n='print -rn --' +elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then + as_echo='printf %s\n' + as_echo_n='printf %s' +else + if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then + as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' + as_echo_n='/usr/ucb/echo -n' + else + as_echo_body='eval expr "X$1" : "X\\(.*\\)"' + as_echo_n_body='eval + arg=$1; + case $arg in #( + *"$as_nl"*) + expr "X$arg" : "X\\(.*\\)$as_nl"; + arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; + esac; + expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" + ' + export as_echo_n_body + as_echo_n='sh -c $as_echo_n_body as_echo' + fi + export as_echo_body + as_echo='sh -c $as_echo_body as_echo' +fi + +# The user is always right. +if test "${PATH_SEPARATOR+set}" != set; then + PATH_SEPARATOR=: + (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { + (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || + PATH_SEPARATOR=';' + } +fi + + +# IFS +# We need space, tab and new line, in precisely that order. Quoting is +# there to prevent editors from complaining about space-tab. +# (If _AS_PATH_WALK were called with IFS unset, it would disable word +# splitting by setting IFS to empty value.) +IFS=" "" $as_nl" + +# Find who we are. Look in the path if we contain no directory separator. +as_myself= +case $0 in #(( + *[\\/]* ) as_myself=$0 ;; + *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break + done +IFS=$as_save_IFS + + ;; +esac +# We did not find ourselves, most probably we were run as `sh COMMAND' +# in which case we are not to be found in the path. +if test "x$as_myself" = x; then + as_myself=$0 +fi +if test ! -f "$as_myself"; then + $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + exit 1 +fi + +# Unset variables that we do not need and which cause bugs (e.g. in +# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" +# suppresses any "Segmentation fault" message there. '((' could +# trigger a bug in pdksh 5.2.14. +for as_var in BASH_ENV ENV MAIL MAILPATH +do eval test x\${$as_var+set} = xset \ + && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : +done +PS1='$ ' +PS2='> ' +PS4='+ ' + +# NLS nuisances. +LC_ALL=C +export LC_ALL +LANGUAGE=C +export LANGUAGE + +# CDPATH. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + + +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- +# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are +# provided, also output the error to LOG_FD, referencing LINENO. Then exit the +# script with STATUS, using 1 if that was 0. +as_fn_error () +{ + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + fi + $as_echo "$as_me: error: $2" >&2 + as_fn_exit $as_status +} # as_fn_error + + +# as_fn_set_status STATUS +# ----------------------- +# Set $? to STATUS, without forking. +as_fn_set_status () +{ + return $1 +} # as_fn_set_status + +# as_fn_exit STATUS +# ----------------- +# Exit the shell with STATUS, even in a "trap 0" or "set -e" context. +as_fn_exit () +{ + set +e + as_fn_set_status $1 + exit $1 +} # as_fn_exit + +# as_fn_unset VAR +# --------------- +# Portably unset VAR. +as_fn_unset () +{ + { eval $1=; unset $1;} +} +as_unset=as_fn_unset +# as_fn_append VAR VALUE +# ---------------------- +# Append the text in VALUE to the end of the definition contained in VAR. Take +# advantage of any shell optimizations that allow amortized linear growth over +# repeated appends, instead of the typical quadratic growth present in naive +# implementations. +if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : + eval 'as_fn_append () + { + eval $1+=\$2 + }' +else + as_fn_append () + { + eval $1=\$$1\$2 + } +fi # as_fn_append + +# as_fn_arith ARG... +# ------------------ +# Perform arithmetic evaluation on the ARGs, and store the result in the +# global $as_val. Take advantage of shells that can avoid forks. The arguments +# must be portable across $(()) and expr. +if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : + eval 'as_fn_arith () + { + as_val=$(( $* )) + }' +else + as_fn_arith () + { + as_val=`expr "$@" || test $? -eq 1` + } +fi # as_fn_arith + + +if expr a : '\(a\)' >/dev/null 2>&1 && + test "X`expr 00001 : '.*\(...\)'`" = X001; then + as_expr=expr +else + as_expr=false +fi + +if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then + as_basename=basename +else + as_basename=false +fi + +if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then + as_dirname=dirname +else + as_dirname=false +fi + +as_me=`$as_basename -- "$0" || +$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ + X"$0" : 'X\(//\)$' \| \ + X"$0" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X/"$0" | + sed '/^.*\/\([^/][^/]*\)\/*$/{ + s//\1/ + q + } + /^X\/\(\/\/\)$/{ + s//\1/ + q + } + /^X\/\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + +# Avoid depending upon Character Ranges. +as_cr_letters='abcdefghijklmnopqrstuvwxyz' +as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' +as_cr_Letters=$as_cr_letters$as_cr_LETTERS +as_cr_digits='0123456789' +as_cr_alnum=$as_cr_Letters$as_cr_digits + +ECHO_C= ECHO_N= ECHO_T= +case `echo -n x` in #((((( +-n*) + case `echo 'xy\c'` in + *c*) ECHO_T=' ';; # ECHO_T is single tab character. + xy) ECHO_C='\c';; + *) echo `echo ksh88 bug on AIX 6.1` > /dev/null + ECHO_T=' ';; + esac;; +*) + ECHO_N='-n';; +esac + +rm -f conf$$ conf$$.exe conf$$.file +if test -d conf$$.dir; then + rm -f conf$$.dir/conf$$.file +else + rm -f conf$$.dir + mkdir conf$$.dir 2>/dev/null +fi +if (echo >conf$$.file) 2>/dev/null; then + if ln -s conf$$.file conf$$ 2>/dev/null; then + as_ln_s='ln -s' + # ... but there are two gotchas: + # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. + # In both cases, we have to default to `cp -p'. + ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || + as_ln_s='cp -p' + elif ln conf$$.file conf$$ 2>/dev/null; then + as_ln_s=ln + else + as_ln_s='cp -p' + fi +else + as_ln_s='cp -p' +fi +rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file +rmdir conf$$.dir 2>/dev/null + + +# as_fn_mkdir_p +# ------------- +# Create "$as_dir" as a directory, including parents if necessary. +as_fn_mkdir_p () +{ + + case $as_dir in #( + -*) as_dir=./$as_dir;; + esac + test -d "$as_dir" || eval $as_mkdir_p || { + as_dirs= + while :; do + case $as_dir in #( + *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *) as_qdir=$as_dir;; + esac + as_dirs="'$as_qdir' $as_dirs" + as_dir=`$as_dirname -- "$as_dir" || +$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$as_dir" : 'X\(//\)[^/]' \| \ + X"$as_dir" : 'X\(//\)$' \| \ + X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$as_dir" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + test -d "$as_dir" && break + done + test -z "$as_dirs" || eval "mkdir $as_dirs" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" + + +} # as_fn_mkdir_p +if mkdir -p . 2>/dev/null; then + as_mkdir_p='mkdir -p "$as_dir"' +else + test -d ./-p && rmdir ./-p + as_mkdir_p=false +fi + +if test -x / >/dev/null 2>&1; then + as_test_x='test -x' +else + if ls -dL / >/dev/null 2>&1; then + as_ls_L_option=L + else + as_ls_L_option= + fi + as_test_x=' + eval sh -c '\'' + if test -d "$1"; then + test -d "$1/."; + else + case $1 in #( + -*)set "./$1";; + esac; + case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #(( + ???[sx]*):;;*)false;;esac;fi + '\'' sh + ' +fi +as_executable_p=$as_test_x + +# Sed expression to map a string onto a valid CPP name. +as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" + +# Sed expression to map a string onto a valid variable name. +as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" + + +exec 6>&1 +## ----------------------------------- ## +## Main body of $CONFIG_STATUS script. ## +## ----------------------------------- ## +_ASEOF +test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# Save the log message, to keep $0 and so on meaningful, and to +# report actual input values of CONFIG_FILES etc. instead of their +# values after options handling. +ac_log=" +This file was extended by kyotocabinet $as_me 1.2.76, which was +generated by GNU Autoconf 2.68. Invocation command line was + + CONFIG_FILES = $CONFIG_FILES + CONFIG_HEADERS = $CONFIG_HEADERS + CONFIG_LINKS = $CONFIG_LINKS + CONFIG_COMMANDS = $CONFIG_COMMANDS + $ $0 $@ + +on `(hostname || uname -n) 2>/dev/null | sed 1q` +" + +_ACEOF + +case $ac_config_files in *" +"*) set x $ac_config_files; shift; ac_config_files=$*;; +esac + + + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +# Files that config.status was made for. +config_files="$ac_config_files" + +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +ac_cs_usage="\ +\`$as_me' instantiates files and other configuration actions +from templates according to the current configuration. Unless the files +and actions are specified as TAGs, all are instantiated by default. + +Usage: $0 [OPTION]... [TAG]... + + -h, --help print this help, then exit + -V, --version print version number and configuration settings, then exit + --config print configuration, then exit + -q, --quiet, --silent + do not print progress messages + -d, --debug don't remove temporary files + --recheck update $as_me by reconfiguring in the same conditions + --file=FILE[:TEMPLATE] + instantiate the configuration file FILE + +Configuration files: +$config_files + +Report bugs to the package provider." + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" +ac_cs_version="\\ +kyotocabinet config.status 1.2.76 +configured by $0, generated by GNU Autoconf 2.68, + with options \\"\$ac_cs_config\\" + +Copyright (C) 2010 Free Software Foundation, Inc. +This config.status script is free software; the Free Software Foundation +gives unlimited permission to copy, distribute and modify it." + +ac_pwd='$ac_pwd' +srcdir='$srcdir' +test -n "\$AWK" || AWK=awk +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# The default lists apply if the user does not specify any file. +ac_need_defaults=: +while test $# != 0 +do + case $1 in + --*=?*) + ac_option=`expr "X$1" : 'X\([^=]*\)='` + ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` + ac_shift=: + ;; + --*=) + ac_option=`expr "X$1" : 'X\([^=]*\)='` + ac_optarg= + ac_shift=: + ;; + *) + ac_option=$1 + ac_optarg=$2 + ac_shift=shift + ;; + esac + + case $ac_option in + # Handling of the options. + -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) + ac_cs_recheck=: ;; + --version | --versio | --versi | --vers | --ver | --ve | --v | -V ) + $as_echo "$ac_cs_version"; exit ;; + --config | --confi | --conf | --con | --co | --c ) + $as_echo "$ac_cs_config"; exit ;; + --debug | --debu | --deb | --de | --d | -d ) + debug=: ;; + --file | --fil | --fi | --f ) + $ac_shift + case $ac_optarg in + *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + '') as_fn_error $? "missing file argument" ;; + esac + as_fn_append CONFIG_FILES " '$ac_optarg'" + ac_need_defaults=false;; + --he | --h | --help | --hel | -h ) + $as_echo "$ac_cs_usage"; exit ;; + -q | -quiet | --quiet | --quie | --qui | --qu | --q \ + | -silent | --silent | --silen | --sile | --sil | --si | --s) + ac_cs_silent=: ;; + + # This is an error. + -*) as_fn_error $? "unrecognized option: \`$1' +Try \`$0 --help' for more information." ;; + + *) as_fn_append ac_config_targets " $1" + ac_need_defaults=false ;; + + esac + shift +done + +ac_configure_extra_args= + +if $ac_cs_silent; then + exec 6>/dev/null + ac_configure_extra_args="$ac_configure_extra_args --silent" +fi + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +if \$ac_cs_recheck; then + set X '$SHELL' '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion + shift + \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6 + CONFIG_SHELL='$SHELL' + export CONFIG_SHELL + exec "\$@" +fi + +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +exec 5>>config.log +{ + echo + sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX +## Running $as_me. ## +_ASBOX + $as_echo "$ac_log" +} >&5 + +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 + +# Handling of arguments. +for ac_config_target in $ac_config_targets +do + case $ac_config_target in + "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; + "kyotocabinet.pc") CONFIG_FILES="$CONFIG_FILES kyotocabinet.pc" ;; + + *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; + esac +done + + +# If the user did not use the arguments to specify the items to instantiate, +# then the envvar interface is used. Set only those that are not. +# We use the long form for the default assignment because of an extremely +# bizarre bug on SunOS 4.1.3. +if $ac_need_defaults; then + test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files +fi + +# Have a temporary directory for convenience. Make it in the build tree +# simply because there is no reason against having it here, and in addition, +# creating and moving files from /tmp can sometimes cause problems. +# Hook for its removal unless debugging. +# Note that there is a small window in which the directory will not be cleaned: +# after its creation but before its name has been assigned to `$tmp'. +$debug || +{ + tmp= ac_tmp= + trap 'exit_status=$? + : "${ac_tmp:=$tmp}" + { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status +' 0 + trap 'as_fn_exit 1' 1 2 13 15 +} +# Create a (secure) tmp directory for tmp files. + +{ + tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && + test -d "$tmp" +} || +{ + tmp=./conf$$-$RANDOM + (umask 077 && mkdir "$tmp") +} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 +ac_tmp=$tmp + +# Set up the scripts for CONFIG_FILES section. +# No need to generate them if there are no CONFIG_FILES. +# This happens for instance with `./config.status config.h'. +if test -n "$CONFIG_FILES"; then + + +ac_cr=`echo X | tr X '\015'` +# On cygwin, bash can eat \r inside `` if the user requested igncr. +# But we know of no other shell where ac_cr would be empty at this +# point, so we can use a bashism as a fallback. +if test "x$ac_cr" = x; then + eval ac_cr=\$\'\\r\' +fi +ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' </dev/null 2>/dev/null` +if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then + ac_cs_awk_cr='\\r' +else + ac_cs_awk_cr=$ac_cr +fi + +echo 'BEGIN {' >"$ac_tmp/subs1.awk" && +_ACEOF + + +{ + echo "cat >conf$$subs.awk <<_ACEOF" && + echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && + echo "_ACEOF" +} >conf$$subs.sh || + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 +ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` +ac_delim='%!_!# ' +for ac_last_try in false false false false false :; do + . ./conf$$subs.sh || + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 + + ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` + if test $ac_delim_n = $ac_delim_num; then + break + elif $ac_last_try; then + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 + else + ac_delim="$ac_delim!$ac_delim _$ac_delim!! " + fi +done +rm -f conf$$subs.sh + +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && +_ACEOF +sed -n ' +h +s/^/S["/; s/!.*/"]=/ +p +g +s/^[^!]*!// +:repl +t repl +s/'"$ac_delim"'$// +t delim +:nl +h +s/\(.\{148\}\)..*/\1/ +t more1 +s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/ +p +n +b repl +:more1 +s/["\\]/\\&/g; s/^/"/; s/$/"\\/ +p +g +s/.\{148\}// +t nl +:delim +h +s/\(.\{148\}\)..*/\1/ +t more2 +s/["\\]/\\&/g; s/^/"/; s/$/"/ +p +b +:more2 +s/["\\]/\\&/g; s/^/"/; s/$/"\\/ +p +g +s/.\{148\}// +t delim +' <conf$$subs.awk | sed ' +/^[^""]/{ + N + s/\n// +} +' >>$CONFIG_STATUS || ac_write_fail=1 +rm -f conf$$subs.awk +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +_ACAWK +cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && + for (key in S) S_is_set[key] = 1 + FS = "" + +} +{ + line = $ 0 + nfields = split(line, field, "@") + substed = 0 + len = length(field[1]) + for (i = 2; i < nfields; i++) { + key = field[i] + keylen = length(key) + if (S_is_set[key]) { + value = S[key] + line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3) + len += length(value) + length(field[++i]) + substed = 1 + } else + len += 1 + keylen + } + + print line +} + +_ACAWK +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then + sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" +else + cat +fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \ + || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 +_ACEOF + +# VPATH may cause trouble with some makes, so we remove sole $(srcdir), +# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and +# trailing colons and then remove the whole line if VPATH becomes empty +# (actually we leave an empty line to preserve line numbers). +if test "x$srcdir" = x.; then + ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ +h +s/// +s/^/:/ +s/[ ]*$/:/ +s/:\$(srcdir):/:/g +s/:\${srcdir}:/:/g +s/:@srcdir@:/:/g +s/^:*// +s/:*$// +x +s/\(=[ ]*\).*/\1/ +G +s/\n// +s/^[^=]*=[ ]*$// +}' +fi + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +fi # test -n "$CONFIG_FILES" + + +eval set X " :F $CONFIG_FILES " +shift +for ac_tag +do + case $ac_tag in + :[FHLC]) ac_mode=$ac_tag; continue;; + esac + case $ac_mode$ac_tag in + :[FHL]*:*);; + :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; + :[FH]-) ac_tag=-:-;; + :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; + esac + ac_save_IFS=$IFS + IFS=: + set x $ac_tag + IFS=$ac_save_IFS + shift + ac_file=$1 + shift + + case $ac_mode in + :L) ac_source=$1;; + :[FH]) + ac_file_inputs= + for ac_f + do + case $ac_f in + -) ac_f="$ac_tmp/stdin";; + *) # Look for the file first in the build tree, then in the source tree + # (if the path is not absolute). The absolute path cannot be DOS-style, + # because $ac_f cannot contain `:'. + test -f "$ac_f" || + case $ac_f in + [\\/$]*) false;; + *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; + esac || + as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; + esac + case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac + as_fn_append ac_file_inputs " '$ac_f'" + done + + # Let's still pretend it is `configure' which instantiates (i.e., don't + # use $as_me), people would be surprised to read: + # /* config.h. Generated by config.status. */ + configure_input='Generated from '` + $as_echo "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' + `' by configure.' + if test x"$ac_file" != x-; then + configure_input="$ac_file. $configure_input" + { $as_echo "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5 +$as_echo "$as_me: creating $ac_file" >&6;} + fi + # Neutralize special characters interpreted by sed in replacement strings. + case $configure_input in #( + *\&* | *\|* | *\\* ) + ac_sed_conf_input=`$as_echo "$configure_input" | + sed 's/[\\\\&|]/\\\\&/g'`;; #( + *) ac_sed_conf_input=$configure_input;; + esac + + case $ac_tag in + *:-:* | *:-) cat >"$ac_tmp/stdin" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; + esac + ;; + esac + + ac_dir=`$as_dirname -- "$ac_file" || +$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ac_file" : 'X\(//\)[^/]' \| \ + X"$ac_file" : 'X\(//\)$' \| \ + X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$ac_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + as_dir="$ac_dir"; as_fn_mkdir_p + ac_builddir=. + +case "$ac_dir" in +.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; +*) + ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` + # A ".." for each directory in $ac_dir_suffix. + ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + case $ac_top_builddir_sub in + "") ac_top_builddir_sub=. ac_top_build_prefix= ;; + *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; + esac ;; +esac +ac_abs_top_builddir=$ac_pwd +ac_abs_builddir=$ac_pwd$ac_dir_suffix +# for backward compatibility: +ac_top_builddir=$ac_top_build_prefix + +case $srcdir in + .) # We are building in place. + ac_srcdir=. + ac_top_srcdir=$ac_top_builddir_sub + ac_abs_top_srcdir=$ac_pwd ;; + [\\/]* | ?:[\\/]* ) # Absolute name. + ac_srcdir=$srcdir$ac_dir_suffix; + ac_top_srcdir=$srcdir + ac_abs_top_srcdir=$srcdir ;; + *) # Relative name. + ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix + ac_top_srcdir=$ac_top_build_prefix$srcdir + ac_abs_top_srcdir=$ac_pwd/$srcdir ;; +esac +ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix + + + case $ac_mode in + :F) + # + # CONFIG_FILE + # + +_ACEOF + +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +# If the template does not know about datarootdir, expand it. +# FIXME: This hack should be removed a few years after 2.60. +ac_datarootdir_hack=; ac_datarootdir_seen= +ac_sed_dataroot=' +/datarootdir/ { + p + q +} +/@datadir@/p +/@docdir@/p +/@infodir@/p +/@localedir@/p +/@mandir@/p' +case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in +*datarootdir*) ac_datarootdir_seen=yes;; +*@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 +$as_echo "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} +_ACEOF +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 + ac_datarootdir_hack=' + s&@datadir@&$datadir&g + s&@docdir@&$docdir&g + s&@infodir@&$infodir&g + s&@localedir@&$localedir&g + s&@mandir@&$mandir&g + s&\\\${datarootdir}&$datarootdir&g' ;; +esac +_ACEOF + +# Neutralize VPATH when `$srcdir' = `.'. +# Shell code in configure.ac might set extrasub. +# FIXME: do we really want to maintain this feature? +cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +ac_sed_extra="$ac_vpsub +$extrasub +_ACEOF +cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +:t +/@[a-zA-Z_][a-zA-Z_0-9]*@/!b +s|@configure_input@|$ac_sed_conf_input|;t t +s&@top_builddir@&$ac_top_builddir_sub&;t t +s&@top_build_prefix@&$ac_top_build_prefix&;t t +s&@srcdir@&$ac_srcdir&;t t +s&@abs_srcdir@&$ac_abs_srcdir&;t t +s&@top_srcdir@&$ac_top_srcdir&;t t +s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t +s&@builddir@&$ac_builddir&;t t +s&@abs_builddir@&$ac_abs_builddir&;t t +s&@abs_top_builddir@&$ac_abs_top_builddir&;t t +$ac_datarootdir_hack +" +eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \ + >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + +test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && + { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && + { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ + "$ac_tmp/out"`; test -z "$ac_out"; } && + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' +which seems to be undefined. Please make sure it is defined" >&5 +$as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' +which seems to be undefined. Please make sure it is defined" >&2;} + + rm -f "$ac_tmp/stdin" + case $ac_file in + -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";; + *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";; + esac \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + ;; + + + + esac + +done # for ac_tag + + +as_fn_exit 0 +_ACEOF +ac_clean_files=$ac_clean_files_save + +test $ac_write_fail = 0 || + as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 + + +# configure is writing to config.log, and then calls config.status. +# config.status does its own redirection, appending to config.log. +# Unfortunately, on DOS this fails, as config.log is still kept open +# by configure, so config.status won't be able to write to it; its +# output is simply discarded. So we exec the FD to /dev/null, +# effectively closing config.log, so it can be properly (re)opened and +# appended to by config.status. When coming back to configure, we +# need to make the FD available again. +if test "$no_create" != yes; then + ac_cs_success=: + ac_config_status_args= + test "$silent" = yes && + ac_config_status_args="$ac_config_status_args --quiet" + exec 5>/dev/null + $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false + exec 5>>config.log + # Use ||, not &&, to avoid exiting from the if with $? = 1, which + # would make configure fail if this is the last instruction. + $ac_cs_success || as_fn_exit 1 +fi +if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 +$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} +fi + + +# Messages +printf '#================================================================\n' +printf '# Ready to make.\n' +printf '#================================================================\n' + + + +# END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/configure.in b/plugins/Dbx_kyoto/src/kyotocabinet/configure.in new file mode 100644 index 0000000000..7808806202 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/configure.in @@ -0,0 +1,371 @@ +# Source of configuration for Kyoto Cabinet + + + +#================================================================ +# Generic Settings +#================================================================ + +# Package name +AC_INIT(kyotocabinet, 1.2.76) + +# Package information +MYLIBVER=16 +MYLIBREV=13 +MYFORMATVER=5 + +# Targets +MYHEADERFILES="kccommon.h kcutil.h kcthread.h kcfile.h" +MYHEADERFILES="$MYHEADERFILES kccompress.h kccompare.h kcmap.h kcregex.h" +MYHEADERFILES="$MYHEADERFILES kcdb.h kcplantdb.h kcprotodb.h kcstashdb.h kccachedb.h" +MYHEADERFILES="$MYHEADERFILES kchashdb.h kcdirdb.h kctextdb.h kcpolydb.h kcdbext.h kclangc.h" +MYLIBRARYFILES="libkyotocabinet.a" +MYLIBOBJFILES="kcutil.o kcthread.o kcfile.o kccompress.o kccompare.o kcmap.o kcregex.o" +MYLIBOBJFILES="$MYLIBOBJFILES kcdb.o kcplantdb.o kcprotodb.o kcstashdb.o kccachedb.o" +MYLIBOBJFILES="$MYLIBOBJFILES kchashdb.o kcdirdb.o kctextdb.o kcpolydb.o kcdbext.o kclangc.o" +MYCOMMANDFILES="kcutiltest kcutilmgr kcprototest kcstashtest kccachetest kcgrasstest" +MYCOMMANDFILES="$MYCOMMANDFILES kchashtest kchashmgr kctreetest kctreemgr" +MYCOMMANDFILES="$MYCOMMANDFILES kcdirtest kcdirmgr kcforesttest kcforestmgr" +MYCOMMANDFILES="$MYCOMMANDFILES kcpolytest kcpolymgr kclangctest" +MYMAN1FILES="kcutiltest.1 kcutilmgr.1 kcprototest.1 kcstashtest.1 kccachetest.1 kcgrasstest.1" +MYMAN1FILES="$MYMAN1FILES kchashtest.1 kchashmgr.1 kctreetest.1 kctreemgr.1" +MYMAN1FILES="$MYMAN1FILES kcdirtest.1 kcdirmgr.1 kcforesttest.1 kcforestmgr.1" +MYMAN1FILES="$MYMAN1FILES kcpolytest.1 kcpolymgr.1 kclangctest.1" +MYDOCUMENTFILES="COPYING FOSSEXCEPTION ChangeLog doc kyotocabinet.idl" +MYPCFILES="kyotocabinet.pc" + +# Building flags +MYCFLAGS="-Wall -ansi -pedantic -fPIC -fsigned-char -g0 -O2" +MYCXXFLAGS="-Wall -fPIC -fsigned-char -g0 -O2" +MYCPPFLAGS="-I. -I\$(INCLUDEDIR) -I/usr/local/include" +MYCPPFLAGS="$MYCPPFLAGS -DNDEBUG -D_GNU_SOURCE=1" +MYCPPFLAGS="$MYCPPFLAGS -D_FILE_OFFSET_BITS=64 -D_REENTRANT -D__EXTENSIONS__" +MYLDFLAGS="-L. -L\$(LIBDIR) -L/usr/local/lib" +MYCMDLDFLAGS="" +MYCMDLIBS="" +MYLDLIBPATH="" +MYLDLIBPATHENV="LD_LIBRARY_PATH" +MYPOSTCMD="true" + +# Building paths +PATH=".:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:$PATH" +CPATH=".:/usr/local/include:$CPATH" +LIBRARY_PATH=".:/usr/local/lib:$LIBRARY_PATH" +LD_LIBRARY_PATH=".:/usr/local/lib:$LD_LIBRARY_PATH" +PKG_CONFIG_PATH=".:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" +export PATH CPATH LIBRARY_PATH LD_LIBRARY_PATH PKG_CONFIG_PATH + + + +#================================================================ +# Options +#================================================================ + +# Internal variables +enables="" +is_static="" + +# Debug mode +AC_ARG_ENABLE(debug, + AC_HELP_STRING([--enable-debug], [build for debugging])) +if test "$enable_debug" = "yes" +then + MYCFLAGS="-Wall -ansi -pedantic -fPIC -fsigned-char -g -O0" + MYCXXFLAGS="-Wall -fPIC -fsigned-char -g -O0" + MYCPPFLAGS="$MYCPPFLAGS -UNDEBUG -D_KCDEBUG" + is_static="yes" + enables="$enables (debug)" +fi + +# Developping mode +AC_ARG_ENABLE(devel, + AC_HELP_STRING([--enable-devel], [build for development])) +if test "$enable_devel" = "yes" +then + MYCFLAGS="-Wall -Wextra -Wno-unused-parameter" + MYCFLAGS="$MYCFLAGS -ansi -pedantic -fPIC -fsigned-char -O2 -fno-inline -pipe" + MYCXXFLAGS="-Wall -Wextra -Wno-unused-parameter -Wnon-virtual-dtor" + MYCXXFLAGS="$MYCXXFLAGS -fPIC -fsigned-char -g -O2 -fno-inline -pipe" + MYCPPFLAGS="$MYCPPFLAGS -UNDEBUG -D_KCDEBUG" + enables="$enables (devel)" +fi + +# Disable optimization +AC_ARG_ENABLE(opt, + AC_HELP_STRING([--disable-opt], [build without optimization])) +if test "$enable_opt" = "no" +then + MYCFLAGS="$MYCFLAGS -O0" + MYCXXFLAGS="$MYCXXFLAGS -O0" + enables="$enables (no-opt)" +fi + +# Profiling mode +AC_ARG_ENABLE(profile, + AC_HELP_STRING([--enable-profile], [build for profiling])) +if test "$enable_profile" = "yes" +then + MYCXXFLAGS="-Wall -fPIC -fsigned-char -g -pg -O2 -fno-inline -pipe" + enables="$enables (profile)" +fi + +# Micro yield mode +AC_ARG_ENABLE(uyield, + AC_HELP_STRING([--enable-uyield], [build for detecting race conditions])) +if test "$enable_uyield" = "yes" +then + MYCPPFLAGS="$MYCPPFLAGS -UNDEBUG -D_KCUYIELD" + enables="$enables (uyield)" +fi + +# Static mode +AC_ARG_ENABLE(static, + AC_HELP_STRING([--enable-static], [build by static linking])) +if test "$enable_static" = "yes" +then + is_static="yes" + enables="$enables (static)" +fi + +# Disable shared object +AC_ARG_ENABLE(shared, + AC_HELP_STRING([--disable-shared], [avoid to build shared libraries])) +if test "$enable_shared" = "no" +then + enables="$enables (no-shared)" +fi + +# Disable atomic operations +AC_ARG_ENABLE(atomic, + AC_HELP_STRING([--disable-atomic], [build without atomic operations])) +if test "$enable_atomic" = "no" +then + enables="$enables (no-atomic)" +fi + +# Disable ZLIB compression +AC_ARG_ENABLE(zlib, + AC_HELP_STRING([--disable-zlib], [build without ZLIB compression])) +if test "$enable_zlib" = "no" +then + enables="$enables (no-zlib)" +else + MYCPPFLAGS="$MYCPPFLAGS -D_MYZLIB" +fi + +# Enable LZO compression +AC_ARG_ENABLE(lzo, + AC_HELP_STRING([--enable-lzo], [build with LZO compression])) +if test "$enable_lzo" = "yes" +then + MYCPPFLAGS="$MYCPPFLAGS -D_MYLZO" + enables="$enables (lzo)" +fi + +# Enable LZMA compression +AC_ARG_ENABLE(lzma, + AC_HELP_STRING([--enable-lzma], [build with LZMA compression])) +if test "$enable_lzma" = "yes" +then + MYCPPFLAGS="$MYCPPFLAGS -D_MYLZMA" + enables="$enables (lzma)" +fi + +# Messages +printf '#================================================================\n' +printf '# Configuring Kyoto Cabinet version %s%s.\n' "$PACKAGE_VERSION" "$enables" +printf '#================================================================\n' + + + +#================================================================ +# Checking Commands and Libraries +#================================================================ + +# C and C++ compilers +AC_PROG_CC +AC_PROG_CXX +AC_LANG(C++) + +# Reset variables +if test "$GCC" != "yes" +then + AC_MSG_WARN([another compiler except for GCC was detected]) + MYCFLAGS="" + MYCXXFLAGS="" +fi +test -n "$CFLAGS" && MYCFLAGS="$CFLAGS $MYCFLAGS" +test -n "$CXXFLAGS" && MYCXXFLAGS="$CXXFLAGS $MYCXXFLAGS" +test -n "$CPPFLAGS" && MYCPPFLAGS="$CPPFLAGS $MYCPPFLAGS" +test -n "$LDFLAGS" && MYLDFLAGS="$LDFLAGS $MYLDFLAGS" + +# Byte order +AC_C_BIGENDIAN(MYCPPFLAGS="$MYCPPFLAGS -D_MYBIGEND") + +# System-depending optimization +printf 'checking for 64-bit availability... ' +if printf 'main() {}' | $CC -xc -m64 -o config.tmp - >config.tmp 2>&1 +then + MYCFLAGS="-m64 $MYCFLAGS" + MYCXXFLAGS="-m64 $MYCXXFLAGS" + printf 'yes\n' +else + printf 'no\n' +fi +if test "$enable_opt" != "no" +then + printf 'checking for CPU optimization availability... ' + if printf 'main() {}' | $CC -xc -march=native -o config.tmp - >config.tmp 2>&1 + then + MYCFLAGS="-march=native $MYCFLAGS" + MYCXXFLAGS="-march=native $MYCXXFLAGS" + printf 'yes\n' + else + printf 'no\n' + fi +fi +printf 'checking for useless warnings... ' +if printf 'main() {}' | $CC -xc \ + -Wno-unused-but-set-variable -Wno-unused-but-set-parameter -o config.tmp - >config.tmp 2>&1 +then + MYCFLAGS="$MYCFLAGS -Wno-unused-but-set-variable -Wno-unused-but-set-parameter" + MYCXXFLAGS="$MYCXXFLAGS -Wno-unused-but-set-variable -Wno-unused-but-set-parameter" + printf 'yes\n' +else + printf 'no\n' +fi + +# Atomic operations +if test "$enable_atomic" != "no" +then + printf 'checking for atomic operations... ' + AC_TRY_COMPILE([], [__sync_fetch_and_add], [MYGCCATOMIC=yes], [MYGCCATOMIC=no]) + if test "$MYGCCATOMIC" = "yes" + then + MYCPPFLAGS="$MYCPPFLAGS -D_MYGCCATOMIC" + printf 'yes\n' + else + printf 'no\n' + fi +fi + +# Underlying libraries +AC_CHECK_LIB(c, main) +AC_CHECK_LIB(m, main) +AC_CHECK_LIB(pthread, main) +AC_CHECK_LIB(rt, main) +AC_CHECK_LIB(stdc++, main) +AC_CHECK_LIB(regex, main) +if test "$enable_zlib" != "no" +then + AC_CHECK_LIB(z, main) +fi +if test "$enable_lzo" = "yes" +then + AC_CHECK_LIB(lzo2, main) +fi +if test "$enable_lzma" = "yes" +then + AC_CHECK_LIB(lzma, main) +fi +AC_CHECK_LIB(kyotocabinet, main, AC_MSG_WARN([old version of Kyoto Cabinet was detected])) +MYLDLIBPATH="$LD_LIBRARY_PATH" + +# Necessary headers +AC_CHECK_HEADER(stdlib.h, true, AC_MSG_ERROR([stdlib.h is required])) +AC_CHECK_HEADER(stdint.h, true, AC_MSG_ERROR([stdint.h is required])) +AC_CHECK_HEADER(unistd.h, true, AC_MSG_ERROR([unistd.h is required])) +AC_CHECK_HEADER(fcntl.h, true, AC_MSG_ERROR([fcntl.h is required])) +AC_CHECK_HEADER(dirent.h, true, AC_MSG_ERROR([dirent.h is required])) +AC_CHECK_HEADER(pthread.h, true, AC_MSG_ERROR([pthread.h is required])) +AC_CHECK_HEADER(regex.h, true, AC_MSG_ERROR([regex.h is required])) +if test "$enable_zlib" != "no" +then + AC_CHECK_HEADER(zlib.h, true, AC_MSG_ERROR([zlib.h is required])) +fi +if test "$enable_lzo" = "yes" +then + AC_CHECK_HEADER(lzo/lzo1x.h, true, AC_MSG_ERROR([lzo/lzo1x.h is required])) +fi +if test "$enable_lzma" = "yes" +then + AC_CHECK_HEADER(lzma.h, true, AC_MSG_ERROR([lzma.h is required])) +fi + +# Static linking +if test "$is_static" = "yes" +then + MYCMDLDFLAGS="$MYCMDLDFLAGS -static" + MYCMDLIBS="$MYCMDLIBS $LIBS" +fi + +# As-needed linking +if uname | grep Linux >config.tmp +then + MYLDFLAGS="$MYLDFLAGS -Wl,-rpath-link,.:/usr/local/lib:$MYLDLIBPATH" + MYLDFLAGS="$MYLDFLAGS -Wl,--as-needed" +else + MYCMDLIBS="$MYCMDLIBS $LIBS" +fi + +# Shared libraries +if test "$enable_shared" != "no" && test "$enable_profile" != "yes" +then + if uname | grep Darwin >config.tmp + then + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.$MYLIBVER.$MYLIBREV.0.dylib" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.$MYLIBVER.dylib" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.dylib" + MYLDLIBPATHENV="DYLD_LIBRARY_PATH" + else + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.so.$MYLIBVER.$MYLIBREV.0" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.so.$MYLIBVER" + MYLIBRARYFILES="$MYLIBRARYFILES libkyotocabinet.so" + fi +fi + +# Work around of bugs of some environments +if uname | grep Darwin >config.tmp +then + MYCFLAGS="$MYCFLAGS -Os" + MYCXXFLAGS="$MYCXXFLAGS -Os" +fi + + + +#================================================================ +# Generic Settings +#================================================================ + +# Export variables +AC_SUBST(MYLIBVER) +AC_SUBST(MYLIBREV) +AC_SUBST(MYFORMATVER) +AC_SUBST(MYHEADERFILES) +AC_SUBST(MYLIBRARYFILES) +AC_SUBST(MYLIBOBJFILES) +AC_SUBST(MYCOMMANDFILES) +AC_SUBST(MYMAN1FILES) +AC_SUBST(MYDOCUMENTFILES) +AC_SUBST(MYPCFILES) +AC_SUBST(MYCFLAGS) +AC_SUBST(MYCXXFLAGS) +AC_SUBST(MYCPPFLAGS) +AC_SUBST(MYLDFLAGS) +AC_SUBST(MYCMDLDFLAGS) +AC_SUBST(MYCMDLIBS) +AC_SUBST(MYLDLIBPATH) +AC_SUBST(MYLDLIBPATHENV) +AC_SUBST(MYPOSTCMD) + +# Targets +AC_OUTPUT(Makefile kyotocabinet.pc) + +# Messages +printf '#================================================================\n' +printf '# Ready to make.\n' +printf '#================================================================\n' + + + +# END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccachedb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kccachedb.cc new file mode 100644 index 0000000000..23d81f1c29 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccachedb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Cache hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kccachedb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccachedb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kccachedb.h new file mode 100644 index 0000000000..26938ac32f --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccachedb.h @@ -0,0 +1,2067 @@ +/************************************************************************************************* + * Cache hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCCACHEDB_H // duplication check +#define _KCCACHEDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> +#include <kcplantdb.h> + +namespace kyotocabinet { // common namespace + + +/** + * On-memory hash database with LRU deletion. + * @note This class is a concrete class to operate a hash database on memory. This class can be + * inherited but overwriting methods is forbidden. Before every database operation, it is + * necessary to call the CacheDB::open method in order to open a database file and connect the + * database object to it. To avoid data missing or corruption, it is important to close every + * database file by the CacheDB::close method when the database is no longer in use. It is + * forbidden for multible database objects in a process to open the same database at the same + * time. It is forbidden to share a database object with child processes. + */ +class CacheDB : public BasicDB { + friend class PlantDB<CacheDB, BasicDB::TYPEGRASS>; + public: + class Cursor; + private: + struct Record; + struct TranLog; + struct Slot; + class Repeater; + class Setter; + class Remover; + class ScopedVisitor; + /** An alias of list of cursors. */ + typedef std::list<Cursor*> CursorList; + /** An alias of list of transaction logs. */ + typedef std::list<TranLog> TranLogList; + /** The number of slot tables. */ + static const int32_t SLOTNUM = 16; + /** The default bucket number. */ + static const size_t DEFBNUM = 1048583LL; + /** The mininum number of buckets to use mmap. */ + static const size_t ZMAPBNUM = 32768; + /** The maximum size of each key. */ + static const uint32_t KSIZMAX = 0xfffff; + /** The size of the record buffer. */ + static const size_t RECBUFSIZ = 48; + /** The size of the opaque buffer. */ + static const size_t OPAQUESIZ = 16; + /** The threshold of busy loop and sleep for locking. */ + static const uint32_t LOCKBUSYLOOP = 8192; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class CacheDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(CacheDB* db) : db_(db), sidx_(-1), rec_(NULL) { + _assert_(db); + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.push_back(this); + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + if (!db_) return; + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.remove(this); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(db_->omode_ & OWRITER)) { + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + if (sidx_ < 0 || !rec_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + uint32_t rksiz = rec_->ksiz & KSIZMAX; + char* dbuf = (char*)rec_ + sizeof(*rec_); + const char* rvbuf = dbuf + rksiz; + size_t rvsiz = rec_->vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (db_->comp_) { + zbuf = db_->comp_->decompress(rvbuf, rvsiz, &zsiz); + if (zbuf) { + rvbuf = zbuf; + rvsiz = zsiz; + } + } + size_t vsiz; + const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vsiz); + delete[] zbuf; + if (vbuf == Visitor::REMOVE) { + uint64_t hash = db_->hash_record(dbuf, rksiz) / SLOTNUM; + Slot* slot = db_->slots_ + sidx_; + Repeater repeater(Visitor::REMOVE, 0); + db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, false); + } else if (vbuf == Visitor::NOP) { + if (step) step_impl(); + } else { + uint64_t hash = db_->hash_record(dbuf, rksiz) / SLOTNUM; + Slot* slot = db_->slots_ + sidx_; + Repeater repeater(vbuf, vsiz); + db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, false); + if (step) step_impl(); + } + return true; + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + for (int32_t i = 0; i < SLOTNUM; i++) { + Slot* slot = db_->slots_ + i; + if (slot->first) { + sidx_ = i; + rec_ = slot->first; + return true; + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + sidx_ = -1; + rec_ = NULL; + return false; + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (ksiz > KSIZMAX) ksiz = KSIZMAX; + uint64_t hash = db_->hash_record(kbuf, ksiz); + int32_t sidx = hash % SLOTNUM; + hash /= SLOTNUM; + Slot* slot = db_->slots_ + sidx; + size_t bidx = hash % slot->bnum; + Record* rec = slot->buckets[bidx]; + Record** entp = slot->buckets + bidx; + uint32_t fhash = db_->fold_hash(hash) & ~KSIZMAX; + while (rec) { + uint32_t rhash = rec->ksiz & ~KSIZMAX; + uint32_t rksiz = rec->ksiz & KSIZMAX; + if (fhash > rhash) { + entp = &rec->left; + rec = rec->left; + } else if (fhash < rhash) { + entp = &rec->right; + rec = rec->right; + } else { + char* dbuf = (char*)rec + sizeof(*rec); + int32_t kcmp = db_->compare_keys(kbuf, ksiz, dbuf, rksiz); + if (kcmp < 0) { + entp = &rec->left; + rec = rec->left; + } else if (kcmp > 0) { + entp = &rec->right; + rec = rec->right; + } else { + sidx_ = sidx; + rec_ = rec; + return true; + } + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + sidx_ = -1; + rec_ = NULL; + return false; + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + return jump(key.c_str(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const std::string& key) { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (sidx_ < 0 || !rec_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + bool err = false; + if (!step_impl()) err = true; + return !err; + } + /** + * Step the cursor to the previous record. + * @note This is a dummy implementation for compatibility. + */ + bool step_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Get the database object. + * @return the database object. + */ + CacheDB* db() { + _assert_(true); + return db_; + } + private: + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step_impl() { + _assert_(true); + rec_ = rec_->next; + if (!rec_) { + for (int32_t i = sidx_ + 1; i < SLOTNUM; i++) { + Slot* slot = db_->slots_ + i; + if (slot->first) { + sidx_ = i; + rec_ = slot->first; + return true; + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + sidx_ = -1; + rec_ = NULL; + return false; + } + return true; + } + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + CacheDB* db_; + /** The index of the current slot. */ + int32_t sidx_; + /** The current record. */ + Record* rec_; + }; + /** + * Tuning options. + */ + enum Option { + TSMALL = 1 << 0, ///< dummy for compatibility + TLINEAR = 1 << 1, ///< dummy for compatibility + TCOMPRESS = 1 << 2 ///< compress each record + }; + /** + * Status flags. + */ + enum Flag { + FOPEN = 1 << 0, ///< dummy for compatibility + FFATAL = 1 << 1 ///< dummy for compatibility + }; + /** + * Default constructor. + */ + explicit CacheDB() : + mlock_(), flock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL), + omode_(0), curs_(), path_(""), type_(TYPECACHE), + opts_(0), bnum_(DEFBNUM), capcnt_(-1), capsiz_(-1), + opaque_(), embcomp_(ZLIBRAWCOMP), comp_(NULL), slots_(), rttmode_(true), tran_(false) { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~CacheDB() { + _assert_(true); + if (omode_ != 0) close(); + if (!curs_.empty()) { + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->db_ = NULL; + ++cit; + } + } + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + if (ksiz > KSIZMAX) ksiz = KSIZMAX; + uint64_t hash = hash_record(kbuf, ksiz); + int32_t sidx = hash % SLOTNUM; + hash /= SLOTNUM; + Slot* slot = slots_ + sidx; + slot->lock.lock(); + accept_impl(slot, hash, kbuf, ksiz, visitor, comp_, rttmode_); + slot->lock.unlock(); + return true; + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + size_t knum = keys.size(); + if (knum < 1) return true; + struct RecordKey { + const char* kbuf; + size_t ksiz; + uint64_t hash; + int32_t sidx; + }; + RecordKey* rkeys = new RecordKey[knum]; + std::set<int32_t> sidxs; + for (size_t i = 0; i < knum; i++) { + const std::string& key = keys[i]; + RecordKey* rkey = rkeys + i; + rkey->kbuf = key.data(); + rkey->ksiz = key.size(); + if (rkey->ksiz > KSIZMAX) rkey->ksiz = KSIZMAX; + rkey->hash = hash_record(rkey->kbuf, rkey->ksiz); + rkey->sidx = rkey->hash % SLOTNUM; + sidxs.insert(rkey->sidx); + rkey->hash /= SLOTNUM; + } + std::set<int32_t>::iterator sit = sidxs.begin(); + std::set<int32_t>::iterator sitend = sidxs.end(); + while (sit != sitend) { + Slot* slot = slots_ + *sit; + slot->lock.lock(); + ++sit; + } + for (size_t i = 0; i < knum; i++) { + RecordKey* rkey = rkeys + i; + Slot* slot = slots_ + rkey->sidx; + accept_impl(slot, rkey->hash, rkey->kbuf, rkey->ksiz, visitor, comp_, rttmode_); + } + sit = sidxs.begin(); + sitend = sidxs.end(); + while (sit != sitend) { + Slot* slot = slots_ + *sit; + slot->lock.unlock(); + ++sit; + } + delete[] rkeys; + return true; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + int64_t allcnt = count_impl(); + if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + int64_t curcnt = 0; + for (int32_t i = 0; i < SLOTNUM; i++) { + Slot* slot = slots_ + i; + Record* rec = slot->first; + while (rec) { + Record* next = rec->next; + uint32_t rksiz = rec->ksiz & KSIZMAX; + char* dbuf = (char*)rec + sizeof(*rec); + const char* rvbuf = dbuf + rksiz; + size_t rvsiz = rec->vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp_) { + zbuf = comp_->decompress(rvbuf, rvsiz, &zsiz); + if (zbuf) { + rvbuf = zbuf; + rvsiz = zsiz; + } + } + size_t vsiz; + const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vsiz); + delete[] zbuf; + if (vbuf == Visitor::REMOVE) { + uint64_t hash = hash_record(dbuf, rksiz) / SLOTNUM; + Repeater repeater(Visitor::REMOVE, 0); + accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, false); + } else if (vbuf != Visitor::NOP) { + uint64_t hash = hash_record(dbuf, rksiz) / SLOTNUM; + Repeater repeater(vbuf, vsiz); + accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, false); + } + rec = next; + curcnt++; + if (checker && !checker->check("iterate", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + } + } + if (checker && !checker->check("iterate", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + trigger_meta(MetaTrigger::ITERATE, "iterate"); + return true; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (thnum < 1) thnum = 1; + thnum = std::pow(2.0, (int32_t)(std::log(thnum * std::sqrt(2.0)) / std::log(2.0))); + if (thnum > (size_t)SLOTNUM) thnum = SLOTNUM; + ScopedVisitor svis(visitor); + int64_t allcnt = count_impl(); + if (checker && !checker->check("scan_parallel", "beginning", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + class ThreadImpl : public Thread { + public: + explicit ThreadImpl() : + db_(NULL), visitor_(NULL), checker_(NULL), allcnt_(0), slots_(), error_() {} + void init(CacheDB* db, Visitor* visitor, ProgressChecker* checker, int64_t allcnt) { + db_ = db; + visitor_ = visitor; + checker_ = checker; + allcnt_ = allcnt; + } + void add_slot(Slot* slot) { + slots_.push_back(slot); + } + const Error& error() { + return error_; + } + private: + void run() { + CacheDB* db = db_; + Visitor* visitor = visitor_; + ProgressChecker* checker = checker_; + int64_t allcnt = allcnt_; + Compressor* comp = db->comp_; + std::vector<Slot*>::iterator sit = slots_.begin(); + std::vector<Slot*>::iterator sitend = slots_.end(); + while (sit != sitend) { + Slot* slot = *sit; + Record* rec = slot->first; + while (rec) { + Record* next = rec->next; + uint32_t rksiz = rec->ksiz & KSIZMAX; + char* dbuf = (char*)rec + sizeof(*rec); + const char* rvbuf = dbuf + rksiz; + size_t rvsiz = rec->vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp) { + zbuf = comp->decompress(rvbuf, rvsiz, &zsiz); + if (zbuf) { + rvbuf = zbuf; + rvsiz = zsiz; + } + } + size_t vsiz; + visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vsiz); + delete[] zbuf; + rec = next; + if (checker && !checker->check("scan_parallel", "processing", -1, allcnt)) { + db->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + error_ = db->error(); + break; + } + } + ++sit; + } + } + CacheDB* db_; + Visitor* visitor_; + ProgressChecker* checker_; + int64_t allcnt_; + std::vector<Slot*> slots_; + Error error_; + }; + bool err = false; + bool orttmode = rttmode_; + rttmode_ = false; + ThreadImpl* threads = new ThreadImpl[thnum]; + for (int32_t i = 0; i < SLOTNUM; i++) { + ThreadImpl* thread = threads + (i % thnum); + thread->add_slot(slots_ + i); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->init(this, visitor, checker, allcnt); + thread->start(); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->join(); + if (thread->error() != Error::SUCCESS) { + *error_ = thread->error(); + err = true; + } + } + delete[] threads; + rttmode_ = orttmode; + if (err) return false; + if (checker && !checker->check("scan_parallel", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + trigger_meta(MetaTrigger::ITERATE, "scan_parallel"); + return true; + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + return error_; + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + error_->set(code, message); + if (logger_) { + Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ? + Logger::ERROR : Logger::INFO; + if (kind & logkinds_) + report(file, line, func, kind, "%d: %s: %s", code, Error::codename(code), message); + } + } + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. CacheDB::OWRITER as a writer, CacheDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: CacheDB::OCREATE, + * which means it creates a new database if the file does not exist, CacheDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, CacheDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, CacheDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: CacheDB::ONOLOCK, which means it opens the database file without file locking, + * CacheDB::OTRYLOCK, which means locking is performed without blocking, CacheDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the CacheDB::close method when it is no + * longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", path.c_str()); + omode_ = mode; + path_.append(path); + size_t bnum = nearbyprime(bnum_ / SLOTNUM); + size_t capcnt = capcnt_ > 0 ? capcnt_ / SLOTNUM + 1 : (1ULL << (sizeof(capcnt) * 8 - 1)); + size_t capsiz = capsiz_ > 0 ? capsiz_ / SLOTNUM + 1 : (1ULL << (sizeof(capsiz) * 8 - 1)); + if (capsiz > sizeof(*this) / SLOTNUM) capsiz -= sizeof(*this) / SLOTNUM; + if (capsiz > bnum * sizeof(Record*)) capsiz -= bnum * sizeof(Record*); + for (int32_t i = 0; i < SLOTNUM; i++) { + initialize_slot(slots_ + i, bnum, capcnt, capsiz); + } + comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL; + std::memset(opaque_, 0, sizeof(opaque_)); + trigger_meta(MetaTrigger::OPEN, "open"); + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", path_.c_str()); + tran_ = false; + for (int32_t i = SLOTNUM - 1; i >= 0; i--) { + destroy_slot(slots_ + i); + } + path_.clear(); + omode_ = 0; + trigger_meta(MetaTrigger::CLOSE, "close"); + return true; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bool err = false; + if ((omode_ & OWRITER) && checker && + !checker->check("synchronize", "nothing to be synchronized", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (proc) { + if (checker && !checker->check("synchronize", "running the post processor", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!proc->process(path_, count_impl(), size_impl())) { + set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed"); + err = true; + } + } + trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize"); + return !err; + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, writable); + bool err = false; + if (proc && !proc->process(path_, count_impl(), size_impl())) { + set_error(_KCCODELINE_, Error::LOGIC, "processing failed"); + err = true; + } + trigger_meta(MetaTrigger::OCCUPY, "occupy"); + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + uint32_t wcnt = 0; + while (true) { + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!tran_) break; + mlock_.unlock(); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); + mlock_.unlock(); + return true; + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (tran_) { + set_error(_KCCODELINE_, Error::LOGIC, "competition avoided"); + mlock_.unlock(); + return false; + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try"); + mlock_.unlock(); + return true; + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!tran_) { + set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); + return false; + } + if (!commit) disable_cursors(); + for (int32_t i = 0; i < SLOTNUM; i++) { + if (!commit) apply_slot_trlogs(slots_ + i); + slots_[i].trlogs.clear(); + adjust_slot_capacity(slots_ + i); + } + tran_ = false; + trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction"); + return true; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + disable_cursors(); + for (int32_t i = 0; i < SLOTNUM; i++) { + Slot* slot = slots_ + i; + clear_slot(slot); + } + std::memset(opaque_, 0, sizeof(opaque_)); + trigger_meta(MetaTrigger::CLEAR, "clear"); + return true; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return count_impl(); + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return size_impl(); + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return ""; + } + return path_; + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + (*strmap)["type"] = strprintf("%u", (unsigned)TYPECACHE); + (*strmap)["realtype"] = strprintf("%u", (unsigned)type_); + (*strmap)["path"] = path_; + (*strmap)["libver"] = strprintf("%u", LIBVER); + (*strmap)["librev"] = strprintf("%u", LIBREV); + (*strmap)["fmtver"] = strprintf("%u", FMTVER); + (*strmap)["chksum"] = strprintf("%u", 0xff); + (*strmap)["opts"] = strprintf("%u", opts_); + (*strmap)["bnum"] = strprintf("%lld", (long long)bnum_); + (*strmap)["capcnt"] = strprintf("%lld", (long long)capcnt_); + (*strmap)["capsiz"] = strprintf("%lld", (long long)capsiz_); + (*strmap)["recovered"] = strprintf("%d", false); + (*strmap)["reorganized"] = strprintf("%d", false); + if (strmap->count("opaque") > 0) + (*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_)); + if (strmap->count("bnum_used") > 0) { + int64_t cnt = 0; + for (int32_t i = 0; i < SLOTNUM; i++) { + Slot* slot = slots_ + i; + Record** buckets = slot->buckets; + size_t bnum = slot->bnum; + for (size_t j = 0; j < bnum; j++) { + if (buckets[j]) cnt++; + } + } + (*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt); + } + (*strmap)["count"] = strprintf("%lld", (long long)count_impl()); + (*strmap)["size"] = strprintf("%lld", (long long)size_impl()); + return true; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + ScopedRWLock lock(&mlock_, false); + if (!logger_) return; + logger_->log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + logger_ = logger; + logkinds_ = kinds; + return true; + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + /** + * Set the optional features. + * @param opts the optional features by bitwise-or: DirDB::TCOMPRESS to compress each record. + * @return true on success, or false on failure. + */ + bool tune_options(int8_t opts) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + opts_ = opts; + return true; + } + /** + * Set the number of buckets of the hash table. + * @param bnum the number of buckets of the hash table. + * @return true on success, or false on failure. + */ + bool tune_buckets(int64_t bnum) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + bnum_ = bnum >= 0 ? bnum : DEFBNUM; + return true; + } + /** + * Set the data compressor. + * @param comp the data compressor object. + * @return true on success, or false on failure. + */ + bool tune_compressor(Compressor* comp) { + _assert_(comp); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + embcomp_ = comp; + return true; + } + /** + * Set the capacity by record number. + * @param count the maximum number of records. + * @return true on success, or false on failure. + */ + bool cap_count(int64_t count) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + capcnt_ = count; + return true; + } + /** + * Set the capacity by memory usage. + * @param size the maximum size of memory usage. + * @return true on success, or false on failure. + */ + bool cap_size(int64_t size) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + capsiz_ = size; + return true; + } + /** + * Switch the mode of LRU rotation. + * @param rttmode true to enable LRU rotation, false to disable LRU rotation. + * @return true on success, or false on failure. + * @note This function can be called while the database is opened. + */ + bool switch_rotation(bool rttmode) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + rttmode_ = rttmode; + return true; + } + /** + * Get the opaque data. + * @return the pointer to the opaque data region, whose size is 16 bytes. + */ + char* opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return opaque_; + } + /** + * Synchronize the opaque data. + * @return true on success, or false on failure. + */ + bool synchronize_opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + return true; + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, ...) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + va_list ap; + va_start(ap, format); + vstrprintf(&message, format, ap); + va_end(ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report a message for debugging with variable number of arguments. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ap used according to the format string. + */ + void report_valist(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, va_list ap) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + vstrprintf(&message, format, ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report the content of a binary buffer for debugging. + * @param file the file name of the epicenter. + * @param line the line number of the epicenter. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param name the name of the information. + * @param buf the binary buffer. + * @param size the size of the binary buffer + */ + void report_binary(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* name, const char* buf, size_t size) { + _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ); + if (!logger_) return; + char* hex = hexencode(buf, size); + report(file, line, func, kind, "%s=%s", name, hex); + delete[] hex; + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN for beginning + * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaTrigger::ABORTTRAN + * for aborting transaction, and MetaTrigger::MISC for miscellaneous operations. + * @param message the supplement message. + */ + void trigger_meta(MetaTrigger::Kind kind, const char* message) { + _assert_(message); + if (mtrigger_) mtrigger_->trigger(kind, message); + } + /** + * Set the database type. + * @param type the database type. + * @return true on success, or false on failure. + */ + bool tune_type(int8_t type) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + type_ = type; + return true; + } + /** + * Get the library version. + * @return the library version, or 0 on failure. + */ + uint8_t libver() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return LIBVER; + } + /** + * Get the library revision. + * @return the library revision, or 0 on failure. + */ + uint8_t librev() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return LIBREV; + } + /** + * Get the format version. + * @return the format version, or 0 on failure. + */ + uint8_t fmtver() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return FMTVER; + } + /** + * Get the module checksum. + * @return the module checksum, or 0 on failure. + */ + uint8_t chksum() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return 0xff; + } + /** + * Get the database type. + * @return the database type, or 0 on failure. + */ + uint8_t type() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return type_; + } + /** + * Get the options. + * @return the options, or 0 on failure. + */ + uint8_t opts() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return opts_; + } + /** + * Get the data compressor. + * @return the data compressor, or NULL on failure. + */ + Compressor* comp() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return comp_; + } + /** + * Check whether the database was recovered or not. + * @return true if recovered, or false if not. + */ + bool recovered() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return false; + } + /** + * Check whether the database was reorganized or not. + * @return true if reorganized, or false if not. + */ + bool reorganized() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return false; + } + private: + /** + * Set the power of the alignment of record size. + * @note This is a dummy implementation for compatibility. + */ + bool tune_alignment(int8_t apow) { + return true; + } + /** + * Set the power of the capacity of the free block pool. + * @note This is a dummy implementation for compatibility. + */ + bool tune_fbp(int8_t fpow) { + return true; + } + /** + * Set the size of the internal memory-mapped region. + * @note This is a dummy implementation for compatibility. + */ + bool tune_map(int64_t msiz) { + return true; + } + /** + * Set the unit step number of auto defragmentation. + * @note This is a dummy implementation for compatibility. + */ + bool tune_defrag(int64_t dfunit) { + return true; + } + /** + * Perform defragmentation of the file. + * @note This is a dummy implementation for compatibility. + */ + bool defrag(int64_t step = 0) { + return true; + } + /** + * Get the status flags. + * @note This is a dummy implementation for compatibility. + */ + uint8_t flags() { + return 0; + } + /** + * Get the alignment power. + * @note This is a dummy implementation for compatibility. + */ + uint8_t apow() { + return 0; + } + /** + * Get the free block pool power. + * @note This is a dummy implementation for compatibility. + */ + uint8_t fpow() { + return 0; + } + /** + * Get the bucket number. + * @note This is a dummy implementation for compatibility. + */ + int64_t bnum() { + return 1; + } + /** + * Get the size of the internal memory-mapped region. + * @note This is a dummy implementation for compatibility. + */ + int64_t msiz() { + return 0; + } + /** + * Get the unit step number of auto defragmentation. + * @note This is a dummy implementation for compatibility. + */ + int64_t dfunit() { + return 0; + } + private: + /** + * Record data. + */ + struct Record { + uint32_t ksiz; ///< size of the key + uint32_t vsiz; ///< size of the value + Record* left; ///< left child record + Record* right; ///< right child record + Record* prev; ///< privious record + Record* next; ///< next record + }; + /** + * Transaction log. + */ + struct TranLog { + bool full; ///< flag whether full + std::string key; ///< old key + std::string value; ///< old value + /** constructor for a full record */ + explicit TranLog(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) : + full(true), key(kbuf, ksiz), value(vbuf, vsiz) { + _assert_(true); + } + /** constructor for an empty record */ + explicit TranLog(const char* kbuf, size_t ksiz) : full(false), key(kbuf, ksiz) { + _assert_(true); + } + }; + /** + * Slot table. + */ + struct Slot { + Mutex lock; ///< lock + Record** buckets; ///< bucket array + size_t bnum; ///< number of buckets + size_t capcnt; ///< cap of record number + size_t capsiz; ///< cap of memory usage + Record* first; ///< first record + Record* last; ///< last record + size_t count; ///< number of records + size_t size; ///< total size of records + TranLogList trlogs; ///< transaction logs + size_t trsize; ///< size before transaction + }; + /** + * Repeating visitor. + */ + class Repeater : public Visitor { + public: + /** constructor */ + explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsiz) {} + private: + /** process a full record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; ///< region of the value + size_t vsiz_; ///< size of the value + }; + /** + * Setting visitor. + */ + class Setter : public Visitor { + public: + /** constructor */ + explicit Setter(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsiz) {} + private: + /** process a full record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + *sp = vsiz_; + return vbuf_; + } + /** process an empty record */ + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && sp); + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; ///< region of the value + size_t vsiz_; ///< size of the value + }; + /** + * Removing visitor. + */ + class Remover : public Visitor { + private: + /** visit a record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + return REMOVE; + } + }; + /** + * Scoped visitor. + */ + class ScopedVisitor { + public: + /** constructor */ + explicit ScopedVisitor(Visitor* visitor) : visitor_(visitor) { + _assert_(visitor); + visitor_->visit_before(); + } + /** destructor */ + ~ScopedVisitor() { + _assert_(true); + visitor_->visit_after(); + } + private: + Visitor* visitor_; ///< visitor + }; + /** + * Accept a visitor to a record. + * @param slot the slot of the record. + * @param hash the hash value of the key. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param comp the data compressor. + * @param rtt whether to move the record to the last. + */ + void accept_impl(Slot* slot, uint64_t hash, const char* kbuf, size_t ksiz, Visitor* visitor, + Compressor* comp, bool rtt) { + _assert_(slot && kbuf && ksiz <= MEMMAXSIZ && visitor); + size_t bidx = hash % slot->bnum; + Record* rec = slot->buckets[bidx]; + Record** entp = slot->buckets + bidx; + uint32_t fhash = fold_hash(hash) & ~KSIZMAX; + while (rec) { + uint32_t rhash = rec->ksiz & ~KSIZMAX; + uint32_t rksiz = rec->ksiz & KSIZMAX; + if (fhash > rhash) { + entp = &rec->left; + rec = rec->left; + } else if (fhash < rhash) { + entp = &rec->right; + rec = rec->right; + } else { + char* dbuf = (char*)rec + sizeof(*rec); + int32_t kcmp = compare_keys(kbuf, ksiz, dbuf, rksiz); + if (kcmp < 0) { + entp = &rec->left; + rec = rec->left; + } else if (kcmp > 0) { + entp = &rec->right; + rec = rec->right; + } else { + const char* rvbuf = dbuf + rksiz; + size_t rvsiz = rec->vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp) { + zbuf = comp->decompress(rvbuf, rvsiz, &zsiz); + if (zbuf) { + rvbuf = zbuf; + rvsiz = zsiz; + } + } + size_t vsiz; + const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vsiz); + delete[] zbuf; + if (vbuf == Visitor::REMOVE) { + if (tran_) { + TranLog log(kbuf, ksiz, dbuf + rksiz, rec->vsiz); + slot->trlogs.push_back(log); + } + if (!curs_.empty()) escape_cursors(rec); + if (rec == slot->first) slot->first = rec->next; + if (rec == slot->last) slot->last = rec->prev; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + if (rec->left && !rec->right) { + *entp = rec->left; + } else if (!rec->left && rec->right) { + *entp = rec->right; + } else if (!rec->left) { + *entp = NULL; + } else { + Record* pivot = rec->left; + if (pivot->right) { + Record** pentp = &pivot->right; + pivot = pivot->right; + while (pivot->right) { + pentp = &pivot->right; + pivot = pivot->right; + } + *entp = pivot; + *pentp = pivot->left; + pivot->left = rec->left; + pivot->right = rec->right; + } else { + *entp = pivot; + pivot->right = rec->right; + } + } + slot->count--; + slot->size -= sizeof(Record) + rksiz + rec->vsiz; + xfree(rec); + } else { + bool adj = false; + if (vbuf != Visitor::NOP) { + char* zbuf = NULL; + size_t zsiz = 0; + if (comp) { + zbuf = comp->compress(vbuf, vsiz, &zsiz); + if (zbuf) { + vbuf = zbuf; + vsiz = zsiz; + } + } + if (tran_) { + TranLog log(kbuf, ksiz, dbuf + rksiz, rec->vsiz); + slot->trlogs.push_back(log); + } else { + adj = vsiz > rec->vsiz; + } + slot->size -= rec->vsiz; + slot->size += vsiz; + if (vsiz > rec->vsiz) { + Record* old = rec; + rec = (Record*)xrealloc(rec, sizeof(*rec) + ksiz + vsiz); + if (rec != old) { + if (!curs_.empty()) adjust_cursors(old, rec); + if (slot->first == old) slot->first = rec; + if (slot->last == old) slot->last = rec; + *entp = rec; + if (rec->prev) rec->prev->next = rec; + if (rec->next) rec->next->prev = rec; + dbuf = (char*)rec + sizeof(*rec); + } + } + std::memcpy(dbuf + ksiz, vbuf, vsiz); + rec->vsiz = vsiz; + delete[] zbuf; + } + if (rtt && slot->last != rec) { + if (!curs_.empty()) escape_cursors(rec); + if (slot->first == rec) slot->first = rec->next; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + rec->prev = slot->last; + rec->next = NULL; + slot->last->next = rec; + slot->last = rec; + } + if (adj) adjust_slot_capacity(slot); + } + return; + } + } + } + size_t vsiz; + const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + char* zbuf = NULL; + size_t zsiz = 0; + if (comp) { + zbuf = comp->compress(vbuf, vsiz, &zsiz); + if (zbuf) { + vbuf = zbuf; + vsiz = zsiz; + } + } + if (tran_) { + TranLog log(kbuf, ksiz); + slot->trlogs.push_back(log); + } + slot->size += sizeof(Record) + ksiz + vsiz; + rec = (Record*)xmalloc(sizeof(*rec) + ksiz + vsiz); + char* dbuf = (char*)rec + sizeof(*rec); + std::memcpy(dbuf, kbuf, ksiz); + rec->ksiz = ksiz | fhash; + std::memcpy(dbuf + ksiz, vbuf, vsiz); + rec->vsiz = vsiz; + rec->left = NULL; + rec->right = NULL; + rec->prev = slot->last; + rec->next = NULL; + *entp = rec; + if (!slot->first) slot->first = rec; + if (slot->last) slot->last->next = rec; + slot->last = rec; + slot->count++; + if (!tran_) adjust_slot_capacity(slot); + delete[] zbuf; + } + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count_impl() { + _assert_(true); + int64_t sum = 0; + for (int32_t i = 0; i < SLOTNUM; i++) { + Slot* slot = slots_ + i; + ScopedMutex lock(&slot->lock); + sum += slot->count; + } + return sum; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes. + */ + int64_t size_impl() { + _assert_(true); + int64_t sum = sizeof(*this); + for (int32_t i = 0; i < SLOTNUM; i++) { + Slot* slot = slots_ + i; + ScopedMutex lock(&slot->lock); + sum += slot->bnum * sizeof(Record*); + sum += slot->size; + } + return sum; + } + /** + * Initialize a slot table. + * @param slot the slot table. + * @param bnum the number of buckets. + * @param capcnt the capacity of record number. + * @param capsiz the capacity of memory usage. + */ + void initialize_slot(Slot* slot, size_t bnum, size_t capcnt, size_t capsiz) { + _assert_(slot); + Record** buckets; + if (bnum >= ZMAPBNUM) { + buckets = (Record**)mapalloc(sizeof(*buckets) * bnum); + } else { + buckets = new Record*[bnum]; + for (size_t i = 0; i < bnum; i++) { + buckets[i] = NULL; + } + } + slot->buckets = buckets; + slot->bnum = bnum; + slot->capcnt = capcnt; + slot->capsiz = capsiz; + slot->first = NULL; + slot->last = NULL; + slot->count = 0; + slot->size = 0; + } + /** + * Destroy a slot table. + * @param slot the slot table. + */ + void destroy_slot(Slot* slot) { + _assert_(slot); + slot->trlogs.clear(); + Record* rec = slot->last; + while (rec) { + Record* prev = rec->prev; + xfree(rec); + rec = prev; + } + if (slot->bnum >= ZMAPBNUM) { + mapfree(slot->buckets); + } else { + delete[] slot->buckets; + } + } + /** + * Clear a slot table. + * @param slot the slot table. + */ + void clear_slot(Slot* slot) { + _assert_(slot); + Record* rec = slot->last; + while (rec) { + if (tran_) { + uint32_t rksiz = rec->ksiz & KSIZMAX; + char* dbuf = (char*)rec + sizeof(*rec); + TranLog log(dbuf, rksiz, dbuf + rksiz, rec->vsiz); + slot->trlogs.push_back(log); + } + Record* prev = rec->prev; + xfree(rec); + rec = prev; + } + Record** buckets = slot->buckets; + size_t bnum = slot->bnum; + for (size_t i = 0; i < bnum; i++) { + buckets[i] = NULL; + } + slot->first = NULL; + slot->last = NULL; + slot->count = 0; + slot->size = 0; + } + /** + * Apply transaction logs of a slot table. + * @param slot the slot table. + */ + void apply_slot_trlogs(Slot* slot) { + _assert_(slot); + const TranLogList& logs = slot->trlogs; + TranLogList::const_iterator it = logs.end(); + TranLogList::const_iterator itbeg = logs.begin(); + while (it != itbeg) { + --it; + const char* kbuf = it->key.c_str(); + size_t ksiz = it->key.size(); + const char* vbuf = it->value.c_str(); + size_t vsiz = it->value.size(); + uint64_t hash = hash_record(kbuf, ksiz) / SLOTNUM; + if (it->full) { + Setter setter(vbuf, vsiz); + accept_impl(slot, hash, kbuf, ksiz, &setter, NULL, false); + } else { + Remover remover; + accept_impl(slot, hash, kbuf, ksiz, &remover, NULL, false); + } + } + } + /** + * Addjust a slot table to the capacity. + * @param slot the slot table. + */ + void adjust_slot_capacity(Slot* slot) { + _assert_(slot); + if ((slot->count > slot->capcnt || slot->size > slot->capsiz) && slot->first) { + Record* rec = slot->first; + uint32_t rksiz = rec->ksiz & KSIZMAX; + char* dbuf = (char*)rec + sizeof(*rec); + char stack[RECBUFSIZ]; + char* kbuf = rksiz > sizeof(stack) ? new char[rksiz] : stack; + std::memcpy(kbuf, dbuf, rksiz); + uint64_t hash = hash_record(kbuf, rksiz) / SLOTNUM; + Remover remover; + accept_impl(slot, hash, dbuf, rksiz, &remover, NULL, false); + if (kbuf != stack) delete[] kbuf; + } + } + /** + * Get the hash value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the hash value. + */ + uint64_t hash_record(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + return hashmurmur(kbuf, ksiz); + } + /** + * Fold a hash value into a small number. + * @param hash the hash number. + * @return the result number. + */ + uint32_t fold_hash(uint64_t hash) { + _assert_(true); + return ((hash & 0xffffffff00000000ULL) >> 32) ^ ((hash & 0x0000ffffffff0000ULL) >> 16) ^ + ((hash & 0x000000000000ffffULL) << 16) ^ ((hash & 0x00000000ffff0000ULL) >> 0); + } + /** + * Compare two keys in lexical order. + * @param abuf one key. + * @param asiz the size of the one key. + * @param bbuf the other key. + * @param bsiz the size of the other key. + * @return positive if the former is big, or negative if the latter is big, or 0 if both are + * equivalent. + */ + int32_t compare_keys(const char* abuf, size_t asiz, const char* bbuf, size_t bsiz) { + _assert_(abuf && asiz <= MEMMAXSIZ && bbuf && bsiz <= MEMMAXSIZ); + if (asiz != bsiz) return (int32_t)asiz - (int32_t)bsiz; + return std::memcmp(abuf, bbuf, asiz); + } + /** + * Escape cursors on a shifted or removed records. + * @param rec the record. + */ + void escape_cursors(Record* rec) { + _assert_(rec); + ScopedMutex lock(&flock_); + if (curs_.empty()) return; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->rec_ == rec) cur->step_impl(); + ++cit; + } + } + /** + * Adjust cursors on re-allocated records. + * @param orec the old address. + * @param nrec the new address. + */ + void adjust_cursors(Record* orec, Record* nrec) { + _assert_(orec && nrec); + ScopedMutex lock(&flock_); + if (curs_.empty()) return; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->rec_ == orec) cur->rec_ = nrec; + ++cit; + } + } + /** + * Disable all cursors. + */ + void disable_cursors() { + _assert_(true); + ScopedMutex lock(&flock_); + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->sidx_ = -1; + cur->rec_ = NULL; + ++cit; + } + } + /** Dummy constructor to forbid the use. */ + CacheDB(const CacheDB&); + /** Dummy Operator to forbid the use. */ + CacheDB& operator =(const CacheDB&); + /** The method lock. */ + RWLock mlock_; + /** The file lock. */ + Mutex flock_; + /** The last happened error. */ + TSD<Error> error_; + /** The internal logger. */ + Logger* logger_; + /** The kinds of logged messages. */ + uint32_t logkinds_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The open mode. */ + uint32_t omode_; + /** The cursor objects. */ + CursorList curs_; + /** The path of the database file. */ + std::string path_; + /** The database type. */ + uint8_t type_; + /** The options. */ + uint8_t opts_; + /** The bucket number. */ + int64_t bnum_; + /** The capacity of record number. */ + int64_t capcnt_; + /** The capacity of memory usage. */ + int64_t capsiz_; + /** The opaque data. */ + char opaque_[OPAQUESIZ]; + /** The embedded data compressor. */ + Compressor* embcomp_; + /** The data compressor. */ + Compressor* comp_; + /** The slot tables. */ + Slot slots_[SLOTNUM]; + /** The flag whether in LRU rotation. */ + bool rttmode_; + /** The flag whether in transaction. */ + bool tran_; +}; + + +/** An alias of the cache tree database. */ +typedef PlantDB<CacheDB, BasicDB::TYPEGRASS> GrassDB; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccachetest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kccachetest.cc new file mode 100644 index 0000000000..1ad9217205 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccachetest.cc @@ -0,0 +1,2236 @@ +/************************************************************************************************* + * The test cases of the cache hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kccachedb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t procorder(int64_t rnum, int32_t thnum, bool rnd, bool etc, bool tran, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv); +static int32_t procqueue(int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv); +static int32_t procwicked(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv); +static int32_t proctran(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the cache hash database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-etc] [-tran] [-tc] [-bnum num]" + " [-capcnt num] [-capsiz num] [-lv] rnum\n", g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-tc] [-bnum num]" + " [-capcnt num] [-capsiz num] [-lv] rnum\n", g_progname); + eprintf(" %s wicked [-th num] [-it num] [-tc] [-bnum num]" + " [-capcnt num] [-capsiz num] [-lv] rnum\n", g_progname); + eprintf(" %s tran [-th num] [-it num] [-tc] [-bnum num]" + " [-capcnt num] [-capsiz num] [-lv] rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["bnum_used"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t rtype = kc::atoi(status["realtype"].c_str()); + if (rtype > 0 && rtype != type) + oprintf("real type: %s (%s) (realtype=0x%02X)\n", + kc::BasicDB::typecname(rtype), kc::BasicDB::typestring(rtype), rtype); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::CacheDB::FOPEN) oprintf(" open"); + if (flags & kc::CacheDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + oprintf("\n", flags); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::CacheDB::TSMALL) oprintf(" small"); + if (opts & kc::CacheDB::TLINEAR) oprintf(" linear"); + if (opts & kc::CacheDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t bnumused = kc::atoi(status["bnum_used"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + double load = 0; + if (count > 0 && bnumused > 0) { + load = (double)count / bnumused; + if (!(opts & kc::CacheDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (used=%lld) (load=%.2f)\n", + (long long)bnum, (long long)bnumused, load); + std::string cntstr = unitnumstr(count); + int64_t capcnt = kc::atoi(status["capcnt"].c_str()); + oprintf("count: %lld (%s) (capcnt=%lld)\n", count, cntstr.c_str(), (long long)capcnt); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + int64_t capsiz = kc::atoi(status["capsiz"].c_str()); + oprintf("size: %lld (%s) (capsiz=%lld)\n", size, sizestr.c_str(), (long long)capsiz); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + bool etc = false; + bool tran = false; + int32_t opts = 0; + int64_t bnum = -1; + int64_t capcnt = -1; + int64_t capsiz = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-etc")) { + etc = true; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capcnt")) { + if (++i >= argc) usage(); + capcnt = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capsiz")) { + if (++i >= argc) usage(); + capsiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(rnum, thnum, rnd, etc, tran, opts, bnum, capcnt, capsiz, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int32_t opts = 0; + int64_t bnum = -1; + int64_t capcnt = -1; + int64_t capsiz = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capcnt")) { + if (++i >= argc) usage(); + capcnt = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capsiz")) { + if (++i >= argc) usage(); + capsiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(rnum, thnum, itnum, rnd, opts, bnum, capcnt, capsiz, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t capcnt = -1; + int64_t capsiz = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capcnt")) { + if (++i >= argc) usage(); + capcnt = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capsiz")) { + if (++i >= argc) usage(); + capsiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(rnum, thnum, itnum, opts, bnum, capcnt, capsiz, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t capcnt = -1; + int64_t capsiz = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capcnt")) { + if (++i >= argc) usage(); + capcnt = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-capsiz")) { + if (++i >= argc) usage(); + capsiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(rnum, thnum, itnum, opts, bnum, capcnt, capsiz, lv); + return rv; +} + + +// perform order command +static int32_t procorder(int64_t rnum, int32_t thnum, bool rnd, bool etc, bool tran, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv) { + oprintf("<In-order Test>\n seed=%u rnum=%lld thnum=%d rnd=%d etc=%d tran=%d" + " opts=%d bnum=%lld capcnt=%lld capsiz=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, rnd, etc, tran, + opts, (long long)bnum, (long long)capcnt, (long long)capsiz, lv); + bool err = false; + kc::CacheDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (capcnt > 0) db.cap_count(capcnt); + if (capsiz > 0) db.cap_size(capsiz); + if (!db.open("*", kc::CacheDB::OWRITER | kc::CacheDB::OCREATE | kc::CacheDB::OTRUNCATE)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::CacheDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size) : + rnum_(rnum), rnd_(rnd), size_(size) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + } syncprocessor(rnum, rnd, db.size()); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc && capcnt < 1 && capsiz < 1 && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool etc, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + etc_ = etc; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && !etc_) || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool etc_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, etc, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, etc, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, true); + oprintf("time: %.3f\n", etime - stime); + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv) { + oprintf("<Queue Test>\n seed=%u rnum=%lld thnum=%d itnum=%d rnd=%d" + " opts=%d bnum=%lld capcnt=%lld capsiz=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, rnd, + opts, (long long)bnum, (long long)capcnt, (long long)capsiz, lv); + bool err = false; + kc::CacheDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (capcnt > 0) db.cap_count(capcnt); + if (capsiz > 0) db.cap_size(capsiz); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::CacheDB::OWRITER | kc::CacheDB::OCREATE; + if (itcnt == 1) omode |= kc::CacheDB::OTRUNCATE; + if (!db.open("*", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::CacheDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::CacheDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv) { + oprintf("<Wicked Test>\n seed=%u rnum=%lld thnum=%d itnum=%d" + " opts=%d bnum=%lld capcnt=%lld capsiz=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, + opts, (long long)bnum, (long long)capcnt, (long long)capsiz, lv); + bool err = false; + kc::CacheDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (capcnt > 0) db.cap_count(capcnt); + if (capsiz > 0) db.cap_size(capsiz); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::CacheDB::OWRITER | kc::CacheDB::OCREATE; + if (itcnt == 1) omode |= kc::CacheDB::OTRUNCATE; + if (!db.open("*", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::CacheDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(9)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(5) > 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + if (!db_->switch_rotation(myrand(4) > 0)) { + dberrprint(db_, __LINE__, "DB::switch_rotation"); + err_ = true; + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::CacheDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int64_t capcnt, int64_t capsiz, bool lv) { + oprintf("<Transaction Test>\n seed=%u rnum=%lld thnum=%d itnum=%d" + " opts=%d bnum=%lld capcnt=%lld capsiz=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, + opts, (long long)bnum, (long long)capcnt, (long long)capsiz, lv); + bool err = false; + kc::CacheDB db; + kc::CacheDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (capcnt > 0) db.cap_count(capcnt); + if (capsiz > 0) db.cap_size(capsiz); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::CacheDB::OWRITER | kc::CacheDB::OCREATE; + if (itcnt == 1) omode |= kc::CacheDB::OTRUNCATE; + if (!db.open("*", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + if (!paradb.open("para", omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::CacheDB* db, kc::CacheDB* paradb, int64_t rnum, + int32_t thnum, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::CacheDB* db_; + kc::CacheDB* paradb_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccommon.h b/plugins/Dbx_kyoto/src/kyotocabinet/kccommon.h new file mode 100644 index 0000000000..e6995b1ddc --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccommon.h @@ -0,0 +1,205 @@ +/************************************************************************************************* + * Common symbols for the library + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCCOMMON_H // duplication check +#define _KCCOMMON_H + +#define _CRT_SECURE_NO_WARNINGS + +extern "C" { +#include <stdint.h> +} + +#include <cassert> +#include <cctype> +#include <cerrno> +#include <cfloat> +#include <climits> +#include <clocale> +#include <cmath> +#include <csetjmp> +#include <cstdarg> +#include <cstddef> +#include <cstdio> +#include <cstdlib> +#include <csignal> +#include <cstring> +#include <ctime> +#include <cwchar> + +#include <stdexcept> +#include <exception> +#include <limits> +#include <new> +#include <typeinfo> + +#include <utility> +#include <functional> +#include <memory> +#include <iterator> +#include <algorithm> +#include <locale> + +#include <string> +#include <vector> +#include <list> +#include <queue> +#include <deque> +#include <map> +#include <set> + +#include <ios> +#include <iostream> +#include <streambuf> +#include <fstream> +#include <sstream> + +#if defined(_MSC_VER) +#define snprintf _snprintf +#endif + +#if defined(__CYGWIN__) +inline long double modfl(long double val, long double* iptr) { + double integ; + double fract = std::modf(val, &integ); + *iptr = integ; + return fract; +} +#endif + +namespace std { +using ::modfl; +using ::snprintf; +} + +#if __cplusplus > 199711L || defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(_MSC_VER) + +#include <unordered_map> +#include <unordered_set> + +#else + +#include <tr1/unordered_map> +#include <tr1/unordered_set> + +namespace std { +using tr1::hash; +using tr1::unordered_map; +using tr1::unordered_set; +} + +#endif + +#undef VERSION +#undef LIBVER +#undef LIBREV +#undef OSNAME +#undef BIGEND +#undef CLOCKTICK +#undef PAGESIZ +#undef FEATURES +#undef NUMBUFSIZ +#undef MEMMAXSIZ + +#undef IOBUFSIZ +#undef SUCCESS +#undef NOIMPL +#undef INVALID +#undef NOREPOS +#undef NOPERM +#undef BROKEN +#undef DUPREC +#undef NOREC +#undef LOGIC +#undef SYSTEM +#undef MISC + +#undef DEBUG +#undef INFO +#undef WARN +#undef ERROR +#undef OPEN +#undef CLOSE +#undef CLEAR +#undef ITERATE +#undef SYNCHRONIZE +#undef OCCUPY +#undef BEGINTRAN +#undef COMMITTRAN +#undef ABORTTRAN + +#undef INT8MAX +#undef INT16MAX +#undef INT32MAX +#undef INT64MAX +#undef INT8MIN +#undef INT16MIN +#undef INT32MIN +#undef INT64MIN +#undef UINT8MAX +#undef UINT16MAX +#undef UINT32MAX +#undef UINT64MAX +#undef SIZEMAX +#undef FLTMAX +#undef DBLMAX + +#if defined(_KCUYIELD) +#if defined(_MSC_VER) +#include <windows.h> +#define _yield_() ::Sleep(0) +#else +#include <sched.h> +#define _yield_() ::sched_yield() +#endif +#define _testyield_() \ + do { \ + static uint32_t _KC_seed = 725; \ + _KC_seed = _KC_seed * 123456761 + 211; \ + if (_KC_seed % 0x100 == 0) _yield_(); \ + } while(false) +#define _assert_(KC_a) \ + do { \ + _testyield_(); \ + assert(KC_a); \ + } while(false) +#elif defined(_KCDEBUG) +#define _yield_() +#define _testyield_() +#define _assert_(KC_a) assert(KC_a) +#else +#define _yield_() ///< for debugging +#define _testyield_() ///< for debugging +#define _assert_(KC_a) ///< for debugging +#endif + +#if defined(__GNUC__) +#define __KCFUNC__ __func__ ///< for debugging +#elif defined(_MSC_VER) +#define __KCFUNC__ __FUNCTION__ ///< for debugging +#else +#define __KCFUNC__ "-" ///< for debugging +#endif +#define _KCCODELINE_ __FILE__, __LINE__, __KCFUNC__ ///< for debugging + +/** + * All symbols of Kyoto Cabinet. + */ +namespace kyotocabinet {} + + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccompare.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kccompare.cc new file mode 100644 index 0000000000..65a3822ff9 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccompare.cc @@ -0,0 +1,52 @@ +/************************************************************************************************* + * Comparator functions + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kccompare.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +/** + * Prepared pointer of the comparator in the lexical order. + */ +LexicalComparator lexicalfunc; +LexicalComparator* const LEXICALCOMP = &lexicalfunc; + + +/** + * Prepared pointer of the comparator in the lexical descending order. + */ +LexicalDescendingComparator lexicaldescfunc; +LexicalDescendingComparator* const LEXICALDESCCOMP = &lexicaldescfunc; + + +/** + * Prepared pointer of the comparator in the decimal order. + */ +DecimalComparator decimalfunc; +DecimalComparator* const DECIMALCOMP = &decimalfunc; + + +/** + * Prepared pointer of the comparator in the decimal descending order. + */ +DecimalDescendingComparator decimaldescfunc; +DecimalDescendingComparator* const DECIMALDESCCOMP = &decimaldescfunc; + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccompare.h b/plugins/Dbx_kyoto/src/kyotocabinet/kccompare.h new file mode 100644 index 0000000000..3630527157 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccompare.h @@ -0,0 +1,215 @@ +/************************************************************************************************* + * Comparator functions + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCCOMPARE_H // duplication check +#define _KCCOMPARE_H + +#include <kccommon.h> +#include <kcutil.h> + +namespace kyotocabinet { // common namespace + + +/** + * Interfrace of comparator of record keys. + */ +class Comparator { + public: + /** + * Destructor. + */ + virtual ~Comparator() {} + /** + * Compare two keys. + * @param akbuf the pointer to the region of one key. + * @param aksiz the size of the region of one key. + * @param bkbuf the pointer to the region of the other key. + * @param bksiz the size of the region of the other key. + * @return positive if the former is big, negative if the latter is big, 0 if both are + * equivalent. + */ + virtual int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) = 0; +}; + + +/** + * Comparator in the lexical order. + */ +class LexicalComparator : public Comparator { + public: + explicit LexicalComparator() {} + int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { + _assert_(akbuf && bkbuf); + size_t msiz = aksiz < bksiz ? aksiz : bksiz; + for (size_t i = 0; i < msiz; i++) { + if (((uint8_t*)akbuf)[i] != ((uint8_t*)bkbuf)[i]) + return ((uint8_t*)akbuf)[i] - ((uint8_t*)bkbuf)[i]; + } + return (int32_t)aksiz - (int32_t)bksiz; + } +}; + + +/** + * Comparator in the lexical descending order. + */ +class LexicalDescendingComparator : public Comparator { + public: + explicit LexicalDescendingComparator() : comp_() {} + int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { + return -comp_.compare(akbuf, aksiz, bkbuf, bksiz); + } + private: + LexicalComparator comp_; +}; + + +/** + * Comparator in the decimal order. + */ +class DecimalComparator : public Comparator { + public: + explicit DecimalComparator() {} + int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { + _assert_(akbuf && bkbuf); + const int32_t LDBLCOLMAX = 16; + const unsigned char* arp = (unsigned char*)akbuf; + int32_t alen = aksiz; + while (alen > 0 && (*arp <= ' ' || *arp == 0x7f)) { + arp++; + alen--; + } + int64_t anum = 0; + int32_t asign = 1; + if (alen > 0 && *arp == '-') { + arp++; + alen--; + asign = -1; + } + while (alen > 0) { + int32_t c = *arp; + if (c < '0' || c > '9') break; + anum = anum * 10 + c - '0'; + arp++; + alen--; + } + anum *= asign; + const unsigned char* brp = (unsigned char*)bkbuf; + int32_t blen = bksiz; + while (blen > 0 && (*brp <= ' ' || *brp == 0x7f)) { + brp++; + blen--; + } + int64_t bnum = 0; + int32_t bsign = 1; + if (blen > 0 && *brp == '-') { + brp++; + blen--; + bsign = -1; + } + while (blen > 0) { + int32_t c = *brp; + if (c < '0' || c > '9') break; + bnum = bnum * 10 + c - '0'; + brp++; + blen--; + } + bnum *= bsign; + if (anum < bnum) return -1; + if (anum > bnum) return 1; + if ((alen > 1 && *arp == '.') || (blen > 1 && *brp == '.')) { + long double aflt = 0; + if (alen > 1 && *arp == '.') { + arp++; + alen--; + if (alen > LDBLCOLMAX) alen = LDBLCOLMAX; + long double base = 10; + while (alen > 0) { + if (*arp < '0' || *arp > '9') break; + aflt += (*arp - '0') / base; + arp++; + alen--; + base *= 10; + } + aflt *= asign; + } + long double bflt = 0; + if (blen > 1 && *brp == '.') { + brp++; + blen--; + if (blen > LDBLCOLMAX) blen = LDBLCOLMAX; + long double base = 10; + while (blen > 0) { + if (*brp < '0' || *brp > '9') break; + bflt += (*brp - '0') / base; + brp++; + blen--; + base *= 10; + } + bflt *= bsign; + } + if (aflt < bflt) return -1; + if (aflt > bflt) return 1; + } + LexicalComparator lexcomp; + int32_t rv = lexcomp.compare(akbuf, aksiz, bkbuf, bksiz); + return rv; + } +}; + + +/** + * Comparator in the decimal descending order. + */ +class DecimalDescendingComparator : public Comparator { + public: + explicit DecimalDescendingComparator() : comp_() {} + int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { + return -comp_.compare(akbuf, aksiz, bkbuf, bksiz); + } + private: + DecimalComparator comp_; +}; + + +/** + * Prepared pointer of the comparator in the lexical order. + */ +extern LexicalComparator* const LEXICALCOMP; + + +/** + * Prepared pointer of the comparator in the lexical descending order. + */ +extern LexicalDescendingComparator* const LEXICALDESCCOMP; + + +/** + * Prepared pointer of the comparator in the decimal order. + */ +extern DecimalComparator* const DECIMALCOMP; + + +/** + * Prepared pointer of the comparator in the decimal descending order. + */ +extern DecimalDescendingComparator* const DECIMALDESCCOMP; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccompress.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kccompress.cc new file mode 100644 index 0000000000..be4533141a --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccompress.cc @@ -0,0 +1,403 @@ +/************************************************************************************************* + * Data compressor and decompressor + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kccompress.h" +#include "myconf.h" + +#if _KC_ZLIB +extern "C" { +#include <zlib.h> +} +#endif + +#if _KC_LZO +extern "C" { +#include <lzo/lzo1x.h> +} +#endif + +#if _KC_LZMA +extern "C" { +#include <lzma.h> +} +#endif + +namespace kyotocabinet { // common namespace + + +/** + * Compress a serial data. + */ +char* ZLIB::compress(const void* buf, size_t size, size_t* sp, Mode mode) { +#if _KC_ZLIB + _assert_(buf && size <= MEMMAXSIZ && sp); + z_stream zs; + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + switch (mode) { + default: { + if (deflateInit2(&zs, 6, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY) != Z_OK) return NULL; + break; + } + case DEFLATE: { + if (deflateInit2(&zs, 6, Z_DEFLATED, 15, 9, Z_DEFAULT_STRATEGY) != Z_OK) return NULL; + break; + } + case GZIP: { + if (deflateInit2(&zs, 6, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY) != Z_OK) return NULL; + break; + } + } + const char* rp = (const char*)buf; + size_t zsiz = size + size / 8 + 32; + char* zbuf = new char[zsiz+1]; + char* wp = zbuf; + zs.next_in = (Bytef*)rp; + zs.avail_in = size; + zs.next_out = (Bytef*)wp; + zs.avail_out = zsiz; + if (deflate(&zs, Z_FINISH) != Z_STREAM_END) { + delete[] zbuf; + deflateEnd(&zs); + return NULL; + } + deflateEnd(&zs); + zsiz -= zs.avail_out; + zbuf[zsiz] = '\0'; + if (mode == RAW) zsiz++; + *sp = zsiz; + return zbuf; +#else + _assert_(buf && size <= MEMMAXSIZ && sp); + char* zbuf = new char[size+2]; + char* wp = zbuf; + *(wp++) = 'z'; + *(wp++) = (uint8_t)mode; + std::memcpy(wp, buf, size); + *sp = size + 2; + return zbuf; +#endif +} + + +/** + * Decompress a serial data. + */ +char* ZLIB::decompress(const void* buf, size_t size, size_t* sp, Mode mode) { +#if _KC_ZLIB + _assert_(buf && size <= MEMMAXSIZ && sp); + size_t zsiz = size * 8 + 32; + while (true) { + z_stream zs; + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + switch (mode) { + default: { + if (inflateInit2(&zs, -15) != Z_OK) return NULL; + break; + } + case DEFLATE: { + if (inflateInit2(&zs, 15) != Z_OK) return NULL; + break; + } + case GZIP: { + if (inflateInit2(&zs, 15 + 16) != Z_OK) return NULL; + break; + } + } + char* zbuf = new char[zsiz+1]; + zs.next_in = (Bytef*)buf; + zs.avail_in = size; + zs.next_out = (Bytef*)zbuf; + zs.avail_out = zsiz; + int32_t rv = inflate(&zs, Z_FINISH); + inflateEnd(&zs); + if (rv == Z_STREAM_END) { + zsiz -= zs.avail_out; + zbuf[zsiz] = '\0'; + *sp = zsiz; + return zbuf; + } else if (rv == Z_BUF_ERROR) { + delete[] zbuf; + zsiz *= 2; + } else { + delete[] zbuf; + break; + } + } + return NULL; +#else + _assert_(buf && size <= MEMMAXSIZ && sp); + if (size < 2 || ((char*)buf)[0] != 'z' || ((char*)buf)[1] != (uint8_t)mode) return NULL; + buf = (char*)buf + 2; + size -= 2; + char* zbuf = new char[size+1]; + std::memcpy(zbuf, buf, size); + zbuf[size] = '\0'; + *sp = size; + return zbuf; +#endif +} + + +/** + * Calculate the CRC32 checksum of a serial data. + */ +uint32_t ZLIB::calculate_crc(const void* buf, size_t size, uint32_t seed) { +#if _KC_ZLIB + _assert_(buf && size <= MEMMAXSIZ); + return crc32(seed, (unsigned char*)buf, size); +#else + _assert_(buf && size <= MEMMAXSIZ); + return 0; +#endif +} + + +/** + * Hidden resources for LZO. + */ +#if _KC_LZO +static int32_t lzo_init_func() { + if (lzo_init() != LZO_E_OK) throw std::runtime_error("lzo_init"); + return 0; +} +int32_t lzo_init_var = lzo_init_func(); +#endif + + +/** + * Compress a serial data. + */ +char* LZO::compress(const void* buf, size_t size, size_t* sp, Mode mode) { +#if _KC_LZO + _assert_(buf && size <= MEMMAXSIZ && sp); + char* zbuf = new char[size+size/16+80]; + lzo_uint zsiz; + char wrkmem[LZO1X_1_MEM_COMPRESS]; + if (lzo1x_1_compress((lzo_bytep)buf, size, (lzo_bytep)zbuf, &zsiz, wrkmem) != LZO_E_OK) { + delete[] zbuf; + return NULL; + } + if (mode == CRC) { + uint32_t hash = lzo_crc32(0, (const lzo_bytep)zbuf, zsiz); + writefixnum(zbuf + zsiz, hash, sizeof(hash)); + zsiz += sizeof(hash); + } + zbuf[zsiz] = '\0'; + *sp = zsiz; + return (char*)zbuf; +#else + _assert_(buf && size <= MEMMAXSIZ && sp); + char* zbuf = new char[size+2]; + char* wp = zbuf; + *(wp++) = 'o'; + *(wp++) = mode; + std::memcpy(wp, buf, size); + *sp = size + 2; + return zbuf; +#endif +} + + +/** + * Decompress a serial data. + */ +char* LZO::decompress(const void* buf, size_t size, size_t* sp, Mode mode) { +#if _KC_LZO + _assert_(buf && size <= MEMMAXSIZ && sp); + if (mode == CRC) { + if (size < sizeof(uint32_t)) return NULL; + uint32_t hash = readfixnum((const char*)buf + size - sizeof(hash), sizeof(hash)); + size -= sizeof(hash); + if (lzo_crc32(0, (const lzo_bytep)buf, size) != hash) return NULL; + } + char* zbuf; + lzo_uint zsiz; + int32_t rat = 6; + while (true) { + zsiz = (size + 256) * rat + 3; + zbuf = new char[zsiz+1]; + int32_t rv; + if (mode == RAW) { + rv = lzo1x_decompress_safe((lzo_bytep)buf, size, (lzo_bytep)zbuf, &zsiz, NULL); + } else { + rv = lzo1x_decompress((lzo_bytep)buf, size, (lzo_bytep)zbuf, &zsiz, NULL); + } + if (rv == LZO_E_OK) { + break; + } else if (rv == LZO_E_OUTPUT_OVERRUN) { + delete[] zbuf; + rat *= 2; + } else { + delete[] zbuf; + return NULL; + } + } + zbuf[zsiz] = '\0'; + if (sp) *sp = zsiz; + return (char*)zbuf; +#else + _assert_(buf && size <= MEMMAXSIZ && sp); + if (size < 2 || ((char*)buf)[0] != 'o' || ((char*)buf)[1] != mode) return NULL; + buf = (char*)buf + 2; + size -= 2; + char* zbuf = new char[size+1]; + std::memcpy(zbuf, buf, size); + zbuf[size] = '\0'; + *sp = size; + return zbuf; +#endif +} + + +/** + * Calculate the CRC32 checksum of a serial data. + */ +uint32_t LZO::calculate_crc(const void* buf, size_t size, uint32_t seed) { +#if _KC_LZO + _assert_(buf && size <= MEMMAXSIZ); + return lzo_crc32(seed, (const lzo_bytep)buf, size); +#else + _assert_(buf && size <= MEMMAXSIZ); + return 0; +#endif +} + + +/** + * Compress a serial data. + */ +char* LZMA::compress(const void* buf, size_t size, size_t* sp, Mode mode) { +#if _KC_LZMA + _assert_(buf && size <= MEMMAXSIZ && sp); + lzma_stream zs = LZMA_STREAM_INIT; + const char* rp = (const char*)buf; + size_t zsiz = size + 1024; + char* zbuf = new char[zsiz+1]; + char* wp = zbuf; + zs.next_in = (const uint8_t*)rp; + zs.avail_in = size; + zs.next_out = (uint8_t*)wp; + zs.avail_out = zsiz; + switch (mode) { + default: { + if (lzma_easy_encoder(&zs, 6, LZMA_CHECK_NONE) != LZMA_OK) return NULL; + break; + } + case CRC: { + if (lzma_easy_encoder(&zs, 6, LZMA_CHECK_CRC32) != LZMA_OK) return NULL; + break; + } + case SHA: { + if (lzma_easy_encoder(&zs, 6, LZMA_CHECK_SHA256) != LZMA_OK) return NULL; + break; + } + } + if (lzma_code(&zs, LZMA_FINISH) != LZMA_STREAM_END) { + delete[] zbuf; + lzma_end(&zs); + return NULL; + } + lzma_end(&zs); + zsiz -= zs.avail_out; + *sp = zsiz; + return zbuf; +#else + _assert_(buf && size <= MEMMAXSIZ && sp); + char* zbuf = new char[size+2]; + char* wp = zbuf; + *(wp++) = 'x'; + *(wp++) = mode; + std::memcpy(wp, buf, size); + *sp = size + 2; + return zbuf; +#endif +} + + +/** + * Decompress a serial data. + */ +char* LZMA::decompress(const void* buf, size_t size, size_t* sp, Mode mode) { +#if _KC_LZMA + _assert_(buf && size <= MEMMAXSIZ && sp); + size_t zsiz = size * 8 + 32; + while (true) { + lzma_stream zs = LZMA_STREAM_INIT; + const char* rp = (const char*)buf; + char* zbuf = new char[zsiz+1]; + char* wp = zbuf; + zs.next_in = (const uint8_t*)rp; + zs.avail_in = size; + zs.next_out = (uint8_t*)wp; + zs.avail_out = zsiz; + if (lzma_auto_decoder(&zs, 1ULL << 30, 0) != LZMA_OK) return NULL; + int32_t rv = lzma_code(&zs, LZMA_FINISH); + lzma_end(&zs); + if (rv == LZMA_STREAM_END) { + zsiz -= zs.avail_out; + zbuf[zsiz] = '\0'; + *sp = zsiz; + return zbuf; + } else if (rv == LZMA_OK) { + delete[] zbuf; + zsiz *= 2; + } else { + delete[] zbuf; + break; + } + } + return NULL; +#else + _assert_(buf && size <= MEMMAXSIZ && sp); + if (size < 2 || ((char*)buf)[0] != 'x' || ((char*)buf)[1] != mode) return NULL; + buf = (char*)buf + 2; + size -= 2; + char* zbuf = new char[size+1]; + std::memcpy(zbuf, buf, size); + zbuf[size] = '\0'; + *sp = size; + return zbuf; +#endif +} + + +/** + * Calculate the CRC32 checksum of a serial data. + */ +uint32_t LZMA::calculate_crc(const void* buf, size_t size, uint32_t seed) { +#if _KC_LZMA + _assert_(buf && size <= MEMMAXSIZ); + return lzma_crc32((const uint8_t*)buf, size, seed); +#else + _assert_(buf && size <= MEMMAXSIZ); + return 0; +#endif +} + + +/** + * Prepared pointer of the ZLIB raw mode. + */ +ZLIBCompressor<ZLIB::RAW> zlibrawfunc; +ZLIBCompressor<ZLIB::RAW>* const ZLIBRAWCOMP = &zlibrawfunc; + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kccompress.h b/plugins/Dbx_kyoto/src/kyotocabinet/kccompress.h new file mode 100644 index 0000000000..309e5fbd61 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kccompress.h @@ -0,0 +1,403 @@ +/************************************************************************************************* + * Data compressor and decompressor + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCCOMPRESS_H // duplication check +#define _KCCOMPRESS_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> + +namespace kyotocabinet { // common namespace + + +/** + * Interfrace of data compression and decompression. + */ +class Compressor { + public: + /** + * Destructor. + */ + virtual ~Compressor() {} + /** + * Compress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the result data, or NULL on failure. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ + virtual char* compress(const void* buf, size_t size, size_t* sp) = 0; + /** + * Decompress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the result data, or NULL on failure. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a C-style string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the + * delete[] operator when it is no longer in use. + */ + virtual char* decompress(const void* buf, size_t size, size_t* sp) = 0; +}; + + +/** + * ZLIB compressor. + */ +class ZLIB { + public: + /** + * Compression modes. + */ + enum Mode { + RAW, ///< without any checksum + DEFLATE, ///< with Adler32 checksum + GZIP ///< with CRC32 checksum and various meta data + }; + /** + * Compress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param mode the compression mode. + * @return the pointer to the result data, or NULL on failure. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ + static char* compress(const void* buf, size_t size, size_t* sp, Mode mode = RAW); + /** + * Decompress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param mode the compression mode. + * @return the pointer to the result data, or NULL on failure. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a C-style string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the + * delete[] operator when it is no longer in use. + */ + static char* decompress(const void* buf, size_t size, size_t* sp, Mode mode = RAW); + /** + * Calculate the CRC32 checksum of a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param seed the cyclic seed value. + * @return the CRC32 checksum. + */ + static uint32_t calculate_crc(const void* buf, size_t size, uint32_t seed = 0); +}; + + +/** + * LZO compressor. + */ +class LZO { + public: + /** + * Compression modes. + */ + enum Mode { + RAW, ///< without any checksum + CRC ///< with CRC32 checksum + }; + /** + * Compress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param mode the compression mode. + * @return the pointer to the result data, or NULL on failure. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ + static char* compress(const void* buf, size_t size, size_t* sp, Mode mode = RAW); + /** + * Decompress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param mode the compression mode. + * @return the pointer to the result data, or NULL on failure. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a C-style string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the + * delete[] operator when it is no longer in use. + */ + static char* decompress(const void* buf, size_t size, size_t* sp, Mode mode = RAW); + /** + * Calculate the CRC32 checksum of a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param seed the cyclic seed value. + * @return the CRC32 checksum. + */ + static uint32_t calculate_crc(const void* buf, size_t size, uint32_t seed = 0); +}; + + +/** + * LZMA compressor. + */ +class LZMA { + public: + /** + * Compression modes. + */ + enum Mode { + RAW, ///< without any checksum + CRC, ///< with CRC32 checksum + SHA ///< with SHA256 checksum + }; + /** + * Compress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param mode the compression mode. + * @return the pointer to the result data, or NULL on failure. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ + static char* compress(const void* buf, size_t size, size_t* sp, Mode mode = RAW); + /** + * Decompress a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param mode the compression mode. + * @return the pointer to the result data, or NULL on failure. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a C-style string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the + * delete[] operator when it is no longer in use. + */ + static char* decompress(const void* buf, size_t size, size_t* sp, Mode mode = RAW); + /** + * Calculate the CRC32 checksum of a serial data. + * @param buf the input buffer. + * @param size the size of the input buffer. + * @param seed the cyclic seed value. + * @return the CRC32 checksum. + */ + static uint32_t calculate_crc(const void* buf, size_t size, uint32_t seed = 0); +}; + + +/** + * Compressor with ZLIB. + */ +template <ZLIB::Mode MODE> +class ZLIBCompressor : public Compressor { + private: + /** + * Compress a serial data. + */ + char* compress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + return ZLIB::compress(buf, size, sp, MODE); + } + /** + * Decompress a serial data. + */ + char* decompress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + return ZLIB::decompress(buf, size, sp, MODE); + } +}; + + +/** + * Compressor with LZO. + */ +template <LZO::Mode MODE> +class LZOCompressor : public Compressor { + private: + /** + * Compress a serial data. + */ + char* compress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + return LZO::compress(buf, size, sp, MODE); + } + /** + * Decompress a serial data. + */ + char* decompress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + return LZO::decompress(buf, size, sp, MODE); + } +}; + + +/** + * Compressor with LZMA. + */ +template <LZMA::Mode MODE> +class LZMACompressor : public Compressor { + private: + /** + * Compress a serial data. + */ + char* compress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + return LZMA::compress(buf, size, sp, MODE); + } + /** + * Decompress a serial data. + */ + char* decompress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + return LZMA::decompress(buf, size, sp, MODE); + } +}; + + +/** + * Compressor with the Arcfour cipher. + */ +class ArcfourCompressor : public Compressor { + public: + /** + * Constructor. + */ + ArcfourCompressor() : kbuf_(NULL), ksiz_(0), comp_(NULL), salt_(0), cycle_(false) { + _assert_(true); + kbuf_ = new char[1]; + ksiz_ = 0; + } + /** + * Destructor. + */ + ~ArcfourCompressor() { + _assert_(true); + delete[] kbuf_; + } + /** + * Set the cipher key. + * @param kbuf the pointer to the region of the cipher key. + * @param ksiz the size of the region of the cipher key. + */ + void set_key(const void* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + delete[] kbuf_; + if (ksiz > NUMBUFSIZ) ksiz = NUMBUFSIZ; + kbuf_ = new char[ksiz]; + std::memcpy(kbuf_, kbuf, ksiz); + ksiz_ = ksiz; + } + /** + * Set an additional data compressor. + * @param comp the additional data data compressor. + */ + void set_compressor(Compressor* comp) { + _assert_(comp); + comp_ = comp; + } + /** + * Begin the cycle of ciper salt. + * @param salt the additional cipher salt. + */ + void begin_cycle(uint64_t salt = 0) { + salt_ = salt; + cycle_ = true; + } + private: + /** + * Compress a serial data. + */ + char* compress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + uint64_t salt = cycle_ ? salt_.add(1) : 0; + char kbuf[NUMBUFSIZ*2]; + writefixnum(kbuf, salt, sizeof(salt)); + std::memcpy(kbuf + sizeof(salt), kbuf_, ksiz_); + char* tbuf = NULL; + if (comp_) { + tbuf = comp_->compress(buf, size, &size); + if (!tbuf) return NULL; + buf = tbuf; + } + size_t zsiz = sizeof(salt) + size; + char* zbuf = new char[zsiz]; + writefixnum(zbuf, salt, sizeof(salt)); + arccipher(buf, size, kbuf, sizeof(salt) + ksiz_, zbuf + sizeof(salt)); + delete[] tbuf; + if (cycle_) { + size_t range = zsiz - sizeof(salt); + if (range > (size_t)INT8MAX) range = INT8MAX; + salt_.add(hashmurmur(zbuf + sizeof(salt), range) << 32); + } + *sp = zsiz; + return zbuf; + } + /** + * Decompress a serial data. + */ + char* decompress(const void* buf, size_t size, size_t* sp) { + _assert_(buf && size <= MEMMAXSIZ && sp); + if (size < sizeof(uint64_t)) return NULL; + char kbuf[NUMBUFSIZ*2]; + std::memcpy(kbuf, buf, sizeof(uint64_t)); + std::memcpy(kbuf + sizeof(uint64_t), kbuf_, ksiz_); + buf = (char*)buf + sizeof(uint64_t); + size -= sizeof(uint64_t); + char* zbuf = new char[size]; + arccipher(buf, size, kbuf, sizeof(uint64_t) + ksiz_, zbuf); + if (comp_) { + char* tbuf = comp_->decompress(zbuf, size, &size); + delete[] zbuf; + if (!tbuf) return NULL; + zbuf = tbuf; + } + *sp = size; + return zbuf; + } + /** The pointer to the key. */ + char* kbuf_; + /** The size of the key. */ + size_t ksiz_; + /** The data compressor. */ + Compressor* comp_; + /** The cipher salt. */ + AtomicInt64 salt_; + /** The flag of the salt cycle */ + bool cycle_; +}; + + +/** + * Prepared pointer of the compressor with ZLIB raw mode. + */ +extern ZLIBCompressor<ZLIB::RAW>* const ZLIBRAWCOMP; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcdb.cc new file mode 100644 index 0000000000..0e0d6c73fb --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdb.cc @@ -0,0 +1,31 @@ +/************************************************************************************************* + * Database interface + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcdb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +/** Special pointer for no operation. */ +const char* const DB::Visitor::NOP = (const char*)0; + +/** Special pointer to remove the record. */ +const char* const DB::Visitor::REMOVE = (const char*)1; + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcdb.h new file mode 100644 index 0000000000..8abffb4c6b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdb.h @@ -0,0 +1,2520 @@ +/************************************************************************************************* + * Database interface + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCDB_H // duplication check +#define _KCDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> + +#define KCDBSSMAGICDATA "KCSS\n" ///< The magic data of the snapshot file + +namespace kyotocabinet { // common namespace + + +/** + * Interface of database abstraction. + * @note This class is an abstract class to prescribe the interface of record access. + */ +class DB { + public: + /** + * Interface to access a record. + */ + class Visitor { + public: + /** Special pointer for no operation. */ + static const char* const NOP; + /** Special pointer to remove the record. */ + static const char* const REMOVE; + /** + * Destructor. + */ + virtual ~Visitor() { + _assert_(true); + } + /** + * Visit a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return If it is the pointer to a region, the value is replaced by the content. If it + * is Visitor::NOP, nothing is modified. If it is Visitor::REMOVE, the record is removed. + */ + virtual const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + return NOP; + } + /** + * Visit a empty record space. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return If it is the pointer to a region, the value is replaced by the content. If it + * is Visitor::NOP or Visitor::REMOVE, nothing is modified. + */ + virtual const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && sp); + return NOP; + } + /** + * Preprocess the main operations. + */ + virtual void visit_before() { + _assert_(true); + } + /** + * Postprocess the main operations. + */ + virtual void visit_after() { + _assert_(true); + } + }; + /** + * Interface of cursor to indicate a record. + */ + class Cursor { + public: + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + virtual bool accept(Visitor* visitor, bool writable = true, bool step = false) = 0; + /** + * Set the value of the current record. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + */ + virtual bool set_value(const char* vbuf, size_t vsiz, bool step = false) = 0; + /** + * Set the value of the current record. + * @note Equal to the original Cursor::set_value method except that the parameter is + * std::string. + */ + virtual bool set_value_str(const std::string& value, bool step = false) = 0; + /** + * Remove the current record. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. The cursor is moved to the + * next record implicitly. + */ + virtual bool remove() = 0; + /** + * Get the key of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the key region of the current record, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + virtual char* get_key(size_t* sp, bool step = false) = 0; + /** + * Get the key of the current record. + * @note Equal to the original Cursor::get_key method except that a parameter is a string to + * contain the result and the return value is bool for success. + */ + virtual bool get_key(std::string* key, bool step = false) = 0; + /** + * Get the value of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the value region of the current record, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + virtual char* get_value(size_t* sp, bool step = false) = 0; + /** + * Get the value of the current record. + * @note Equal to the original Cursor::get_value method except that a parameter is a string + * to contain the result and the return value is bool for success. + */ + virtual bool get_value(std::string* value, bool step = false) = 0; + /** + * Get a pair of the key and the value of the current record. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the key region, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero code is + * appended at the end of each region of the key and the value, each region can be treated + * as a C-style string. The return value should be deleted explicitly by the caller with + * the detele[] operator. + */ + virtual char* get(size_t* ksp, const char** vbp, size_t* vsp, bool step = false) = 0; + /** + * Get a pair of the key and the value of the current record. + * @note Equal to the original Cursor::get method except that parameters are strings + * to contain the result and the return value is bool for success. + */ + virtual bool get(std::string* key, std::string* value, bool step = false) = 0; + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + virtual bool jump() = 0; + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + virtual bool jump(const char* kbuf, size_t ksiz) = 0; + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + virtual bool jump(const std::string& key) = 0; + /** + * Jump the cursor to the last record for backward scan. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, will provide a dummy implementation. + */ + virtual bool jump_back() = 0; + /** + * Jump the cursor to a record for backward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, will provide a dummy implementation. + */ + virtual bool jump_back(const char* kbuf, size_t ksiz) = 0; + /** + * Jump the cursor to a record for backward scan. + * @note Equal to the original Cursor::jump_back method except that the parameter is + * std::string. + */ + virtual bool jump_back(const std::string& key) = 0; + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + virtual bool step() = 0; + /** + * Step the cursor to the previous record. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, will provide a dummy implementation. + */ + virtual bool step_back() = 0; + /** + * Get the database object. + * @return the database object. + */ + virtual DB* db() = 0; + }; + /** + * Destructor. + */ + virtual ~DB() { + _assert_(true); + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + virtual bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) = 0; + /** + * Set the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the value is overwritten. + */ + virtual bool set(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0; + /** + * Set the value of a record. + * @note Equal to the original DB::set method except that the parameters are std::string. + */ + virtual bool set(const std::string& key, const std::string& value) = 0; + /** + * Add a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the record is not modified and false is returned. + */ + virtual bool add(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0; + /** + * Set the value of a record. + * @note Equal to the original DB::add method except that the parameters are std::string. + */ + virtual bool add(const std::string& key, const std::string& value) = 0; + /** + * Replace the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, no new record is created and false is returned. + * If the corresponding record exists, the value is modified. + */ + virtual bool replace(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0; + /** + * Replace the value of a record. + * @note Equal to the original DB::replace method except that the parameters are std::string. + */ + virtual bool replace(const std::string& key, const std::string& value) = 0; + /** + * Append the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the given value is appended at the end of the existing value. + */ + virtual bool append(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0; + /** + * Set the value of a record. + * @note Equal to the original DB::append method except that the parameters are std::string. + */ + virtual bool append(const std::string& key, const std::string& value) = 0; + /** + * Add a number to the numeric integer value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param num the additional number. + * @param orig the origin number if no record corresponds to the key. If it is INT64MIN and + * no record corresponds, this function fails. If it is INT64MAX, the value is set as the + * additional number regardless of the current value. + * @return the result value, or kyotocabinet::INT64MIN on failure. + * @note The value is serialized as an 8-byte binary integer in big-endian order, not a decimal + * string. If existing value is not 8-byte, this function fails. + */ + virtual int64_t increment(const char* kbuf, size_t ksiz, int64_t num, int64_t orig = 0) = 0; + /** + * Add a number to the numeric integer value of a record. + * @note Equal to the original DB::increment method except that the parameter is std::string. + */ + virtual int64_t increment(const std::string& key, int64_t num, int64_t orig = 0) = 0; + /** + * Add a number to the numeric double value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param num the additional number. + * @param orig the origin number if no record corresponds to the key. If it is negative + * infinity and no record corresponds, this function fails. If it is positive infinity, the + * value is set as the additional number regardless of the current value. + * @return the result value, or Not-a-number on failure. + * @note The value is serialized as an 16-byte binary fixed-point number in big-endian order, + * not a decimal string. If existing value is not 16-byte, this function fails. + */ + virtual double increment_double(const char* kbuf, size_t ksiz, double num, + double orig = 0) = 0; + /** + * Add a number to the numeric double value of a record. + * @note Equal to the original DB::increment_double method except that the parameter is + * std::string. + */ + virtual double increment_double(const std::string& key, double num, double orig = 0) = 0; + /** + * Perform compare-and-swap. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param ovbuf the pointer to the old value region. NULL means that no record corresponds. + * @param ovsiz the size of the old value region. + * @param nvbuf the pointer to the new value region. NULL means that the record is removed. + * @param nvsiz the size of new old value region. + * @return true on success, or false on failure. + */ + virtual bool cas(const char* kbuf, size_t ksiz, + const char* ovbuf, size_t ovsiz, const char* nvbuf, size_t nvsiz) = 0; + /** + * Perform compare-and-swap. + * @note Equal to the original DB::cas method except that the parameters are std::string. + */ + virtual bool cas(const std::string& key, + const std::string& ovalue, const std::string& nvalue) = 0; + /** + * Remove a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. + */ + virtual bool remove(const char* kbuf, size_t ksiz) = 0; + /** + * Remove a record. + * @note Equal to the original DB::remove method except that the parameter is std::string. + */ + virtual bool remove(const std::string& key) = 0; + /** + * Retrieve the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + * @note If no record corresponds to the key, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + virtual char* get(const char* kbuf, size_t ksiz, size_t* sp) = 0; + /** + * Retrieve the value of a record. + * @note Equal to the original DB::get method except that the first parameters is the key + * string and the second parameter is a string to contain the result and the return value is + * bool for success. + */ + virtual bool get(const std::string& key, std::string* value) = 0; + /** + * Retrieve the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the buffer into which the value of the corresponding record is + * written. + * @param max the size of the buffer. + * @return the size of the value, or -1 on failure. + */ + virtual int32_t get(const char* kbuf, size_t ksiz, char* vbuf, size_t max) = 0; + /** + * Check the existence of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the size of the value, or -1 on failure. + */ + virtual int32_t check(const char* kbuf, size_t ksiz) = 0; + /** + * Check the existence of a record. + * @note Equal to the original DB::check method except that the parameter is std::string. + */ + virtual int32_t check(const std::string& key) = 0; + /** + * Remove all records. + * @return true on success, or false on failure. + */ + virtual bool clear() = 0; + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + virtual int64_t count() = 0; + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + virtual Cursor* cursor() = 0; +}; + + +/** + * Basic implementation of database. + * @note This class is an abstract class to prescribe the interface of file operations and + * provide mix-in methods. This class can be inherited but overwriting methods is forbidden. + * Before every database operation, it is necessary to call the BasicDB::open method in order to + * open a database file and connect the database object to it. To avoid data missing or + * corruption, it is important to close every database file by the BasicDB::close method when the + * database is no longer in use. It is forbidden for multible database objects in a process to + * open the same database at the same time. It is forbidden to share a database object with + * child processes. + */ +class BasicDB : public DB { + public: + class Cursor; + class Error; + class ProgressChecker; + class FileProcessor; + class Logger; + class MetaTrigger; + private: + /** The size of the IO buffer. */ + static const size_t IOBUFSIZ = 8192; + public: + /** + * Database types. + */ + enum Type { + TYPEVOID = 0x00, ///< void database + TYPEPHASH = 0x10, ///< prototype hash database + TYPEPTREE = 0x11, ///< prototype tree database + TYPESTASH = 0x18, ///< stash database + TYPECACHE = 0x20, ///< cache hash database + TYPEGRASS = 0x21, ///< cache tree database + TYPEHASH = 0x30, ///< file hash database + TYPETREE = 0x31, ///< file tree database + TYPEDIR = 0x40, ///< directory hash database + TYPEFOREST = 0x41, ///< directory tree database + TYPETEXT = 0x50, ///< plain text database + TYPEMISC = 0x80 ///< miscellaneous database + }; + /** + * Interface of cursor to indicate a record. + */ + class Cursor : public DB::Cursor { + public: + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + } + /** + * Set the value of the current record. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + */ + bool set_value(const char* vbuf, size_t vsiz, bool step = false) { + _assert_(vbuf && vsiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz) : + vbuf_(vbuf), vsiz_(vsiz), ok_(false) {} + bool ok() const { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + ok_ = true; + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; + size_t vsiz_; + bool ok_; + }; + VisitorImpl visitor(vbuf, vsiz); + if (!accept(&visitor, true, step)) return false; + if (!visitor.ok()) return false; + return true; + } + /** + * Set the value of the current record. + * @note Equal to the original Cursor::set_value method except that the parameter is + * std::string. + */ + bool set_value_str(const std::string& value, bool step = false) { + _assert_(true); + return set_value(value.c_str(), value.size(), step); + } + /** + * Remove the current record. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. The cursor is moved to the + * next record implicitly. + */ + bool remove() { + _assert_(true); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : ok_(false) {} + bool ok() const { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + ok_ = true; + return REMOVE; + } + bool ok_; + }; + VisitorImpl visitor; + if (!accept(&visitor, true, false)) return false; + if (!visitor.ok()) return false; + return true; + } + /** + * Get the key of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the key region of the current record, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + char* get_key(size_t* sp, bool step = false) { + _assert_(sp); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : kbuf_(NULL), ksiz_(0) {} + char* pop(size_t* sp) { + *sp = ksiz_; + return kbuf_; + } + void clear() { + delete[] kbuf_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + kbuf_ = new char[ksiz+1]; + std::memcpy(kbuf_, kbuf, ksiz); + kbuf_[ksiz] = '\0'; + ksiz_ = ksiz; + return NOP; + } + char* kbuf_; + size_t ksiz_; + }; + VisitorImpl visitor; + if (!accept(&visitor, false, step)) { + visitor.clear(); + *sp = 0; + return NULL; + } + size_t ksiz; + char* kbuf = visitor.pop(&ksiz); + if (!kbuf) { + *sp = 0; + return NULL; + } + *sp = ksiz; + return kbuf; + } + /** + * Get the key of the current record. + * @note Equal to the original Cursor::get_key method except that a parameter is a string to + * contain the result and the return value is bool for success. + */ + bool get_key(std::string* key, bool step = false) { + _assert_(key); + size_t ksiz; + char* kbuf = get_key(&ksiz, step); + if (!kbuf) return false; + key->clear(); + key->append(kbuf, ksiz); + delete[] kbuf; + return true; + } + /** + * Get the value of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the value region of the current record, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + char* get_value(size_t* sp, bool step = false) { + _assert_(sp); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : vbuf_(NULL), vsiz_(0) {} + char* pop(size_t* sp) { + *sp = vsiz_; + return vbuf_; + } + void clear() { + delete[] vbuf_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + vbuf_ = new char[vsiz+1]; + std::memcpy(vbuf_, vbuf, vsiz); + vbuf_[vsiz] = '\0'; + vsiz_ = vsiz; + return NOP; + } + char* vbuf_; + size_t vsiz_; + }; + VisitorImpl visitor; + if (!accept(&visitor, false, step)) { + visitor.clear(); + *sp = 0; + return NULL; + } + size_t vsiz; + char* vbuf = visitor.pop(&vsiz); + if (!vbuf) { + *sp = 0; + return NULL; + } + *sp = vsiz; + return vbuf; + } + /** + * Get the value of the current record. + * @note Equal to the original Cursor::get_value method except that a parameter is a string + * to contain the result and the return value is bool for success. + */ + bool get_value(std::string* value, bool step = false) { + _assert_(value); + size_t vsiz; + char* vbuf = get_value(&vsiz, step); + if (!vbuf) return false; + value->clear(); + value->append(vbuf, vsiz); + delete[] vbuf; + return true; + } + /** + * Get a pair of the key and the value of the current record. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the key region, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero code is + * appended at the end of each region of the key and the value, each region can be treated + * as a C-style string. The return value should be deleted explicitly by the caller with + * the detele[] operator. + */ + char* get(size_t* ksp, const char** vbp, size_t* vsp, bool step = false) { + _assert_(ksp && vbp && vsp); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : kbuf_(NULL), ksiz_(0), vbuf_(NULL), vsiz_(0) {} + char* pop(size_t* ksp, const char** vbp, size_t* vsp) { + *ksp = ksiz_; + *vbp = vbuf_; + *vsp = vsiz_; + return kbuf_; + } + void clear() { + delete[] kbuf_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + size_t rsiz = ksiz + 1 + vsiz + 1; + kbuf_ = new char[rsiz]; + std::memcpy(kbuf_, kbuf, ksiz); + kbuf_[ksiz] = '\0'; + ksiz_ = ksiz; + vbuf_ = kbuf_ + ksiz + 1; + std::memcpy(vbuf_, vbuf, vsiz); + vbuf_[vsiz] = '\0'; + vsiz_ = vsiz; + return NOP; + } + char* kbuf_; + size_t ksiz_; + char* vbuf_; + size_t vsiz_; + }; + VisitorImpl visitor; + if (!accept(&visitor, false, step)) { + visitor.clear(); + *ksp = 0; + *vbp = NULL; + *vsp = 0; + return NULL; + } + return visitor.pop(ksp, vbp, vsp); + } + /** + * Get a pair of the key and the value of the current record. + * @note Equal to the original Cursor::get method except that parameters are strings + * to contain the result and the return value is bool for success. + */ + bool get(std::string* key, std::string* value, bool step = false) { + _assert_(key && value); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(std::string* key, std::string* value) : + key_(key), value_(value), ok_(false) {} + bool ok() { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + key_->clear(); + key_->append(kbuf, ksiz); + value_->clear(); + value_->append(vbuf, vsiz); + ok_ = true; + return NOP; + } + std::string* key_; + std::string* value_; + bool ok_; + }; + VisitorImpl visitor(key, value); + if (!accept(&visitor, false, step)) return false; + return visitor.ok(); + } + /** + * Get a pair of the key and the value of the current record and remove it atomically. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @return the pointer to the key region, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero code is + * appended at the end of each region of the key and the value, each region can be treated + * as a C-style string. The return value should be deleted explicitly by the caller with + * the detele[] operator. The cursor is moved to the next record implicitly. + */ + char* seize(size_t* ksp, const char** vbp, size_t* vsp) { + _assert_(ksp && vbp && vsp); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : kbuf_(NULL), ksiz_(0), vbuf_(NULL), vsiz_(0) {} + char* pop(size_t* ksp, const char** vbp, size_t* vsp) { + *ksp = ksiz_; + *vbp = vbuf_; + *vsp = vsiz_; + return kbuf_; + } + void clear() { + delete[] kbuf_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + size_t rsiz = ksiz + 1 + vsiz + 1; + kbuf_ = new char[rsiz]; + std::memcpy(kbuf_, kbuf, ksiz); + kbuf_[ksiz] = '\0'; + ksiz_ = ksiz; + vbuf_ = kbuf_ + ksiz + 1; + std::memcpy(vbuf_, vbuf, vsiz); + vbuf_[vsiz] = '\0'; + vsiz_ = vsiz; + return REMOVE; + } + char* kbuf_; + size_t ksiz_; + char* vbuf_; + size_t vsiz_; + }; + VisitorImpl visitor; + if (!accept(&visitor, true, false)) { + visitor.clear(); + *ksp = 0; + *vbp = NULL; + *vsp = 0; + return NULL; + } + return visitor.pop(ksp, vbp, vsp); + } + /** + * Get a pair of the key and the value of the current record and remove it atomically. + * @note Equal to the original Cursor::seize method except that parameters are strings + * to contain the result and the return value is bool for success. + */ + bool seize(std::string* key, std::string* value) { + _assert_(key && value); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(std::string* key, std::string* value) : + key_(key), value_(value), ok_(false) {} + bool ok() { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + key_->clear(); + key_->append(kbuf, ksiz); + value_->clear(); + value_->append(vbuf, vsiz); + ok_ = true; + return REMOVE; + } + std::string* key_; + std::string* value_; + bool ok_; + }; + VisitorImpl visitor(key, value); + if (!accept(&visitor, true, false)) return false; + return visitor.ok(); + } + /** + * Get the database object. + * @return the database object. + */ + virtual BasicDB* db() = 0; + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() { + _assert_(true); + return db()->error(); + } + }; + /** + * Error data. + */ + class Error { + public: + /** + * Error codes. + */ + enum Code { + SUCCESS, ///< success + NOIMPL, ///< not implemented + INVALID, ///< invalid operation + NOREPOS, ///< no repository + NOPERM, ///< no permission + BROKEN, ///< broken file + DUPREC, ///< record duplication + NOREC, ///< no record + LOGIC, ///< logical inconsistency + SYSTEM, ///< system error + MISC = 15 ///< miscellaneous error + }; + /** + * Default constructor. + */ + explicit Error() : code_(SUCCESS), message_("no error") { + _assert_(true); + } + /** + * Copy constructor. + * @param src the source object. + */ + Error(const Error& src) : code_(src.code_), message_(src.message_) { + _assert_(true); + } + /** + * Constructor. + * @param code an error code. + * @param message a supplement message. + */ + explicit Error(Code code, const char* message) : code_(code), message_(message) { + _assert_(message); + } + /** + * Destructor. + */ + ~Error() { + _assert_(true); + } + /** + * Set the error information. + * @param code an error code. + * @param message a supplement message. + */ + void set(Code code, const char* message) { + _assert_(message); + code_ = code; + message_ = message; + } + /** + * Get the error code. + * @return the error code. + */ + Code code() const { + _assert_(true); + return code_; + } + /** + * Get the readable string of the code. + * @return the readable string of the code. + */ + const char* name() const { + _assert_(true); + return codename(code_); + } + /** + * Get the supplement message. + * @return the supplement message. + */ + const char* message() const { + _assert_(true); + return message_; + } + /** + * Get the readable string of an error code. + * @param code the error code. + * @return the readable string of the error code. + */ + static const char* codename(Code code) { + _assert_(true); + switch (code) { + case SUCCESS: return "success"; + case NOIMPL: return "not implemented"; + case INVALID: return "invalid operation"; + case NOREPOS: return "no repository"; + case NOPERM: return "no permission"; + case BROKEN: return "broken file"; + case DUPREC: return "record duplication"; + case NOREC: return "no record"; + case LOGIC: return "logical inconsistency"; + case SYSTEM: return "system error"; + default: break; + } + return "miscellaneous error"; + } + /** + * Assignment operator from the self type. + * @param right the right operand. + * @return the reference to itself. + */ + Error& operator =(const Error& right) { + _assert_(true); + if (&right == this) return *this; + code_ = right.code_; + message_ = right.message_; + return *this; + } + /** + * Cast operator to integer. + * @return the error code. + */ + operator int32_t() const { + return code_; + } + private: + /** The error code. */ + Code code_; + /** The supplement message. */ + const char* message_; + }; + /** + * Interface to check progress status of long process. + */ + class ProgressChecker { + public: + /** + * Destructor. + */ + virtual ~ProgressChecker() { + _assert_(true); + } + /** + * Check the progress status. + * @param name the name of the process. + * @param message a supplement message. + * @param curcnt the count of the current step of the progress, or -1 if not applicable. + * @param allcnt the estimation count of all steps of the progress, or -1 if not applicable. + * @return true to continue the process, or false to stop the process. + */ + virtual bool check(const char* name, const char* message, + int64_t curcnt, int64_t allcnt) = 0; + }; + /** + * Interface to process the database file. + */ + class FileProcessor { + public: + /** + * Destructor. + */ + virtual ~FileProcessor() { + _assert_(true); + } + /** + * Process the database file. + * @param path the path of the database file. + * @param count the number of records. A negative value means omission. + * @param size the size of the available region. A negative value means omission. + * @return true on success, or false on failure. + */ + virtual bool process(const std::string& path, int64_t count, int64_t size) = 0; + }; + /** + * Interface to log internal information and errors. + */ + class Logger { + public: + /** + * Event kinds. + */ + enum Kind { + DEBUG = 1 << 0, ///< debugging + INFO = 1 << 1, ///< normal information + WARN = 1 << 2, ///< warning + ERROR = 1 << 3 ///< error + }; + /** + * Destructor. + */ + virtual ~Logger() { + _assert_(true); + } + /** + * Process a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + virtual void log(const char* file, int32_t line, const char* func, Kind kind, + const char* message) = 0; + }; + /** + * Interface to trigger meta database operations. + */ + class MetaTrigger { + public: + /** + * Event kinds. + */ + enum Kind { + OPEN, ///< opening + CLOSE, ///< closing + CLEAR, ///< clearing + ITERATE, ///< iteration + SYNCHRONIZE, ///< synchronization + OCCUPY, ///< occupation + BEGINTRAN, ///< beginning transaction + COMMITTRAN, ///< committing transaction + ABORTTRAN, ///< aborting transaction + MISC = 15 ///< miscellaneous operation + }; + /** + * Destructor. + */ + virtual ~MetaTrigger() { + _assert_(true); + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::OCCUPY for occupation, + * MetaTrigger::BEGINTRAN for beginning transaction, MetaTrigger::COMMITTRAN for committing + * transaction, MetaTrigger::ABORTTRAN for aborting transaction, and MetaTrigger::MISC for + * miscellaneous operations. + * @param message the supplement message. + */ + virtual void trigger(Kind kind, const char* message) = 0; + }; + /** + * Open modes. + */ + enum OpenMode { + OREADER = 1 << 0, ///< open as a reader + OWRITER = 1 << 1, ///< open as a writer + OCREATE = 1 << 2, ///< writer creating + OTRUNCATE = 1 << 3, ///< writer truncating + OAUTOTRAN = 1 << 4, ///< auto transaction + OAUTOSYNC = 1 << 5, ///< auto synchronization + ONOLOCK = 1 << 6, ///< open without locking + OTRYLOCK = 1 << 7, ///< lock without blocking + ONOREPAIR = 1 << 8 ///< open without auto repair + }; + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~BasicDB() { + _assert_(true); + } + /** + * Get the last happened error. + * @return the last happened error. + */ + virtual Error error() const = 0; + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + virtual void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) = 0; + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. BasicDB::OWRITER as a writer, BasicDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: BasicDB::OCREATE, + * which means it creates a new database if the file does not exist, BasicDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, BasicDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, BasicDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: BasicDB::ONOLOCK, which means it opens the database file without file locking, + * BasicDB::OTRYLOCK, which means locking is performed without blocking, File::ONOREPAIR, which + * means the database file is not repaired implicitly even if file destruction is detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the BasicDB::close method when it is no longer + * in use. It is not allowed for two or more database objects in the same process to keep + * their connections to the same database file at the same time. + */ + virtual bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) = 0; + /** + * Close the database file. + * @return true on success, or false on failure. + */ + virtual bool close() = 0; + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + virtual bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) = 0; + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + virtual bool iterate(Visitor *visitor, bool writable = true, + ProgressChecker* checker = NULL) = 0; + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + virtual bool scan_parallel(Visitor *visitor, size_t thnum, + ProgressChecker* checker = NULL) = 0; + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + virtual bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) = 0; + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + virtual bool occupy(bool writable = true, FileProcessor* proc = NULL) = 0; + /** + * Create a copy of the database file. + * @param dest the path of the destination file. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + */ + bool copy(const std::string& dest, ProgressChecker* checker = NULL) { + _assert_(true); + class FileProcessorImpl : public FileProcessor { + public: + explicit FileProcessorImpl(const std::string& dest, ProgressChecker* checker, + BasicDB* db) : + dest_(dest), checker_(checker), db_(db) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + File::Status sbuf; + if (!File::status(path, &sbuf)) return false; + if (sbuf.isdir) { + if (!File::make_directory(dest_)) return false; + bool err = false; + DirStream dir; + if (dir.open(path)) { + if (checker_ && !checker_->check("copy", "beginning", 0, -1)) { + db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + std::string name; + int64_t curcnt = 0; + while (!err && dir.read(&name)) { + const std::string& spath = path + File::PATHCHR + name; + const std::string& dpath = dest_ + File::PATHCHR + name; + int64_t dsiz; + char* dbuf = File::read_file(spath, &dsiz); + if (dbuf) { + if (!File::write_file(dpath, dbuf, dsiz)) err = true; + delete[] dbuf; + } else { + err = true; + } + curcnt++; + if (checker_ && !checker_->check("copy", "processing", curcnt, -1)) { + db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + break; + } + } + if (checker_ && !checker_->check("copy", "ending", -1, -1)) { + db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + if (!dir.close()) err = true; + } else { + err = true; + } + return !err; + } + std::ofstream ofs; + ofs.open(dest_.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + if (!ofs) return false; + bool err = false; + std::ifstream ifs; + ifs.open(path.c_str(), std::ios_base::in | std::ios_base::binary); + if (checker_ && !checker_->check("copy", "beginning", 0, size)) { + db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + if (ifs) { + char buf[IOBUFSIZ]; + int64_t curcnt = 0; + while (!err && !ifs.eof()) { + size_t n = ifs.read(buf, sizeof(buf)).gcount(); + if (n > 0) { + ofs.write(buf, n); + if (!ofs) { + err = true; + break; + } + } + curcnt += n; + if (checker_ && !checker_->check("copy", "processing", curcnt, size)) { + db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + break; + } + } + ifs.close(); + if (ifs.bad()) err = true; + } else { + err = true; + } + if (checker_ && !checker_->check("copy", "ending", -1, size)) { + db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + ofs.close(); + if (!ofs) err = true; + return !err; + } + const std::string& dest_; + ProgressChecker* checker_; + BasicDB* db_; + }; + FileProcessorImpl proc(dest, checker, this); + return synchronize(false, &proc, checker); + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + virtual bool begin_transaction(bool hard = false) = 0; + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + virtual bool begin_transaction_try(bool hard = false) = 0; + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + virtual bool end_transaction(bool commit = true) = 0; + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + virtual int64_t size() = 0; + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + virtual std::string path() = 0; + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + virtual bool status(std::map<std::string, std::string>* strmap) = 0; + /** + * Set the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the value is overwritten. + */ + bool set(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsiz) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + *sp = vsiz_; + return vbuf_; + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; + size_t vsiz_; + }; + VisitorImpl visitor(vbuf, vsiz); + if (!accept(kbuf, ksiz, &visitor, true)) return false; + return true; + } + /** + * Set the value of a record. + * @note Equal to the original DB::set method except that the parameters are std::string. + */ + bool set(const std::string& key, const std::string& value) { + _assert_(true); + return set(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Add a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the record is not modified and false is returned. + */ + bool add(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz) : + vbuf_(vbuf), vsiz_(vsiz), ok_(false) {} + bool ok() const { + return ok_; + } + private: + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + ok_ = true; + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; + size_t vsiz_; + bool ok_; + }; + VisitorImpl visitor(vbuf, vsiz); + if (!accept(kbuf, ksiz, &visitor, true)) return false; + if (!visitor.ok()) { + set_error(_KCCODELINE_, Error::DUPREC, "record duplication"); + return false; + } + return true; + } + /** + * Set the value of a record. + * @note Equal to the original DB::add method except that the parameters are std::string. + */ + bool add(const std::string& key, const std::string& value) { + _assert_(true); + return add(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Replace the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, no new record is created and false is returned. + * If the corresponding record exists, the value is modified. + */ + bool replace(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz) : + vbuf_(vbuf), vsiz_(vsiz), ok_(false) {} + bool ok() const { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + ok_ = true; + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; + size_t vsiz_; + bool ok_; + }; + VisitorImpl visitor(vbuf, vsiz); + if (!accept(kbuf, ksiz, &visitor, true)) return false; + if (!visitor.ok()) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Replace the value of a record. + * @note Equal to the original DB::replace method except that the parameters are std::string. + */ + bool replace(const std::string& key, const std::string& value) { + _assert_(true); + return replace(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Append the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the given value is appended at the end of the existing value. + */ + bool append(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz) : + vbuf_(vbuf), vsiz_(vsiz), nbuf_(NULL) {} + ~VisitorImpl() { + if (nbuf_) delete[] nbuf_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + size_t nsiz = vsiz + vsiz_; + nbuf_ = new char[nsiz]; + std::memcpy(nbuf_, vbuf, vsiz); + std::memcpy(nbuf_ + vsiz, vbuf_, vsiz_); + *sp = nsiz; + return nbuf_; + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; + size_t vsiz_; + char* nbuf_; + }; + VisitorImpl visitor(vbuf, vsiz); + if (!accept(kbuf, ksiz, &visitor, true)) return false; + return true; + } + /** + * Set the value of a record. + * @note Equal to the original DB::append method except that the parameters are std::string. + */ + bool append(const std::string& key, const std::string& value) { + _assert_(true); + return append(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Add a number to the numeric value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param num the additional number. + * @param orig the origin number if no record corresponds to the key. If it is INT64MIN and + * no record corresponds, this function fails. If it is INT64MAX, the value is set as the + * additional number regardless of the current value. + * @return the result value, or kyotocabinet::INT64MIN on failure. + * @note The value is serialized as an 8-byte binary integer in big-endian order, not a decimal + * string. If existing value is not 8-byte, this function fails. + */ + int64_t increment(const char* kbuf, size_t ksiz, int64_t num, int64_t orig = 0) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(int64_t num, int64_t orig) : num_(num), orig_(orig), big_(0) {} + int64_t num() { + return num_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (vsiz != sizeof(num_)) { + num_ = INT64MIN; + return NOP; + } + int64_t onum; + if (orig_ == INT64MAX) { + onum = 0; + } else { + std::memcpy(&onum, vbuf, vsiz); + onum = ntoh64(onum); + if (num_ == 0) { + num_ = onum; + return NOP; + } + } + num_ += onum; + big_ = hton64(num_); + *sp = sizeof(big_); + return (const char*)&big_; + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + if (orig_ == INT64MIN) { + num_ = INT64MIN; + return NOP; + } + if (orig_ != INT64MAX) num_ += orig_; + big_ = hton64(num_); + *sp = sizeof(big_); + return (const char*)&big_; + } + int64_t num_; + int64_t orig_; + uint64_t big_; + }; + VisitorImpl visitor(num, orig); + if (!accept(kbuf, ksiz, &visitor, num != 0 || orig != INT64MIN)) return INT64MIN; + num = visitor.num(); + if (num == INT64MIN) { + set_error(_KCCODELINE_, Error::LOGIC, "logical inconsistency"); + return num; + } + return num; + } + /** + * Add a number to the numeric value of a record. + * @note Equal to the original DB::increment method except that the parameter is std::string. + */ + int64_t increment(const std::string& key, int64_t num, int64_t orig = 0) { + _assert_(true); + return increment(key.c_str(), key.size(), num, orig); + } + /** + * Add a number to the numeric double value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param num the additional number. + * @param orig the origin number if no record corresponds to the key. If it is negative + * infinity and no record corresponds, this function fails. If it is positive infinity, the + * value is set as the additional number regardless of the current value. + * @return the result value, or Not-a-number on failure. + * @note The value is serialized as an 16-byte binary fixed-point number in big-endian order, + * not a decimal string. If existing value is not 16-byte, this function fails. + */ + double increment_double(const char* kbuf, size_t ksiz, double num, double orig = 0) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(double num, double orig) : + DECUNIT(1000000000000000LL), num_(num), orig_(orig), buf_() {} + double num() { + return num_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (vsiz != sizeof(buf_)) { + num_ = nan(); + return NOP; + } + int64_t linteg, lfract; + if (chkinf(orig_) && orig_ >= 0) { + linteg = 0; + lfract = 0; + } else { + std::memcpy(&linteg, vbuf, sizeof(linteg)); + linteg = ntoh64(linteg); + std::memcpy(&lfract, vbuf + sizeof(linteg), sizeof(lfract)); + lfract = ntoh64(lfract); + } + if (lfract == INT64MIN && linteg == INT64MIN) { + num_ = nan(); + return NOP; + } else if (linteg == INT64MAX) { + num_ = HUGE_VAL; + return NOP; + } else if (linteg == INT64MIN) { + num_ = -HUGE_VAL; + return NOP; + } + if (num_ == 0.0 && !(chkinf(orig_) && orig_ >= 0)) { + num_ = linteg + (double)lfract / DECUNIT; + return NOP; + } + long double dinteg; + long double dfract = std::modfl(num_, &dinteg); + if (chknan(dinteg)) { + linteg = INT64MIN; + lfract = INT64MIN; + num_ = nan(); + } else if (chkinf(dinteg)) { + linteg = dinteg > 0 ? INT64MAX : INT64MIN; + lfract = 0; + num_ = dinteg; + } else { + linteg += (int64_t)dinteg; + lfract += (int64_t)(dfract * DECUNIT); + if (lfract >= DECUNIT) { + linteg += 1; + lfract -= DECUNIT; + } + num_ = linteg + (double)lfract / DECUNIT; + } + linteg = hton64(linteg); + std::memcpy(buf_, &linteg, sizeof(linteg)); + lfract = hton64(lfract); + std::memcpy(buf_ + sizeof(linteg), &lfract, sizeof(lfract)); + *sp = sizeof(buf_); + return buf_; + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + if (chknan(orig_) || (chkinf(orig_) && orig_ < 0)) { + num_ = nan(); + return NOP; + } + if (!chkinf(orig_)) num_ += orig_; + long double dinteg; + long double dfract = std::modfl(num_, &dinteg); + int64_t linteg, lfract; + if (chknan(dinteg)) { + linteg = INT64MIN; + lfract = INT64MIN; + } else if (chkinf(dinteg)) { + linteg = dinteg > 0 ? INT64MAX : INT64MIN; + lfract = 0; + } else { + linteg = (int64_t)dinteg; + lfract = (int64_t)(dfract * DECUNIT); + } + linteg = hton64(linteg); + std::memcpy(buf_, &linteg, sizeof(linteg)); + lfract = hton64(lfract); + std::memcpy(buf_ + sizeof(linteg), &lfract, sizeof(lfract)); + *sp = sizeof(buf_); + return buf_; + } + const int64_t DECUNIT; + double num_; + double orig_; + char buf_[sizeof(int64_t)*2]; + }; + VisitorImpl visitor(num, orig); + if (!accept(kbuf, ksiz, &visitor, true)) return nan(); + num = visitor.num(); + if (chknan(num)) { + set_error(_KCCODELINE_, Error::LOGIC, "logical inconsistency"); + return nan(); + } + return num; + } + /** + * Add a number to the numeric double value of a record. + * @note Equal to the original DB::increment_double method except that the parameter is + * std::string. + */ + double increment_double(const std::string& key, double num, double orig) { + _assert_(true); + return increment_double(key.c_str(), key.size(), num, orig); + } + /** + * Perform compare-and-swap. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param ovbuf the pointer to the old value region. NULL means that no record corresponds. + * @param ovsiz the size of the old value region. + * @param nvbuf the pointer to the new value region. NULL means that the record is removed. + * @param nvsiz the size of new old value region. + * @return true on success, or false on failure. + */ + bool cas(const char* kbuf, size_t ksiz, + const char* ovbuf, size_t ovsiz, const char* nvbuf, size_t nvsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(const char* ovbuf, size_t ovsiz, const char* nvbuf, size_t nvsiz) : + ovbuf_(ovbuf), ovsiz_(ovsiz), nvbuf_(nvbuf), nvsiz_(nvsiz), ok_(false) {} + bool ok() const { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (!ovbuf_ || vsiz != ovsiz_ || std::memcmp(vbuf, ovbuf_, vsiz)) return NOP; + ok_ = true; + if (!nvbuf_) return REMOVE; + *sp = nvsiz_; + return nvbuf_; + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + if (ovbuf_) return NOP; + ok_ = true; + if (!nvbuf_) return NOP; + *sp = nvsiz_; + return nvbuf_; + } + const char* ovbuf_; + size_t ovsiz_; + const char* nvbuf_; + size_t nvsiz_; + bool ok_; + }; + VisitorImpl visitor(ovbuf, ovsiz, nvbuf, nvsiz); + if (!accept(kbuf, ksiz, &visitor, true)) return false; + if (!visitor.ok()) { + set_error(_KCCODELINE_, Error::LOGIC, "status conflict"); + return false; + } + return true; + } + /** + * Perform compare-and-swap. + * @note Equal to the original DB::cas method except that the parameters are std::string. + */ + bool cas(const std::string& key, + const std::string& ovalue, const std::string& nvalue) { + _assert_(true); + return cas(key.c_str(), key.size(), + ovalue.c_str(), ovalue.size(), nvalue.c_str(), nvalue.size()); + } + /** + * Remove a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. + */ + bool remove(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : ok_(false) {} + bool ok() const { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + ok_ = true; + return REMOVE; + } + bool ok_; + }; + VisitorImpl visitor; + if (!accept(kbuf, ksiz, &visitor, true)) return false; + if (!visitor.ok()) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Remove a record. + * @note Equal to the original DB::remove method except that the parameter is std::string. + */ + bool remove(const std::string& key) { + _assert_(true); + return remove(key.c_str(), key.size()); + } + /** + * Retrieve the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + * @note If no record corresponds to the key, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + char* get(const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && sp); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : vbuf_(NULL), vsiz_(0) {} + char* pop(size_t* sp) { + *sp = vsiz_; + return vbuf_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + vbuf_ = new char[vsiz+1]; + std::memcpy(vbuf_, vbuf, vsiz); + vbuf_[vsiz] = '\0'; + vsiz_ = vsiz; + return NOP; + } + char* vbuf_; + size_t vsiz_; + }; + VisitorImpl visitor; + if (!accept(kbuf, ksiz, &visitor, false)) { + *sp = 0; + return NULL; + } + size_t vsiz; + char* vbuf = visitor.pop(&vsiz); + if (!vbuf) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + *sp = 0; + return NULL; + } + *sp = vsiz; + return vbuf; + } + /** + * Retrieve the value of a record. + * @note Equal to the original DB::get method except that the first parameters is the key + * string and the second parameter is a string to contain the result and the return value is + * bool for success. + */ + bool get(const std::string& key, std::string* value) { + _assert_(value); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(std::string* value) : value_(value), ok_(false) {} + bool ok() { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + value_->clear(); + value_->append(vbuf, vsiz); + ok_ = true; + return NOP; + } + std::string* value_; + bool ok_; + }; + VisitorImpl visitor(value); + if (!accept(key.data(), key.size(), &visitor, false)) return false; + if (!visitor.ok()) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Retrieve the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the buffer into which the value of the corresponding record is + * written. + * @param max the size of the buffer. + * @return the size of the value, or -1 on failure. + */ + int32_t get(const char* kbuf, size_t ksiz, char* vbuf, size_t max) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(char* vbuf, size_t max) : vbuf_(vbuf), max_(max), vsiz_(-1) {} + int32_t vsiz() { + return vsiz_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + vsiz_ = vsiz; + size_t max = vsiz < max_ ? vsiz : max_; + std::memcpy(vbuf_, vbuf, max); + return NOP; + } + char* vbuf_; + size_t max_; + int32_t vsiz_; + }; + VisitorImpl visitor(vbuf, max); + if (!accept(kbuf, ksiz, &visitor, false)) return -1; + int32_t vsiz = visitor.vsiz(); + if (vsiz < 0) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + return -1; + } + return vsiz; + } + /** + * Check the existence of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the size of the value, or -1 on failure. + */ + int32_t check(const char* kbuf, size_t ksiz) { + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : vsiz_(-1) {} + int32_t vsiz() { + return vsiz_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + vsiz_ = vsiz; + return NOP; + } + size_t vsiz_; + }; + VisitorImpl visitor; + if (!accept(kbuf, ksiz, &visitor, false)) return -1; + int32_t vsiz = visitor.vsiz(); + if (vsiz < 0) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + return -1; + } + return vsiz; + } + /** + * Check the existence of a record. + * @note Equal to the original DB::check method except that the parameter is std::string. + */ + int32_t check(const std::string& key) { + return check(key.data(), key.size()); + } + /** + * Retrieve the value of a record and remove it atomically. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + * @note If no record corresponds to the key, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + char* seize(const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && sp); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : vbuf_(NULL), vsiz_(0) {} + char* pop(size_t* sp) { + *sp = vsiz_; + return vbuf_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + vbuf_ = new char[vsiz+1]; + std::memcpy(vbuf_, vbuf, vsiz); + vbuf_[vsiz] = '\0'; + vsiz_ = vsiz; + return REMOVE; + } + char* vbuf_; + size_t vsiz_; + }; + VisitorImpl visitor; + if (!accept(kbuf, ksiz, &visitor, true)) { + *sp = 0; + return NULL; + } + size_t vsiz; + char* vbuf = visitor.pop(&vsiz); + if (!vbuf) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + *sp = 0; + return NULL; + } + *sp = vsiz; + return vbuf; + } + /** + * Retrieve the value of a record and remove it atomically. + * @note Equal to the original DB::seize method except that the first parameters is the key + * string and the second parameter is a string to contain the result and the return value is + * bool for success. + */ + bool seize(const std::string& key, std::string* value) { + _assert_(value); + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(std::string* value) : value_(value), ok_(false) {} + bool ok() { + return ok_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + value_->clear(); + value_->append(vbuf, vsiz); + ok_ = true; + return REMOVE; + } + std::string* value_; + bool ok_; + }; + VisitorImpl visitor(value); + if (!accept(key.data(), key.size(), &visitor, true)) return false; + if (!visitor.ok()) { + set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Store records at once. + * @param recs the records to store. + * @param atomic true to perform all operations atomically, or false for non-atomic operations. + * @return the number of stored records, or -1 on failure. + */ + int64_t set_bulk(const std::map<std::string, std::string>& recs, bool atomic = true) { + _assert_(true); + if (atomic) { + std::vector<std::string> keys; + keys.reserve(recs.size()); + std::map<std::string, std::string>::const_iterator rit = recs.begin(); + std::map<std::string, std::string>::const_iterator ritend = recs.end(); + while (rit != ritend) { + keys.push_back(rit->first); + ++rit; + } + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(const std::map<std::string, std::string>& recs) : recs_(recs) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + std::map<std::string, std::string>::const_iterator rit = + recs_.find(std::string(kbuf, ksiz)); + if (rit == recs_.end()) return NOP; + *sp = rit->second.size(); + return rit->second.data(); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + std::map<std::string, std::string>::const_iterator rit = + recs_.find(std::string(kbuf, ksiz)); + if (rit == recs_.end()) return NOP; + *sp = rit->second.size(); + return rit->second.data(); + } + const std::map<std::string, std::string>& recs_; + }; + VisitorImpl visitor(recs); + if (!accept_bulk(keys, &visitor, true)) return -1; + return keys.size(); + } + std::map<std::string, std::string>::const_iterator rit = recs.begin(); + std::map<std::string, std::string>::const_iterator ritend = recs.end(); + while (rit != ritend) { + if (!set(rit->first.data(), rit->first.size(), rit->second.data(), rit->second.size())) + return -1; + ++rit; + } + return recs.size(); + } + /** + * Remove records at once. + * @param keys the keys of the records to remove. + * @param atomic true to perform all operations atomically, or false for non-atomic operations. + * @return the number of removed records, or -1 on failure. + */ + int64_t remove_bulk(const std::vector<std::string>& keys, bool atomic = true) { + _assert_(true); + if (atomic) { + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl() : cnt_(0) {} + int64_t cnt() const { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + return REMOVE; + } + int64_t cnt_; + }; + VisitorImpl visitor; + if (!accept_bulk(keys, &visitor, true)) return -1; + return visitor.cnt(); + } + int64_t cnt = 0; + std::vector<std::string>::const_iterator kit = keys.begin(); + std::vector<std::string>::const_iterator kitend = keys.end(); + while (kit != kitend) { + if (remove(kit->data(), kit->size())) { + cnt++; + } else if (error() != Error::NOREC) { + return -1; + } + ++kit; + } + return cnt; + } + /** + * Retrieve records at once. + * @param keys the keys of the records to retrieve. + * @param recs a string map to contain the retrieved records. + * @param atomic true to perform all operations atomically, or false for non-atomic operations. + * @return the number of retrieved records, or -1 on failure. + */ + int64_t get_bulk(const std::vector<std::string>& keys, + std::map<std::string, std::string>* recs, bool atomic = true) { + _assert_(recs); + if (atomic) { + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(std::map<std::string, std::string>* recs) : recs_(recs) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + (*recs_)[std::string(kbuf, ksiz)] = std::string(vbuf, vsiz); + return NOP; + } + std::map<std::string, std::string>* recs_; + }; + VisitorImpl visitor(recs); + if (!accept_bulk(keys, &visitor, false)) return -1; + return recs->size(); + } + std::vector<std::string>::const_iterator kit = keys.begin(); + std::vector<std::string>::const_iterator kitend = keys.end(); + while (kit != kitend) { + size_t vsiz; + const char* vbuf = get(kit->data(), kit->size(), &vsiz); + if (vbuf) { + (*recs)[*kit] = std::string(vbuf, vsiz); + delete[] vbuf; + } else if (error() != Error::NOREC) { + return -1; + } + ++kit; + } + return recs->size(); + } + /** + * Dump records into a data stream. + * @param dest the destination stream. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + */ + bool dump_snapshot(std::ostream* dest, ProgressChecker* checker = NULL) { + _assert_(dest); + if (dest->fail()) { + set_error(_KCCODELINE_, Error::INVALID, "invalid stream"); + return false; + } + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(std::ostream* dest) : dest_(dest), stack_() {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + char* wp = stack_; + *(wp++) = 0x00; + wp += writevarnum(wp, ksiz); + wp += writevarnum(wp, vsiz); + dest_->write(stack_, wp - stack_); + dest_->write(kbuf, ksiz); + dest_->write(vbuf, vsiz); + return NOP; + } + std::ostream* dest_; + char stack_[NUMBUFSIZ*2]; + }; + VisitorImpl visitor(dest); + bool err = false; + dest->write(KCDBSSMAGICDATA, sizeof(KCDBSSMAGICDATA)); + if (iterate(&visitor, false, checker)) { + unsigned char c = 0xff; + dest->write((char*)&c, 1); + if (dest->fail()) { + set_error(_KCCODELINE_, Error::SYSTEM, "stream output error"); + err = true; + } + } else { + err = true; + } + return !err; + } + /** + * Dump records into a file. + * @param dest the path of the destination file. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + */ + bool dump_snapshot(const std::string& dest, ProgressChecker* checker = NULL) { + _assert_(true); + std::ofstream ofs; + ofs.open(dest.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + if (!ofs) { + set_error(_KCCODELINE_, Error::NOREPOS, "open failed"); + return false; + } + bool err = false; + if (!dump_snapshot(&ofs, checker)) err = true; + ofs.close(); + if (!ofs) { + set_error(_KCCODELINE_, Error::SYSTEM, "close failed"); + err = true; + } + return !err; + } + /** + * Load records from a data stream. + * @param src the source stream. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + */ + bool load_snapshot(std::istream* src, ProgressChecker* checker = NULL) { + _assert_(src); + if (src->fail()) { + set_error(_KCCODELINE_, Error::INVALID, "invalid stream"); + return false; + } + char buf[IOBUFSIZ]; + src->read(buf, sizeof(KCDBSSMAGICDATA)); + if (src->fail()) { + set_error(_KCCODELINE_, Error::SYSTEM, "stream input error"); + return false; + } + if (std::memcmp(buf, KCDBSSMAGICDATA, sizeof(KCDBSSMAGICDATA))) { + set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of input stream"); + return false; + } + bool err = false; + if (checker && !checker->check("load_snapshot", "beginning", 0, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + int64_t curcnt = 0; + while (!err) { + int32_t c = src->get(); + if (src->fail()) { + set_error(_KCCODELINE_, Error::SYSTEM, "stream input error"); + err = true; + break; + } + if (c == 0xff) break; + if (c == 0x00) { + size_t ksiz = 0; + do { + c = src->get(); + ksiz = (ksiz << 7) + (c & 0x7f); + } while (c >= 0x80); + size_t vsiz = 0; + do { + c = src->get(); + vsiz = (vsiz << 7) + (c & 0x7f); + } while (c >= 0x80); + size_t rsiz = ksiz + vsiz; + char* rbuf = rsiz > sizeof(buf) ? new char[rsiz] : buf; + src->read(rbuf, ksiz + vsiz); + if (src->fail()) { + set_error(_KCCODELINE_, Error::SYSTEM, "stream input error"); + err = true; + if (rbuf != buf) delete[] rbuf; + break; + } + if (!set(rbuf, ksiz, rbuf + ksiz, vsiz)) { + err = true; + if (rbuf != buf) delete[] rbuf; + break; + } + if (rbuf != buf) delete[] rbuf; + } else { + set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of input stream"); + err = true; + break; + } + curcnt++; + if (checker && !checker->check("load_snapshot", "processing", curcnt, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + break; + } + } + if (checker && !checker->check("load_snapshot", "ending", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + return !err; + } + /** + * Load records from a file. + * @param src the path of the source file. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + */ + bool load_snapshot(const std::string& src, ProgressChecker* checker = NULL) { + _assert_(true); + std::ifstream ifs; + ifs.open(src.c_str(), std::ios_base::in | std::ios_base::binary); + if (!ifs) { + set_error(_KCCODELINE_, Error::NOREPOS, "open failed"); + return false; + } + bool err = false; + if (!load_snapshot(&ifs, checker)) err = true; + ifs.close(); + if (ifs.bad()) { + set_error(_KCCODELINE_, Error::SYSTEM, "close failed"); + return false; + } + return !err; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + virtual Cursor* cursor() = 0; + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + virtual void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) = 0; + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + virtual bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) = 0; + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + virtual bool tune_meta_trigger(MetaTrigger* trigger) = 0; + /** + * Get the class name of a database type. + * @param type the database type. + * @return the string of the type name. + */ + static const char* typecname(uint32_t type) { + _assert_(true); + switch (type) { + case TYPEVOID: return "void"; + case TYPEPHASH: return "ProtoHashDB"; + case TYPEPTREE: return "ProtoTreeDB"; + case TYPESTASH: return "StashDB"; + case TYPECACHE: return "CacheDB"; + case TYPEGRASS: return "GrassDB"; + case TYPEHASH: return "HashDB"; + case TYPETREE: return "TreeDB"; + case TYPEDIR: return "DirDB"; + case TYPEFOREST: return "ForestDB"; + case TYPEMISC: return "misc"; + } + return "unknown"; + } + /** + * Get the description string of a database type. + * @param type the database type. + * @return the string of the type name. + */ + static const char* typestring(uint32_t type) { + _assert_(true); + switch (type) { + case TYPEVOID: return "void"; + case TYPEPHASH: return "prototype hash database"; + case TYPEPTREE: return "prototype tree database"; + case TYPESTASH: return "stash database"; + case TYPECACHE: return "cache hash database"; + case TYPEGRASS: return "cache tree database"; + case TYPEHASH: return "file hash database"; + case TYPETREE: return "file tree database"; + case TYPEDIR: return "directory hash database"; + case TYPEFOREST: return "directory tree database"; + case TYPEMISC: return "miscellaneous database"; + } + return "unknown"; + } +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdbext.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcdbext.cc new file mode 100644 index 0000000000..a9dfa547b6 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdbext.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Database extension + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcdbext.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdbext.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcdbext.h new file mode 100644 index 0000000000..001c09a1d4 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdbext.h @@ -0,0 +1,1690 @@ +/************************************************************************************************* + * Database extension + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCDBEXT_H // duplication check +#define _KCDBEXT_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> +#include <kcplantdb.h> +#include <kcprotodb.h> +#include <kcstashdb.h> +#include <kccachedb.h> +#include <kchashdb.h> +#include <kcdirdb.h> +#include <kcpolydb.h> + +namespace kyotocabinet { // common namespace + + +/** + * MapReduce framework. + * @note Although this framework is not distributed or concurrent, it is useful for aggregate + * calculation with less CPU loading and less memory usage. + */ +class MapReduce { + public: + class ValueIterator; + private: + class FlushThread; + class ReduceTaskQueue; + class MapVisitor; + struct MergeLine; + /** An alias of vector of loaded values. */ + typedef std::vector<std::string> Values; + /** The default number of temporary databases. */ + static const size_t DEFDBNUM = 8; + /** The maxinum number of temporary databases. */ + static const size_t MAXDBNUM = 256; + /** The default cache limit. */ + static const int64_t DEFCLIM = 512LL << 20; + /** The default cache bucket numer. */ + static const int64_t DEFCBNUM = 1048583LL; + /** The bucket number of temprary databases. */ + static const int64_t DBBNUM = 512LL << 10; + /** The page size of temprary databases. */ + static const int32_t DBPSIZ = 32768; + /** The mapped size of temprary databases. */ + static const int64_t DBMSIZ = 516LL * 4096; + /** The page cache capacity of temprary databases. */ + static const int64_t DBPCCAP = 16LL << 20; + /** The default number of threads in parallel mode. */ + static const size_t DEFTHNUM = 8; + /** The number of slots of the record lock. */ + static const int32_t RLOCKSLOT = 256; + public: + /** + * Value iterator for the reducer. + */ + class ValueIterator { + friend class MapReduce; + public: + /** + * Get the next value. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the next value region, or NULL if no value remains. + */ + const char* next(size_t* sp) { + _assert_(sp); + if (!vptr_) { + if (vit_ == vend_) return NULL; + vptr_ = vit_->data(); + vsiz_ = vit_->size(); + vit_++; + } + uint64_t vsiz; + size_t step = readvarnum(vptr_, vsiz_, &vsiz); + vptr_ += step; + vsiz_ -= step; + const char* vbuf = vptr_; + *sp = vsiz; + vptr_ += vsiz; + vsiz_ -= vsiz; + if (vsiz_ < 1) vptr_ = NULL; + return vbuf; + } + private: + /** + * Default constructor. + */ + explicit ValueIterator(Values::const_iterator vit, Values::const_iterator vend) : + vit_(vit), vend_(vend), vptr_(NULL), vsiz_(0) { + _assert_(true); + } + /** + * Destructor. + */ + ~ValueIterator() { + _assert_(true); + } + /** Dummy constructor to forbid the use. */ + ValueIterator(const ValueIterator&); + /** Dummy Operator to forbid the use. */ + ValueIterator& operator =(const ValueIterator&); + /** The current iterator of loaded values. */ + Values::const_iterator vit_; + /** The ending iterator of loaded values. */ + Values::const_iterator vend_; + /** The pointer of the current value. */ + const char* vptr_; + /** The size of the current value. */ + size_t vsiz_; + }; + /** + * Execution options. + */ + enum Option { + XNOLOCK = 1 << 0, ///< avoid locking against update operations + XPARAMAP = 1 << 1, ///< run mappers in parallel + XPARARED = 1 << 2, ///< run reducers in parallel + XPARAFLS = 1 << 3, ///< run cache flushers in parallel + XNOCOMP = 1 << 8 ///< avoid compression of temporary databases + }; + /** + * Default constructor. + */ + explicit MapReduce() : + db_(NULL), rcomp_(NULL), tmpdbs_(NULL), dbnum_(DEFDBNUM), dbclock_(0), + mapthnum_(DEFTHNUM), redthnum_(DEFTHNUM), flsthnum_(DEFTHNUM), + cache_(NULL), csiz_(0), clim_(DEFCLIM), cbnum_(DEFCBNUM), flsths_(NULL), + redtasks_(NULL), redaborted_(false), rlocks_(NULL) { + _assert_(true); + } + /** + * Destructor. + */ + virtual ~MapReduce() { + _assert_(true); + } + /** + * Map a record data. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note This function can call the MapReduce::emit method to emit a record. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + virtual bool map(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0; + /** + * Reduce a record data. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param iter the iterator to get the values. + * @return true on success, or false on failure. + * @note To avoid deadlock, any explicit database operation must not be performed in this + * function. + */ + virtual bool reduce(const char* kbuf, size_t ksiz, ValueIterator* iter) = 0; + /** + * Preprocess the map operations. + * @return true on success, or false on failure. + * @note This function can call the MapReduce::emit method to emit a record. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + virtual bool preprocess() { + _assert_(true); + return true; + } + /** + * Mediate between the map and the reduce phases. + * @return true on success, or false on failure. + * @note This function can call the MapReduce::emit method to emit a record. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + virtual bool midprocess() { + _assert_(true); + return true; + } + /** + * Postprocess the reduce operations. + * @return true on success, or false on failure. + * @note To avoid deadlock, any explicit database operation must not be performed in this + * function. + */ + virtual bool postprocess() { + _assert_(true); + return true; + } + /** + * Process a log message. + * @param name the name of the event. + * @param message a supplement message. + * @return true on success, or false on failure. + */ + virtual bool log(const char* name, const char* message) { + _assert_(name && message); + return true; + } + /** + * Execute the MapReduce process about a database. + * @param db the source database. + * @param tmppath the path of a directory for the temporary data storage. If it is an empty + * string, temporary data are handled on memory. + * @param opts the optional features by bitwise-or: MapReduce::XNOLOCK to avoid locking + * against update operations by other threads, MapReduce::XPARAMAP to run the mapper in + * parallel, MapReduce::XPARARED to run the reducer in parallel, MapReduce::XNOCOMP to avoid + * compression of temporary databases. + * @return true on success, or false on failure. + */ + bool execute(BasicDB* db, const std::string& tmppath = "", uint32_t opts = 0) { + int64_t count = db->count(); + if (count < 0) { + if (db->error() != BasicDB::Error::NOIMPL) return false; + count = 0; + } + bool err = false; + double stime, etime; + db_ = db; + rcomp_ = LEXICALCOMP; + BasicDB* idb = db; + if (typeid(*db) == typeid(PolyDB)) { + PolyDB* pdb = (PolyDB*)idb; + idb = pdb->reveal_inner_db(); + } + const std::type_info& info = typeid(*idb); + if (info == typeid(GrassDB)) { + GrassDB* gdb = (GrassDB*)idb; + rcomp_ = gdb->rcomp(); + } else if (info == typeid(TreeDB)) { + TreeDB* tdb = (TreeDB*)idb; + rcomp_ = tdb->rcomp(); + } else if (info == typeid(ForestDB)) { + ForestDB* fdb = (ForestDB*)idb; + rcomp_ = fdb->rcomp(); + } + tmpdbs_ = new BasicDB*[dbnum_]; + if (tmppath.empty()) { + if (!logf("prepare", "started to open temporary databases on memory")) err = true; + stime = time(); + for (size_t i = 0; i < dbnum_; i++) { + GrassDB* gdb = new GrassDB; + int32_t myopts = 0; + if (!(opts & XNOCOMP)) myopts |= GrassDB::TCOMPRESS; + gdb->tune_options(myopts); + gdb->tune_buckets(DBBNUM / 2); + gdb->tune_page(DBPSIZ); + gdb->tune_page_cache(DBPCCAP); + gdb->tune_comparator(rcomp_); + gdb->open("%", GrassDB::OWRITER | GrassDB::OCREATE | GrassDB::OTRUNCATE); + tmpdbs_[i] = gdb; + } + etime = time(); + if (!logf("prepare", "opening temporary databases finished: time=%.6f", etime - stime)) + err = true; + if (err) { + delete[] tmpdbs_; + return false; + } + } else { + File::Status sbuf; + if (!File::status(tmppath, &sbuf) || !sbuf.isdir) { + db->set_error(_KCCODELINE_, BasicDB::Error::NOREPOS, "no such directory"); + delete[] tmpdbs_; + return false; + } + if (!logf("prepare", "started to open temporary databases under %s", tmppath.c_str())) + err = true; + stime = time(); + uint32_t pid = getpid() & UINT16MAX; + uint32_t tid = Thread::hash() & UINT16MAX; + uint32_t ts = time() * 1000; + for (size_t i = 0; i < dbnum_; i++) { + std::string childpath = + strprintf("%s%cmr-%04x-%04x-%08x-%03d%ckct", + tmppath.c_str(), File::PATHCHR, pid, tid, ts, (int)(i + 1), File::EXTCHR); + TreeDB* tdb = new TreeDB; + int32_t myopts = TreeDB::TSMALL | TreeDB::TLINEAR; + if (!(opts & XNOCOMP)) myopts |= TreeDB::TCOMPRESS; + tdb->tune_options(myopts); + tdb->tune_buckets(DBBNUM); + tdb->tune_page(DBPSIZ); + tdb->tune_map(DBMSIZ); + tdb->tune_page_cache(DBPCCAP); + tdb->tune_comparator(rcomp_); + if (!tdb->open(childpath, TreeDB::OWRITER | TreeDB::OCREATE | TreeDB::OTRUNCATE)) { + const BasicDB::Error& e = tdb->error(); + db->set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + tmpdbs_[i] = tdb; + } + etime = time(); + if (!logf("prepare", "opening temporary databases finished: time=%.6f", etime - stime)) + err = true; + if (err) { + for (size_t i = 0; i < dbnum_; i++) { + delete tmpdbs_[i]; + } + delete[] tmpdbs_; + return false; + } + } + if (opts & XPARARED) redtasks_ = new ReduceTaskQueue; + if (opts & XPARAFLS) flsths_ = new std::deque<FlushThread*>; + if (opts & XNOLOCK) { + MapChecker mapchecker; + MapVisitor mapvisitor(this, &mapchecker, count); + mapvisitor.visit_before(); + if (!err) { + BasicDB::Cursor* cur = db->cursor(); + if (!cur->jump() && cur->error() != BasicDB::Error::NOREC) err = true; + while (!err) { + if (!cur->accept(&mapvisitor, false, true)) { + if (cur->error() != BasicDB::Error::NOREC) err = true; + break; + } + } + delete cur; + } + if (mapvisitor.error()) { + db_->set_error(_KCCODELINE_, BasicDB::Error::LOGIC, "mapper failed"); + err = true; + } + mapvisitor.visit_after(); + } else if (opts & XPARAMAP) { + MapChecker mapchecker; + MapVisitor mapvisitor(this, &mapchecker, count); + rlocks_ = new SlottedMutex(RLOCKSLOT); + if (!err && !db->scan_parallel(&mapvisitor, mapthnum_, &mapchecker)) { + db_->set_error(_KCCODELINE_, BasicDB::Error::LOGIC, "mapper failed"); + err = true; + } + delete rlocks_; + rlocks_ = NULL; + if (mapvisitor.error()) err = true; + } else { + MapChecker mapchecker; + MapVisitor mapvisitor(this, &mapchecker, count); + if (!err && !db->iterate(&mapvisitor, false, &mapchecker)) err = true; + if (mapvisitor.error()) { + db_->set_error(_KCCODELINE_, BasicDB::Error::LOGIC, "mapper failed"); + err = true; + } + } + if (flsths_) { + delete flsths_; + flsths_ = NULL; + } + if (redtasks_) { + delete redtasks_; + redtasks_ = NULL; + } + if (!logf("clean", "closing the temporary databases")) err = true; + stime = time(); + for (size_t i = 0; i < dbnum_; i++) { + const std::string& path = tmpdbs_[i]->path(); + if (!tmpdbs_[i]->clear()) { + const BasicDB::Error& e = tmpdbs_[i]->error(); + db->set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + if (!tmpdbs_[i]->close()) { + const BasicDB::Error& e = tmpdbs_[i]->error(); + db->set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + if (!tmppath.empty()) File::remove(path); + delete tmpdbs_[i]; + } + etime = time(); + if (!logf("clean", "closing the temporary databases finished: time=%.6f", + etime - stime)) err = true; + delete[] tmpdbs_; + return !err; + } + /** + * Set the storage configurations. + * @param dbnum the number of temporary databases. + * @param clim the limit size of the internal cache. + * @param cbnum the bucket number of the internal cache. + */ + void tune_storage(int32_t dbnum, int64_t clim, int64_t cbnum) { + _assert_(true); + dbnum_ = dbnum > 0 ? dbnum : DEFDBNUM; + if (dbnum_ > MAXDBNUM) dbnum_ = MAXDBNUM; + clim_ = clim > 0 ? clim : DEFCLIM; + cbnum_ = cbnum > 0 ? cbnum : DEFCBNUM; + if (cbnum_ > INT16MAX) cbnum_ = nearbyprime(cbnum_); + } + /** + * Set the thread configurations. + * @param mapthnum the number of threads for the mapper. + * @param redthnum the number of threads for the reducer. + * @param flsthnum the number of threads for the internal flusher. + */ + void tune_thread(int32_t mapthnum, int32_t redthnum, int32_t flsthnum) { + _assert_(true); + mapthnum_ = mapthnum > 0 ? mapthnum : DEFTHNUM; + redthnum_ = redthnum > 0 ? redthnum : DEFTHNUM; + flsthnum_ = flsthnum > 0 ? flsthnum : DEFTHNUM; + } + protected: + /** + * Emit a record from the mapper. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + */ + bool emit(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + bool err = false; + size_t rsiz = sizevarnum(vsiz) + vsiz; + char stack[NUMBUFSIZ*4]; + char* rbuf = rsiz > sizeof(stack) ? new char[rsiz] : stack; + char* wp = rbuf; + wp += writevarnum(rbuf, vsiz); + std::memcpy(wp, vbuf, vsiz); + if (rlocks_) { + size_t bidx = TinyHashMap::hash_record(kbuf, ksiz) % cbnum_; + size_t lidx = bidx % RLOCKSLOT; + rlocks_->lock(lidx); + cache_->append(kbuf, ksiz, rbuf, rsiz); + rlocks_->unlock(lidx); + } else { + cache_->append(kbuf, ksiz, rbuf, rsiz); + } + if (rbuf != stack) delete[] rbuf; + csiz_ += sizevarnum(ksiz) + ksiz + rsiz; + return !err; + } + private: + /** + * Cache flusher. + */ + class FlushThread : public Thread { + public: + /** constructor */ + explicit FlushThread(MapReduce* mr, BasicDB* tmpdb, + TinyHashMap* cache, size_t csiz, bool cown) : + mr_(mr), tmpdb_(tmpdb), cache_(cache), csiz_(csiz), cown_(cown), err_(false) {} + /** perform the concrete process */ + void run() { + if (!mr_->logf("map", "started to flushing the cache: count=%lld size=%lld", + (long long)cache_->count(), (long long)csiz_)) err_ = true; + double stime = time(); + BasicDB* tmpdb = tmpdb_; + TinyHashMap* cache = cache_; + bool cown = cown_; + TinyHashMap::Sorter sorter(cache); + const char* kbuf, *vbuf; + size_t ksiz, vsiz; + while ((kbuf = sorter.get(&ksiz, &vbuf, &vsiz)) != NULL) { + if (!tmpdb->append(kbuf, ksiz, vbuf, vsiz)) { + const BasicDB::Error& e = tmpdb->error(); + mr_->db_->set_error(_KCCODELINE_, e.code(), e.message()); + err_ = true; + } + sorter.step(); + if (cown) cache->remove(kbuf, ksiz); + } + double etime = time(); + if (!mr_->logf("map", "flushing the cache finished: time=%.6f", etime - stime)) + err_ = true; + if (cown) delete cache; + } + /** check the error flag. */ + bool error() { + return err_; + } + private: + MapReduce* mr_; ///< driver + BasicDB* tmpdb_; ///< temprary database + TinyHashMap* cache_; ///< cache for emitter + size_t csiz_; ///< current cache size + bool cown_; ///< cache ownership flag + bool err_; ///< error flag + }; + /** + * Task queue for parallel reducer. + */ + class ReduceTaskQueue : public TaskQueue { + public: + /** + * Task for parallel reducer. + */ + class ReduceTask : public Task { + friend class ReduceTaskQueue; + public: + /** constructor */ + explicit ReduceTask(MapReduce* mr, const char* kbuf, size_t ksiz, const Values& values) : + mr_(mr), key_(kbuf, ksiz), values_(values) {} + private: + MapReduce* mr_; ///< driver + std::string key_; ///< key + Values values_; ///< values + }; + /** constructor */ + explicit ReduceTaskQueue() {} + private: + /** process a task */ + void do_task(Task* task) { + ReduceTask* rtask = (ReduceTask*)task; + ValueIterator iter(rtask->values_.begin(), rtask->values_.end()); + if (!rtask->mr_->reduce(rtask->key_.data(), rtask->key_.size(), &iter)) + rtask->mr_->redaborted_ = true; + delete rtask; + } + }; + /** + * Checker for the map process. + */ + class MapChecker : public BasicDB::ProgressChecker { + public: + /** constructor */ + explicit MapChecker() : stop_(false) {} + /** stop the process */ + void stop() { + stop_ = true; + } + /** check whether stopped */ + bool stopped() { + return stop_; + } + private: + /** check whether stopped */ + bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) { + return !stop_; + } + bool stop_; ///< flag for stop + }; + /** + * Visitor for the map process. + */ + class MapVisitor : public BasicDB::Visitor { + public: + /** constructor */ + explicit MapVisitor(MapReduce* mr, MapChecker* checker, int64_t scale) : + mr_(mr), checker_(checker), scale_(scale), stime_(0), err_(false) {} + /** get the error flag */ + bool error() { + return err_; + } + /** preprocess the mappter */ + void visit_before() { + mr_->dbclock_ = 0; + mr_->cache_ = new TinyHashMap(mr_->cbnum_); + mr_->csiz_ = 0; + if (!mr_->preprocess()) err_ = true; + if (mr_->csiz_ > 0 && !mr_->flush_cache()) err_ = true; + if (!mr_->logf("map", "started the map process: scale=%lld", (long long)scale_)) + err_ = true; + stime_ = time(); + } + /** postprocess the mappter and call the reducer */ + void visit_after() { + if (mr_->csiz_ > 0 && !mr_->flush_cache()) err_ = true; + double etime = time(); + if (!mr_->logf("map", "the map process finished: time=%.6f", etime - stime_)) + err_ = true; + if (!mr_->midprocess()) err_ = true; + if (mr_->csiz_ > 0 && !mr_->flush_cache()) err_ = true; + delete mr_->cache_; + if (mr_->flsths_ && !mr_->flsths_->empty()) { + std::deque<FlushThread*>::iterator flthit = mr_->flsths_->begin(); + std::deque<FlushThread*>::iterator flthitend = mr_->flsths_->end(); + while (flthit != flthitend) { + FlushThread* flth = *flthit; + flth->join(); + if (flth->error()) err_ = true; + delete flth; + ++flthit; + } + } + if (!err_ && !mr_->execute_reduce()) err_ = true; + if (!mr_->postprocess()) err_ = true; + } + private: + /** visit a record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (!mr_->map(kbuf, ksiz, vbuf, vsiz)) { + checker_->stop(); + err_ = true; + } + if (mr_->rlocks_) { + if (mr_->csiz_ >= mr_->clim_) { + mr_->rlocks_->lock_all(); + if (mr_->csiz_ >= mr_->clim_ && !mr_->flush_cache()) { + checker_->stop(); + err_ = true; + } + mr_->rlocks_->unlock_all(); + } + } else { + if (mr_->csiz_ >= mr_->clim_ && !mr_->flush_cache()) { + checker_->stop(); + err_ = true; + } + } + return NOP; + } + MapReduce* mr_; ///< driver + MapChecker* checker_; ///< checker + int64_t scale_; ///< number of records + double stime_; ///< start time + bool err_; ///< error flag + }; + /** + * Front line of a merging list. + */ + struct MergeLine { + BasicDB::Cursor* cur; ///< cursor + Comparator* rcomp; ///< record comparator + char* kbuf; ///< pointer to the key + size_t ksiz; ///< size of the key + const char* vbuf; ///< pointer to the value + size_t vsiz; ///< size of the value + /** comparing operator */ + bool operator <(const MergeLine& right) const { + return rcomp->compare(kbuf, ksiz, right.kbuf, right.ksiz) > 0; + } + }; + /** + * Process a log message. + * @param name the name of the event. + * @param format the printf-like format string. + * @param ... used according to the format string. + * @return true on success, or false on failure. + */ + bool logf(const char* name, const char* format, ...) { + _assert_(name && format); + va_list ap; + va_start(ap, format); + std::string message; + vstrprintf(&message, format, ap); + va_end(ap); + return log(name, message.c_str()); + } + /** + * Flush all cache records. + * @return true on success, or false on failure. + */ + bool flush_cache() { + _assert_(true); + bool err = false; + BasicDB* tmpdb = tmpdbs_[dbclock_]; + dbclock_ = (dbclock_ + 1) % dbnum_; + if (flsths_) { + size_t num = flsths_->size(); + if (num >= flsthnum_ || num >= dbnum_) { + FlushThread* flth = flsths_->front(); + flsths_->pop_front(); + flth->join(); + if (flth->error()) err = true; + delete flth; + } + FlushThread* flth = new FlushThread(this, tmpdb, cache_, csiz_, true); + cache_ = new TinyHashMap(cbnum_); + csiz_ = 0; + flth->start(); + flsths_->push_back(flth); + } else { + FlushThread flth(this, tmpdb, cache_, csiz_, false); + flth.run(); + if (flth.error()) err = true; + cache_->clear(); + csiz_ = 0; + } + return !err; + } + /** + * Execute the reduce part. + * @return true on success, or false on failure. + */ + bool execute_reduce() { + bool err = false; + int64_t scale = 0; + for (size_t i = 0; i < dbnum_; i++) { + scale += tmpdbs_[i]->count(); + } + if (!logf("reduce", "started the reduce process: scale=%lld", (long long)scale)) err = true; + double stime = time(); + if (redtasks_) redtasks_->start(redthnum_); + std::priority_queue<MergeLine> lines; + for (size_t i = 0; i < dbnum_; i++) { + MergeLine line; + line.cur = tmpdbs_[i]->cursor(); + line.rcomp = rcomp_; + line.cur->jump(); + line.kbuf = line.cur->get(&line.ksiz, &line.vbuf, &line.vsiz, true); + if (line.kbuf) { + lines.push(line); + } else { + delete line.cur; + } + } + char* lkbuf = NULL; + size_t lksiz = 0; + Values values; + while (!err && !lines.empty()) { + MergeLine line = lines.top(); + lines.pop(); + if (lkbuf && (lksiz != line.ksiz || std::memcmp(lkbuf, line.kbuf, lksiz))) { + if (!call_reducer(lkbuf, lksiz, values)) { + db_->set_error(_KCCODELINE_, BasicDB::Error::LOGIC, "reducer failed"); + err = true; + } + values.clear(); + } + delete[] lkbuf; + lkbuf = line.kbuf; + lksiz = line.ksiz; + values.push_back(std::string(line.vbuf, line.vsiz)); + line.kbuf = line.cur->get(&line.ksiz, &line.vbuf, &line.vsiz, true); + if (line.kbuf) { + lines.push(line); + } else { + delete line.cur; + } + } + if (lkbuf) { + if (!err && !call_reducer(lkbuf, lksiz, values)) { + db_->set_error(_KCCODELINE_, BasicDB::Error::LOGIC, "reducer failed"); + err = true; + } + delete[] lkbuf; + } + while (!lines.empty()) { + MergeLine line = lines.top(); + lines.pop(); + delete[] line.kbuf; + delete line.cur; + } + if (redtasks_) redtasks_->finish(); + double etime = time(); + if (!logf("reduce", "the reduce process finished: time=%.6f", etime - stime)) err = true; + return !err; + } + /** + * Call the reducer. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param values a vector of the values. + * @return true on success, or false on failure. + */ + bool call_reducer(const char* kbuf, size_t ksiz, const Values& values) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + if (redtasks_) { + if (redaborted_) return false; + ReduceTaskQueue::ReduceTask* task = + new ReduceTaskQueue::ReduceTask(this, kbuf, ksiz, values); + redtasks_->add_task(task); + return true; + } + bool err = false; + ValueIterator iter(values.begin(), values.end()); + if (!reduce(kbuf, ksiz, &iter)) err = true; + return !err; + } + /** Dummy constructor to forbid the use. */ + MapReduce(const MapReduce&); + /** Dummy Operator to forbid the use. */ + MapReduce& operator =(const MapReduce&); + /** The internal database. */ + BasicDB* db_; + /** The record comparator. */ + Comparator* rcomp_; + /** The temporary databases. */ + BasicDB** tmpdbs_; + /** The number of temporary databases. */ + size_t dbnum_; + /** The logical clock for temporary databases. */ + int64_t dbclock_; + /** The number of the mapper threads. */ + size_t mapthnum_; + /** The number of the reducer threads. */ + size_t redthnum_; + /** The number of the flusher threads. */ + size_t flsthnum_; + /** The cache for emitter. */ + TinyHashMap* cache_; + /** The current size of the cache for emitter. */ + AtomicInt64 csiz_; + /** The limit size of the cache for emitter. */ + int64_t clim_; + /** The bucket number of the cache for emitter. */ + int64_t cbnum_; + /** The flush threads. */ + std::deque<FlushThread*>* flsths_; + /** The task queue for parallel reducer. */ + TaskQueue* redtasks_; + /** The flag whether aborted. */ + bool redaborted_; + /** The whole lock. */ + SlottedMutex* rlocks_; +}; + + +/** + * Index database. + * @note This class is designed to implement an indexing storage with an efficient appending + * operation for the existing record values. This class is a wrapper of the polymorphic + * database, featuring buffering mechanism to alleviate IO overhead in the database layer. This + * class can be inherited but overwriting methods is forbidden. Before every database operation, + * it is necessary to call the IndexDB::open method in order to open a database file and connect + * the database object to it. To avoid data missing or corruption, it is important to close + * every database file by the IndexDB::close method when the database is no longer in use. It + * is forbidden for multible database objects in a process to open the same database at the same + * time. It is forbidden to share a database object with child processes. + */ +class IndexDB { + private: + /** The default number of temporary databases. */ + static const size_t DEFDBNUM = 8; + /** The maxinum number of temporary databases. */ + static const size_t MAXDBNUM = 256; + /** The default cache limit size. */ + static const int64_t DEFCLIM = 256LL << 20; + /** The default cache bucket number. */ + static const int64_t DEFCBNUM = 1048583LL; + /** The bucket number of temprary databases. */ + static const int64_t DBBNUM = 512LL << 10; + /** The page size of temprary databases. */ + static const int32_t DBPSIZ = 32768; + /** The mapped size of temprary databases. */ + static const int64_t DBMSIZ = 516LL * 4096; + /** The page cache capacity of temprary databases. */ + static const int64_t DBPCCAP = 16LL << 20; + public: + /** + * Default constructor. + */ + explicit IndexDB() : + mlock_(), db_(), omode_(0), + rcomp_(NULL), tmppath_(""), tmpdbs_(NULL), dbnum_(DEFDBNUM), dbclock_(0), + cache_(NULL), csiz_(0), clim_(0) { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~IndexDB() { + _assert_(true); + if (omode_ != 0) close(); + } + /** + * Get the last happened error. + * @return the last happened error. + */ + BasicDB::Error error() const { + _assert_(true); + return db_.error(); + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + BasicDB::Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + db_.set_error(file, line, func, code, message); + } + /** + * Set the error information without source code information. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(BasicDB::Error::Code code, const char* message) { + _assert_(message); + db_.set_error(_KCCODELINE_, code, message); + } + /** + * Open a database file. + * @param path the path of a database file. The same as with PolyDB. In addition, the + * following tuning parameters are supported. "idxclim" specifies the limit size of the + * internal cache. "idxcbnum" the bucket number of the internal cache. "idxdbnum" specifies + * the number of internal databases. "idxtmppath' specifies the path of the temporary + * directory. + * @param mode the connection mode. The same as with PolyDB. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the IndexDB::close method when it is no longer + * in use. It is not allowed for two or more database objects in the same process to keep + * their connections to the same database file at the same time. + */ + bool open(const std::string& path = ":", + uint32_t mode = BasicDB::OWRITER | BasicDB::OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "already opened"); + return false; + } + std::vector<std::string> elems; + strsplit(path, '#', &elems); + int64_t clim = 0; + int64_t cbnum = 0; + size_t dbnum = 0; + std::string tmppath = ""; + std::vector<std::string>::iterator it = elems.begin(); + std::vector<std::string>::iterator itend = elems.end(); + if (it != itend) ++it; + while (it != itend) { + std::vector<std::string> fields; + if (strsplit(*it, '=', &fields) > 1) { + const char* key = fields[0].c_str(); + const char* value = fields[1].c_str(); + if (!std::strcmp(key, "idxclim") || !std::strcmp(key, "idxcachelimit")) { + clim = atoix(value); + } else if (!std::strcmp(key, "idxcbnum") || !std::strcmp(key, "idxcachebuckets")) { + cbnum = atoix(value); + } else if (!std::strcmp(key, "idxdbnum")) { + dbnum = atoix(value); + } else if (!std::strcmp(key, "idxtmppath")) { + tmppath = value; + } + } + ++it; + } + if (!db_.open(path, mode)) return false; + tmppath_ = tmppath; + rcomp_ = LEXICALCOMP; + BasicDB* idb = &db_; + if (typeid(db_) == typeid(PolyDB)) { + PolyDB* pdb = (PolyDB*)idb; + idb = pdb->reveal_inner_db(); + } + const std::type_info& info = typeid(*idb); + if (info == typeid(GrassDB)) { + GrassDB* gdb = (GrassDB*)idb; + rcomp_ = gdb->rcomp(); + } else if (info == typeid(TreeDB)) { + TreeDB* tdb = (TreeDB*)idb; + rcomp_ = tdb->rcomp(); + } else if (info == typeid(ForestDB)) { + ForestDB* fdb = (ForestDB*)idb; + rcomp_ = fdb->rcomp(); + } + dbnum_ = dbnum < MAXDBNUM ? dbnum : MAXDBNUM; + dbclock_ = 0; + if ((mode & BasicDB::OWRITER) && dbnum > 0) { + tmpdbs_ = new BasicDB*[dbnum_]; + if (tmppath_.empty()) { + report(_KCCODELINE_, "started to open temporary databases on memory"); + double stime = time(); + for (size_t i = 0; i < dbnum_; i++) { + GrassDB* gdb = new GrassDB; + gdb->tune_options(GrassDB::TCOMPRESS); + gdb->tune_buckets(DBBNUM / 2); + gdb->tune_page(DBPSIZ); + gdb->tune_page_cache(DBPCCAP); + gdb->tune_comparator(rcomp_); + gdb->open("%", GrassDB::OWRITER | GrassDB::OCREATE | GrassDB::OTRUNCATE); + tmpdbs_[i] = gdb; + } + double etime = time(); + report(_KCCODELINE_, "opening temporary databases finished: time=%.6f", etime - stime); + } else { + File::Status sbuf; + if (!File::status(tmppath_, &sbuf) || !sbuf.isdir) { + set_error(_KCCODELINE_, BasicDB::Error::NOREPOS, "no such directory"); + delete[] tmpdbs_; + db_.close(); + return false; + } + report(_KCCODELINE_, "started to open temporary databases under %s", tmppath.c_str()); + double stime = time(); + uint32_t pid = getpid() & UINT16MAX; + uint32_t tid = Thread::hash() & UINT16MAX; + uint32_t ts = time() * 1000; + bool err = false; + for (size_t i = 0; i < dbnum_; i++) { + std::string childpath = + strprintf("%s%cidx-%04x-%04x-%08x-%03d%ckct", + tmppath_.c_str(), File::PATHCHR, pid, tid, ts, + (int)(i + 1), File::EXTCHR); + TreeDB* tdb = new TreeDB; + tdb->tune_options(TreeDB::TSMALL | TreeDB::TLINEAR); + tdb->tune_buckets(DBBNUM); + tdb->tune_page(DBPSIZ); + tdb->tune_map(DBMSIZ); + tdb->tune_page_cache(DBPCCAP); + tdb->tune_comparator(rcomp_); + if (!tdb->open(childpath, TreeDB::OWRITER | TreeDB::OCREATE | TreeDB::OTRUNCATE)) { + const BasicDB::Error& e = tdb->error(); + set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + tmpdbs_[i] = tdb; + } + double etime = time(); + report(_KCCODELINE_, "opening temporary databases finished: time=%.6f", etime - stime); + if (err) { + for (size_t i = 0; i < dbnum_; i++) { + delete tmpdbs_[i]; + } + delete[] tmpdbs_; + db_.close(); + return false; + } + } + } else { + tmpdbs_ = NULL; + } + if (mode & BasicDB::OWRITER) { + cache_ = new TinyHashMap(cbnum > 0 ? cbnum : DEFCBNUM); + } else { + cache_ = NULL; + } + clim_ = clim > 0 ? clim : DEFCLIM; + csiz_ = 0; + omode_ = mode; + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + bool err = false; + if (cache_) { + if (!flush_cache()) err = true; + delete cache_; + if (tmpdbs_) { + if (!merge_tmpdbs()) err = true; + report(_KCCODELINE_, "closing the temporary databases"); + double stime = time(); + for (size_t i = 0; i < dbnum_; i++) { + BasicDB* tmpdb = tmpdbs_[i]; + const std::string& path = tmpdb->path(); + if (!tmpdb->close()) { + const BasicDB::Error& e = tmpdb->error(); + set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + if (!tmppath_.empty()) File::remove(path); + delete tmpdb; + } + double etime = time(); + report(_KCCODELINE_, "closing the temporary databases finished: %.6f", etime - stime); + delete[] tmpdbs_; + } + } + if (!db_.close()) err = true; + omode_ = 0; + return !err; + } + /** + * Set the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the value is overwritten. + */ + bool set(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + if (!cache_) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "permission denied"); + return false; + } + bool err = false; + if (!clean_dbs(kbuf, ksiz)) err = true; + cache_->set(kbuf, ksiz, vbuf, vsiz); + csiz_ += ksiz + vsiz; + if (csiz_ > clim_ && !flush_cache()) err = false; + return !err; + } + /** + * Set the value of a record. + * @note Equal to the original DB::set method except that the parameters are std::string. + */ + bool set(const std::string& key, const std::string& value) { + _assert_(true); + return set(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Add a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the record is not modified and false is returned. + */ + bool add(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + if (!cache_) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "permission denied"); + return false; + } + if (check_impl(kbuf, ksiz)) { + set_error(_KCCODELINE_, BasicDB::Error::DUPREC, "record duplication"); + return false; + } + bool err = false; + cache_->set(kbuf, ksiz, vbuf, vsiz); + csiz_ += ksiz + vsiz; + if (csiz_ > clim_ && !flush_cache()) err = false; + return !err; + } + /** + * Set the value of a record. + * @note Equal to the original DB::add method except that the parameters are std::string. + */ + bool add(const std::string& key, const std::string& value) { + _assert_(true); + return add(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Replace the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, no new record is created and false is returned. + * If the corresponding record exists, the value is modified. + */ + bool replace(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + if (!cache_) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "permission denied"); + return false; + } + if (!check_impl(kbuf, ksiz)) { + set_error(_KCCODELINE_, BasicDB::Error::NOREC, "no record"); + return false; + } + bool err = false; + if (!clean_dbs(kbuf, ksiz)) err = true; + cache_->set(kbuf, ksiz, vbuf, vsiz); + csiz_ += ksiz + vsiz; + if (csiz_ > clim_ && !flush_cache()) err = false; + return !err; + } + /** + * Replace the value of a record. + * @note Equal to the original DB::replace method except that the parameters are std::string. + */ + bool replace(const std::string& key, const std::string& value) { + _assert_(true); + return replace(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Append the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the given value is appended at the end of the existing value. + */ + bool append(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + if (!cache_) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "permission denied"); + return false; + } + bool err = false; + cache_->append(kbuf, ksiz, vbuf, vsiz); + csiz_ += ksiz + vsiz; + if (csiz_ > clim_ && !flush_cache()) err = false; + return !err; + } + /** + * Set the value of a record. + * @note Equal to the original DB::append method except that the parameters are std::string. + */ + bool append(const std::string& key, const std::string& value) { + _assert_(true); + return append(key.c_str(), key.size(), value.c_str(), value.size()); + } + /** + * Remove a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. + */ + bool remove(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + if (!cache_) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "permission denied"); + return false; + } + bool err = false; + if (!clean_dbs(kbuf, ksiz)) err = true; + cache_->remove(kbuf, ksiz); + return !err; + } + /** + * Remove a record. + * @note Equal to the original DB::remove method except that the parameter is std::string. + */ + bool remove(const std::string& key) { + _assert_(true); + return remove(key.c_str(), key.size()); + } + /** + * Retrieve the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + * @note If no record corresponds to the key, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. Because the region of the return value is allocated with the + * the new[] operator, it should be released with the delete[] operator when it is no longer + * in use. + */ + char* get(const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && sp); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + *sp = 0; + return false; + } + if (!cache_) return db_.get(kbuf, ksiz, sp); + size_t dvsiz = 0; + char* dvbuf = db_.get(kbuf, ksiz, &dvsiz); + size_t cvsiz = 0; + const char* cvbuf = cache_->get(kbuf, ksiz, &cvsiz); + struct Record { + char* buf; + size_t size; + }; + Record* recs = NULL; + bool hit = false; + size_t rsiz = 0; + if (tmpdbs_) { + recs = new Record[dbnum_]; + for (size_t i = 0; i < dbnum_; i++) { + BasicDB* tmpdb = tmpdbs_[i]; + Record* rp = recs + i; + rp->buf = tmpdb->get(kbuf, ksiz, &rp->size); + if (rp->buf) { + rsiz += rp->size; + hit = true; + } + } + } + if (!hit) { + delete[] recs; + if (!dvbuf && !cvbuf) { + *sp = 0; + return NULL; + } + if (!dvbuf) { + dvbuf = new char[cvsiz+1]; + std::memcpy(dvbuf, cvbuf, cvsiz); + *sp = cvsiz; + return dvbuf; + } + if (!cvbuf) { + *sp = dvsiz; + return dvbuf; + } + char* rbuf = new char[dvsiz+cvsiz+1]; + std::memcpy(rbuf, dvbuf, dvsiz); + std::memcpy(rbuf + dvsiz, cvbuf, cvsiz); + delete[] dvbuf; + *sp = dvsiz + cvsiz; + return rbuf; + } + if (dvbuf) rsiz += dvsiz; + if (cvbuf) rsiz += cvsiz; + char* rbuf = new char[rsiz+1]; + char* wp = rbuf; + if (dvbuf) { + std::memcpy(wp, dvbuf, dvsiz); + wp += dvsiz; + delete[] dvbuf; + } + if (cvbuf) { + std::memcpy(wp, cvbuf, cvsiz); + wp += cvsiz; + } + if (recs) { + for (size_t i = 0; i < dbnum_; i++) { + Record* rp = recs + i; + if (rp->buf) { + std::memcpy(wp, rp->buf, rp->size); + wp += rp->size; + delete[] rp->buf; + } + } + delete[] recs; + } + *sp = rsiz; + return rbuf; + } + /** + * Retrieve the value of a record. + * @note Equal to the original DB::get method except that the first parameters is the key + * string and the second parameter is a string to contain the result and the return value is + * bool for success. + */ + bool get(const std::string& key, std::string* value) { + _assert_(value); + size_t vsiz; + char* vbuf = get(key.c_str(), key.size(), &vsiz); + if (!vbuf) return false; + value->clear(); + value->append(vbuf, vsiz); + delete[] vbuf; + return true; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, BasicDB::FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + if (!cache_) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "permission denied"); + return false; + } + bool err = false; + if (!flush_cache()) err = true; + if (tmpdbs_ && !merge_tmpdbs()) err = true; + if (!db_.synchronize(hard, proc)) err = true; + return !err; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "not opened"); + return false; + } + if (!cache_) { + set_error(_KCCODELINE_, BasicDB::Error::INVALID, "permission denied"); + return false; + } + cache_->clear(); + csiz_ = 0; + return db_.clear(); + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + return count_impl(); + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + return size_impl(); + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + return db_.path(); + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + return db_.status(strmap); + } + /** + * Reveal the inner database object. + * @return the inner database object, or NULL on failure. + */ + PolyDB* reveal_inner_db() { + _assert_(true); + return &db_; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + BasicDB::Cursor* cursor() { + _assert_(true); + return db_.cursor(); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, BasicDB::Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + db_.log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(BasicDB::Logger* logger, + uint32_t kinds = BasicDB::Logger::WARN | BasicDB::Logger::ERROR) { + _assert_(logger); + return db_.tune_logger(logger, kinds); + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(BasicDB::MetaTrigger* trigger) { + _assert_(trigger); + return db_.tune_meta_trigger(trigger); + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, const char* format, ...) { + _assert_(file && line > 0 && func && format); + std::string message; + va_list ap; + va_start(ap, format); + vstrprintf(&message, format, ap); + va_end(ap); + db_.log(file, line, func, BasicDB::Logger::INFO, message.c_str()); + } + private: + /** + * Flush all cache records. + * @return true on success, or false on failure. + */ + bool flush_cache() { + _assert_(true); + bool err = false; + double stime = time(); + report(_KCCODELINE_, "flushing the cache"); + if (tmpdbs_) { + BasicDB* tmpdb = tmpdbs_[dbclock_]; + TinyHashMap::Sorter sorter(cache_); + const char* kbuf, *vbuf; + size_t ksiz, vsiz; + while ((kbuf = sorter.get(&ksiz, &vbuf, &vsiz)) != NULL) { + if (!tmpdb->append(kbuf, ksiz, vbuf, vsiz)) { + const BasicDB::Error& e = tmpdb->error(); + db_.set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + sorter.step(); + } + dbclock_ = (dbclock_ + 1) % dbnum_; + } else { + TinyHashMap::Sorter sorter(cache_); + const char* kbuf, *vbuf; + size_t ksiz, vsiz; + while ((kbuf = sorter.get(&ksiz, &vbuf, &vsiz)) != NULL) { + if (!db_.append(kbuf, ksiz, vbuf, vsiz)) err = true; + sorter.step(); + } + } + cache_->clear(); + csiz_ = 0; + double etime = time(); + report(_KCCODELINE_, "flushing the cache finished: time=%.6f", etime - stime); + return !err; + } + /** + * Merge temporary databases. + * @return true on success, or false on failure. + */ + bool merge_tmpdbs() { + _assert_(true); + bool err = false; + report(_KCCODELINE_, "merging the temporary databases"); + double stime = time(); + if (!db_.merge(tmpdbs_, dbnum_, PolyDB::MAPPEND)) err = true; + dbclock_ = 0; + for (size_t i = 0; i < dbnum_; i++) { + BasicDB* tmpdb = tmpdbs_[i]; + if (!tmpdb->clear()) { + const BasicDB::Error& e = tmpdb->error(); + set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + } + double etime = time(); + report(_KCCODELINE_, "merging the temporary databases finished: %.6f", etime - stime); + return !err; + } + /** + * Remove a record from databases. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool clean_dbs(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + if (db_.remove(kbuf, ksiz)) return true; + bool err = false; + if (db_.error() != BasicDB::Error::NOREC) err = true; + if (tmpdbs_) { + for (size_t i = 0; i < dbnum_; i++) { + BasicDB* tmpdb = tmpdbs_[i]; + if (!tmpdb->remove(kbuf, ksiz)) { + const BasicDB::Error& e = tmpdb->error(); + if (e != BasicDB::Error::NOREC) { + set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + } + } + } + return !err; + } + /** + * Check whether a record exists. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true if the record exists, or false if not. + */ + bool check_impl(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + char vbuf; + if (db_.get(kbuf, ksiz, &vbuf, 1) >= 0) return true; + if (cache_) { + size_t vsiz; + if (cache_->get(kbuf, ksiz, &vsiz)) return true; + if (tmpdbs_) { + for (size_t i = 0; i < dbnum_; i++) { + BasicDB* tmpdb = tmpdbs_[i]; + if (tmpdb->get(kbuf, ksiz, &vbuf, 1)) return true; + } + } + } + return false; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count_impl() { + _assert_(true); + int64_t dbcnt = db_.count(); + if (dbcnt < 0) return -1; + if (!cache_) return dbcnt; + int64_t ccnt = cache_->count(); + return dbcnt > ccnt ? dbcnt : ccnt; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes. + */ + int64_t size_impl() { + _assert_(true); + int64_t dbsiz = db_.size(); + if (dbsiz < 0) return -1; + return dbsiz + csiz_; + } + /** Dummy constructor to forbid the use. */ + IndexDB(const IndexDB&); + /** Dummy Operator to forbid the use. */ + IndexDB& operator =(const IndexDB&); + /** The method lock. */ + RWLock mlock_; + /** The internal database. */ + PolyDB db_; + /** The open mode. */ + uint32_t omode_; + /** The record comparator. */ + Comparator* rcomp_; + /** The base path of temporary databases. */ + std::string tmppath_; + /** The temporary databases. */ + BasicDB** tmpdbs_; + /** The number of temporary databases. */ + size_t dbnum_; + /** The logical clock for temporary databases. */ + int64_t dbclock_; + /** The internal cache. */ + TinyHashMap* cache_; + /** The current size of the internal cache. */ + int64_t csiz_; + /** The limit size of the internal cache. */ + int64_t clim_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdirdb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirdb.cc new file mode 100644 index 0000000000..5331dd1337 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirdb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Directory hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcdirdb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdirdb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirdb.h new file mode 100644 index 0000000000..c80ba2d143 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirdb.h @@ -0,0 +1,2437 @@ +/************************************************************************************************* + * Directory hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCDIRDB_H // duplication check +#define _KCDIRDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> +#include <kcplantdb.h> + +#define KCDDBMAGICFILE "__KCDIR__" ///< magic file of the directory +#define KCDDBMETAFILE "__meta__" ///< meta data file of the directory +#define KCDDBOPAQUEFILE "__opq__" ///< opaque file of the directory +#define KCDDBATRANPREFIX "_x" ///< prefix of files for auto transaction +#define KCDDBCHKSUMSEED "__kyotocabinet__" ///< seed of the module checksum +#define KCDDBMAGICEOF "_EOF_" ///< magic data for the end of file +#define KCDDBWALPATHEXT "wal" ///< extension of the WAL directory +#define KCDDBTMPPATHEXT "tmp" ///< extension of the temporary directory + +namespace kyotocabinet { // common namespace + + +/** + * Directory hash database. + * @note This class is a concrete class to operate a hash database in a directory. This class + * can be inherited but overwriting methods is forbidden. Before every database operation, it is + * necessary to call the DirDB::open method in order to open a database file and connect the + * database object to it. To avoid data missing or corruption, it is important to close every + * database file by the DirDB::close method when the database is no longer in use. It is + * forbidden for multible database objects in a process to open the same database at the same + * time. It is forbidden to share a database object with child processes. + */ +class DirDB : public BasicDB { + friend class PlantDB<DirDB, BasicDB::TYPEFOREST>; + public: + class Cursor; + private: + struct Record; + class ScopedVisitor; + /** An alias of list of cursors. */ + typedef std::list<Cursor*> CursorList; + /** An alias of vector of strings. */ + typedef std::vector<std::string> StringVector; + /** The size of the meta data buffer. */ + static const int64_t METABUFSIZ = 128; + /** The magic data for record. */ + static const uint8_t RECMAGIC = 0xcc; + /** The number of slots of the record lock. */ + static const int32_t RLOCKSLOT = 2048; + /** The unit size of a record. */ + static const int32_t RECUNITSIZ = 32; + /** The size of the opaque buffer. */ + static const size_t OPAQUESIZ = 16; + /** The threshold of busy loop and sleep for locking. */ + static const uint32_t LOCKBUSYLOOP = 8192; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class DirDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(DirDB* db) : db_(db), dir_(), alive_(false), name_("") { + _assert_(db); + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.push_back(this); + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + if (!db_) return; + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.remove(this); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(db_->writer_)) { + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + if (!alive_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + bool err = false; + const std::string& rpath = db_->path_ + File::PATHCHR + name_; + int64_t cnt = db_->count_; + Record rec; + if (db_->read_record(rpath, &rec)) { + if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec.rsiz, + visitor, rpath, name_.c_str())) err = true; + delete[] rec.rbuf; + if (alive_ && step && db_->count_ == cnt) { + do { + if (!dir_.read(&name_)) { + if (!disable()) err = true; + break; + } + } while (*name_.c_str() == *KCDDBMAGICFILE); + } + } else { + while (true) { + if (!dir_.read(&name_)) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + disable(); + break; + } + if (*name_.c_str() == *KCDDBMAGICFILE) continue; + const std::string& npath = db_->path_ + File::PATHCHR + name_; + if (!File::status(npath)) continue; + if (db_->read_record(npath, &rec)) { + if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec.rsiz, + visitor, npath, name_.c_str())) err = true; + delete[] rec.rbuf; + if (alive_ && step && db_->count_ == cnt) { + do { + if (!dir_.read(&name_)) { + if (!disable()) err = true; + break; + } + } while (*name_.c_str() == *KCDDBMAGICFILE); + } + } else { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + err = true; + } + break; + } + } + return !err; + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (alive_ && !disable()) return false; + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!dir_.open(db_->path_)) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + return false; + } + alive_ = true; + do { + if (!dir_.read(&name_)) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + disable(); + return false; + } + } while (*name_.c_str() == *KCDDBMAGICFILE); + return true; + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (alive_ && !disable()) return false; + if (!dir_.open(db_->path_)) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + return false; + } + alive_ = true; + while (true) { + if (!dir_.read(&name_)) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + disable(); + return false; + } + if (*name_.c_str() == *KCDDBMAGICFILE) continue; + const std::string& rpath = db_->path_ + File::PATHCHR + name_; + Record rec; + if (db_->read_record(rpath, &rec)) { + if (rec.ksiz == ksiz && !std::memcmp(rec.kbuf, kbuf, ksiz)) { + delete[] rec.rbuf; + break; + } + delete[] rec.rbuf; + } else { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + disable(); + return false; + } + } + return true; + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + return jump(key.c_str(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const std::string& key) { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!alive_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + do { + if (!dir_.read(&name_)) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + disable(); + return false; + } + } while (*name_.c_str() == *KCDDBMAGICFILE); + return true; + } + /** + * Step the cursor to the previous record. + * @note This is a dummy implementation for compatibility. + */ + bool step_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Get the database object. + * @return the database object. + */ + DirDB* db() { + _assert_(true); + return db_; + } + private: + /** + * Disable the cursor. + * @return true on success, or false on failure. + */ + bool disable() { + bool err = false; + if (!dir_.close()) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); + err = true; + } + alive_ = false; + return !err; + } + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + DirDB* db_; + /** The inner directory stream. */ + DirStream dir_; + /** The flag if alive. */ + bool alive_; + /** The current name. */ + std::string name_; + }; + /** + * Tuning options. + */ + enum Option { + TSMALL = 1 << 0, ///< dummy for compatibility + TLINEAR = 1 << 1, ///< dummy for compatibility + TCOMPRESS = 1 << 2 ///< compress each record + }; + /** + * Status flags. + */ + enum Flag { + FOPEN = 1 << 0, ///< dummy for compatibility + FFATAL = 1 << 1 ///< dummy for compatibility + }; + /** + * Default constructor. + */ + explicit DirDB() : + mlock_(), rlock_(RLOCKSLOT), error_(), + logger_(NULL), logkinds_(0), mtrigger_(NULL), + omode_(0), writer_(false), autotran_(false), autosync_(false), + recov_(false), reorg_(false), + file_(), curs_(), path_(""), + libver_(LIBVER), librev_(LIBREV), fmtver_(FMTVER), chksum_(0), type_(TYPEDIR), + flags_(0), opts_(0), count_(0), size_(0), opaque_(), embcomp_(ZLIBRAWCOMP), comp_(NULL), + tran_(false), trhard_(false), trcount_(0), trsize_(0), walpath_(""), tmppath_("") { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~DirDB() { + _assert_(true); + if (omode_ != 0) close(); + if (!curs_.empty()) { + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->db_ = NULL; + ++cit; + } + } + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + bool err = false; + char name[NUMBUFSIZ]; + size_t lidx = hashpath(kbuf, ksiz, name) % RLOCKSLOT; + if (writable) { + rlock_.lock_writer(lidx); + } else { + rlock_.lock_reader(lidx); + } + if (!accept_impl(kbuf, ksiz, visitor, name)) err = true; + rlock_.unlock(lidx); + return !err; + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + size_t knum = keys.size(); + if (knum < 1) return true; + bool err = false; + struct RecordKey { + const char* kbuf; + size_t ksiz; + char name[NUMBUFSIZ]; + }; + RecordKey* rkeys = new RecordKey[knum]; + std::set<size_t> lidxs; + for (size_t i = 0; i < knum; i++) { + const std::string& key = keys[i]; + RecordKey* rkey = rkeys + i; + rkey->kbuf = key.data(); + rkey->ksiz = key.size(); + lidxs.insert(hashpath(rkey->kbuf, rkey->ksiz, rkey->name) % RLOCKSLOT); + } + std::set<size_t>::iterator lit = lidxs.begin(); + std::set<size_t>::iterator litend = lidxs.end(); + while (lit != litend) { + if (writable) { + rlock_.lock_writer(*lit); + } else { + rlock_.lock_reader(*lit); + } + ++lit; + } + for (size_t i = 0; i < knum; i++) { + RecordKey* rkey = rkeys + i; + if (!accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->name)) { + err = true; + break; + } + } + lit = lidxs.begin(); + litend = lidxs.end(); + while (lit != litend) { + rlock_.unlock(*lit); + ++lit; + } + delete[] rkeys; + return !err; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + bool err = false; + if (!iterate_impl(visitor, checker)) err = true; + trigger_meta(MetaTrigger::ITERATE, "iterate"); + return !err; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (thnum < 1) thnum = 0; + if (thnum > (size_t)INT8MAX) thnum = INT8MAX; + ScopedVisitor svis(visitor); + rlock_.lock_reader_all(); + bool err = false; + if (!scan_parallel_impl(visitor, thnum, checker)) err = true; + rlock_.unlock_all(); + trigger_meta(MetaTrigger::ITERATE, "scan_parallel"); + return !err; + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + return error_; + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + error_->set(code, message); + if (code == Error::BROKEN || code == Error::SYSTEM) flags_ |= FFATAL; + if (logger_) { + Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ? + Logger::ERROR : Logger::INFO; + if (kind & logkinds_) + report(file, line, func, kind, "%d: %s: %s", code, Error::codename(code), message); + } + } + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. DirDB::OWRITER as a writer, DirDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: DirDB::OCREATE, + * which means it creates a new database if the file does not exist, DirDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, DirDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, DirDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: DirDB::ONOLOCK, which means it opens the database file without file locking, + * DirDB::OTRYLOCK, which means locking is performed without blocking, DirDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the DirDB::close method when it is no + * longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", path.c_str()); + writer_ = false; + autotran_ = false; + autosync_ = false; + recov_ = false; + reorg_ = false; + uint32_t fmode = File::OREADER; + if (mode & OWRITER) { + writer_ = true; + fmode = File::OWRITER; + if (mode & OCREATE) fmode |= File::OCREATE; + if (mode & OTRUNCATE) fmode |= File::OTRUNCATE; + if (mode & OAUTOTRAN) autotran_ = true; + if (mode & OAUTOSYNC) autosync_ = true; + } + if (mode & ONOLOCK) fmode |= File::ONOLOCK; + if (mode & OTRYLOCK) fmode |= File::OTRYLOCK; + size_t psiz = path.size(); + while (psiz > 0 && path[psiz-1] == File::PATHCHR) { + psiz--; + } + const std::string& cpath = path.substr(0, psiz); + const std::string& magicpath = cpath + File::PATHCHR + KCDDBMAGICFILE; + const std::string& metapath = cpath + File::PATHCHR + KCDDBMETAFILE; + const std::string& opqpath = cpath + File::PATHCHR + KCDDBOPAQUEFILE; + const std::string& walpath = cpath + File::EXTCHR + KCDDBWALPATHEXT; + const std::string& tmppath = cpath + File::EXTCHR + KCDDBTMPPATHEXT; + bool hot = false; + if (writer_ && (mode & OTRUNCATE) && File::status(magicpath)) { + if (!file_.open(magicpath, fmode)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + if (!remove_files(cpath)) { + file_.close(); + return false; + } + if (File::status(walpath)) { + remove_files(walpath); + File::remove_directory(walpath); + } + if (!file_.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + const std::string& buf = format_magic(0, 0); + if (!File::write_file(magicpath, buf.c_str(), buf.size())) { + set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); + return false; + } + if (File::status(metapath) && !File::remove(metapath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + return false; + } + if (File::status(opqpath) && !File::remove(opqpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + return false; + } + hot = true; + } + File::Status sbuf; + if (File::status(cpath, &sbuf)) { + if (!sbuf.isdir) { + set_error(_KCCODELINE_, Error::NOPERM, "invalid path (not directory)"); + return false; + } + if (!File::status(magicpath)) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data"); + return false; + } + if (!file_.open(magicpath, fmode)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + } else if (writer_ && (mode & OCREATE)) { + hot = true; + if (!File::make_directory(cpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "making a directory failed"); + return false; + } + if (!file_.open(magicpath, fmode)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + } else { + set_error(_KCCODELINE_, Error::NOREPOS, "open failed (file not found)"); + return false; + } + if (hot) { + count_ = 0; + size_ = 0; + comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL; + libver_ = LIBVER; + librev_ = LIBREV; + fmtver_ = FMTVER; + chksum_ = calc_checksum(); + if (!dump_meta(metapath)) { + file_.close(); + return false; + } + std::memset(opaque_, 0, sizeof(opaque_)); + if (autosync_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + file_.close(); + return false; + } + } else { + if (File::status(walpath, &sbuf)) { + if (writer_) { + file_.truncate(0); + } else { + File::write_file(magicpath, "", 0); + file_.refresh(); + } + DirStream dir; + if (dir.open(walpath)) { + std::string name; + while (dir.read(&name)) { + const std::string& srcpath = walpath + File::PATHCHR + name; + const std::string& destpath = cpath + File::PATHCHR + name; + File::Status sbuf; + if (File::status(srcpath, &sbuf)) { + if (sbuf.size > 1) { + File::rename(srcpath, destpath); + } else { + if (File::remove(destpath) || !File::status(destpath)) File::remove(srcpath); + } + } + } + dir.close(); + File::remove_directory(walpath); + recov_ = true; + report(_KCCODELINE_, Logger::WARN, "recovered by the WAL directory"); + } + } + if (!load_meta(metapath)) { + file_.close(); + return false; + } + comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL; + uint8_t chksum = calc_checksum(); + if (chksum != chksum_) { + set_error(_KCCODELINE_, Error::INVALID, "invalid module checksum"); + report(_KCCODELINE_, Logger::WARN, "saved=%02X calculated=%02X", + (unsigned)chksum_, (unsigned)chksum); + file_.close(); + return false; + } + if (!load_magic()) { + if (!calc_magic(cpath)) { + file_.close(); + return false; + } + reorg_ = true; + if (!writer_ && !(mode & ONOLOCK)) { + const std::string& buf = format_magic(count_, size_); + if (!File::write_file(magicpath, buf.c_str(), buf.size())) { + set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); + file_.close(); + return false; + } + if (!file_.refresh()) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + file_.close(); + return false; + } + } + report(_KCCODELINE_, Logger::WARN, "re-calculated magic data"); + } + } + if (writer_ && !file_.truncate(0)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + file_.close(); + return false; + } + if (File::status(walpath)) { + remove_files(walpath); + File::remove_directory(walpath); + } + if (File::status(tmppath)) { + remove_files(tmppath); + File::remove_directory(tmppath); + } + omode_ = mode; + path_ = cpath; + tran_ = false; + walpath_ = walpath; + tmppath_ = tmppath; + load_opaque(); + trigger_meta(MetaTrigger::OPEN, "open"); + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", path_.c_str()); + bool err = false; + if (tran_ && !abort_transaction()) err = true; + if (!disable_cursors()) err = true; + if (writer_) { + if (!dump_magic()) err = true; + if (!dump_opaque()) err = true; + } + if (!file_.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + omode_ = 0; + trigger_meta(MetaTrigger::CLOSE, "close"); + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + rlock_.lock_reader_all(); + bool err = false; + if (!synchronize_impl(hard, proc, checker)) err = true; + trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize"); + rlock_.unlock_all(); + return !err; + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, writable); + bool err = false; + if (proc && !proc->process(path_, count_, size_impl())) { + set_error(_KCCODELINE_, Error::LOGIC, "processing failed"); + err = true; + } + trigger_meta(MetaTrigger::OCCUPY, "occupy"); + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + uint32_t wcnt = 0; + while (true) { + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!tran_) break; + mlock_.unlock(); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + trhard_ = hard; + if (!begin_transaction_impl()) { + mlock_.unlock(); + return false; + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); + mlock_.unlock(); + return true; + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (tran_) { + set_error(_KCCODELINE_, Error::LOGIC, "competition avoided"); + mlock_.unlock(); + return false; + } + trhard_ = hard; + if (!begin_transaction_impl()) { + mlock_.unlock(); + return false; + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try"); + mlock_.unlock(); + return true; + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!tran_) { + set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); + return false; + } + bool err = false; + if (commit) { + if (!commit_transaction()) err = true; + } else { + if (!abort_transaction()) err = true; + } + tran_ = false; + trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction"); + return !err; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + bool err = false; + if (!disable_cursors()) err = true; + if (tran_) { + DirStream dir; + if (dir.open(path_)) { + std::string name; + while (dir.read(&name)) { + if (*name.c_str() == *KCDDBMAGICFILE) continue; + const std::string& rpath = path_ + File::PATHCHR + name; + const std::string& walpath = walpath_ + File::PATHCHR + name; + if (File::status(walpath)) { + if (!File::remove(rpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + err = true; + } + } else if (!File::rename(rpath, walpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed"); + err = true; + } + } + if (!dir.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); + err = true; + } + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + err = true; + } + } else { + if (!remove_files(path_)) err = true; + } + recov_ = false; + reorg_ = false; + flags_ = 0; + std::memset(opaque_, 0, sizeof(opaque_)); + count_ = 0; + size_ = 0; + trigger_meta(MetaTrigger::CLEAR, "clear"); + return !err; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return count_; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return size_impl(); + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return ""; + } + return path_; + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + (*strmap)["type"] = strprintf("%u", (unsigned)TYPEDIR); + (*strmap)["realtype"] = strprintf("%u", (unsigned)type_); + (*strmap)["path"] = path_; + (*strmap)["libver"] = strprintf("%u", libver_); + (*strmap)["librev"] = strprintf("%u", librev_); + (*strmap)["fmtver"] = strprintf("%u", fmtver_); + (*strmap)["chksum"] = strprintf("%u", chksum_); + (*strmap)["flags"] = strprintf("%u", flags_); + (*strmap)["opts"] = strprintf("%u", opts_); + (*strmap)["recovered"] = strprintf("%d", recov_); + (*strmap)["reorganized"] = strprintf("%d", reorg_); + if (strmap->count("opaque") > 0) + (*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_)); + (*strmap)["count"] = strprintf("%lld", (long long)count_); + (*strmap)["size"] = strprintf("%lld", (long long)size_impl()); + return true; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + ScopedRWLock lock(&mlock_, false); + if (!logger_) return; + logger_->log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + logger_ = logger; + logkinds_ = kinds; + return true; + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + /** + * Set the optional features. + * @param opts the optional features by bitwise-or: DirDB::TCOMPRESS to compress each record. + * @return true on success, or false on failure. + */ + bool tune_options(int8_t opts) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + opts_ = opts; + return true; + } + /** + * Set the data compressor. + * @param comp the data compressor object. + * @return true on success, or false on failure. + */ + bool tune_compressor(Compressor* comp) { + _assert_(comp); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + embcomp_ = comp; + return true; + } + /** + * Get the opaque data. + * @return the pointer to the opaque data region, whose size is 16 bytes. + */ + char* opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return opaque_; + } + /** + * Synchronize the opaque data. + * @return true on success, or false on failure. + */ + bool synchronize_opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + bool err = false; + if (!dump_opaque()) err = true; + return !err; + } + /** + * Get the status flags. + * @note This is a dummy implementation for compatibility. + */ + uint8_t flags() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return 0; + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, ...) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + va_list ap; + va_start(ap, format); + vstrprintf(&message, format, ap); + va_end(ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report a message for debugging with variable number of arguments. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ap used according to the format string. + */ + void report_valist(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, va_list ap) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + vstrprintf(&message, format, ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report the content of a binary buffer for debugging. + * @param file the file name of the epicenter. + * @param line the line number of the epicenter. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param name the name of the information. + * @param buf the binary buffer. + * @param size the size of the binary buffer + */ + void report_binary(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* name, const char* buf, size_t size) { + _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ); + if (!logger_) return; + char* hex = hexencode(buf, size); + report(file, line, func, kind, "%s=%s", name, hex); + delete[] hex; + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN for beginning + * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaTrigger::ABORTTRAN + * for aborting transaction, and MetaTrigger::MISC for miscellaneous operations. + * @param message the supplement message. + */ + void trigger_meta(MetaTrigger::Kind kind, const char* message) { + _assert_(message); + if (mtrigger_) mtrigger_->trigger(kind, message); + } + /** + * Set the database type. + * @param type the database type. + * @return true on success, or false on failure. + */ + bool tune_type(int8_t type) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + type_ = type; + return true; + } + /** + * Get the library version. + * @return the library version, or 0 on failure. + */ + uint8_t libver() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return libver_; + } + /** + * Get the library revision. + * @return the library revision, or 0 on failure. + */ + uint8_t librev() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return librev_; + } + /** + * Get the format version. + * @return the format version, or 0 on failure. + */ + uint8_t fmtver() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return fmtver_; + } + /** + * Get the module checksum. + * @return the module checksum, or 0 on failure. + */ + uint8_t chksum() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return chksum_; + } + /** + * Get the database type. + * @return the database type, or 0 on failure. + */ + uint8_t type() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return type_; + } + /** + * Get the options. + * @return the options, or 0 on failure. + */ + uint8_t opts() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return opts_; + } + /** + * Get the data compressor. + * @return the data compressor, or NULL on failure. + */ + Compressor* comp() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return comp_; + } + /** + * Check whether the database was recovered or not. + * @return true if recovered, or false if not. + */ + bool recovered() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return recov_; + } + /** + * Check whether the database was reorganized or not. + * @return true if reorganized, or false if not. + */ + bool reorganized() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return reorg_; + } + private: + /** + * Set the power of the alignment of record size. + * @note This is a dummy implementation for compatibility. + */ + bool tune_alignment(int8_t apow) { + return true; + } + /** + * Set the power of the capacity of the free block pool. + * @note This is a dummy implementation for compatibility. + */ + bool tune_fbp(int8_t fpow) { + return true; + } + /** + * Set the number of buckets of the hash table. + * @note This is a dummy implementation for compatibility. + */ + bool tune_buckets(int64_t bnum) { + return true; + } + /** + * Set the size of the internal memory-mapped region. + * @note This is a dummy implementation for compatibility. + */ + bool tune_map(int64_t msiz) { + return true; + } + /** + * Set the unit step number of auto defragmentation. + * @note This is a dummy implementation for compatibility. + */ + bool tune_defrag(int64_t dfunit) { + return true; + } + /** + * Perform defragmentation of the file. + * @note This is a dummy implementation for compatibility. + */ + bool defrag(int64_t step = 0) { + return true; + } + /** + * Get the alignment power. + * @note This is a dummy implementation for compatibility. + */ + uint8_t apow() { + return 0; + } + /** + * Get the free block pool power. + * @note This is a dummy implementation for compatibility. + */ + uint8_t fpow() { + return 0; + } + /** + * Get the bucket number. + * @note This is a dummy implementation for compatibility. + */ + int64_t bnum() { + return 1; + } + /** + * Get the size of the internal memory-mapped region. + * @note This is a dummy implementation for compatibility. + */ + int64_t msiz() { + return 0; + } + /** + * Get the unit step number of auto defragmentation. + * @note This is a dummy implementation for compatibility. + */ + int64_t dfunit() { + return 0; + } + private: + /** + * Record data. + */ + struct Record { + char* rbuf; ///< record buffer + size_t rsiz; ///< record size + const char* kbuf; ///< key buffer + size_t ksiz; ///< key size + const char* vbuf; ///< value buffer + size_t vsiz; ///< value size + }; + /** + * Scoped visitor. + */ + class ScopedVisitor { + public: + /** constructor */ + explicit ScopedVisitor(Visitor* visitor) : visitor_(visitor) { + _assert_(visitor); + visitor_->visit_before(); + } + /** destructor */ + ~ScopedVisitor() { + _assert_(true); + visitor_->visit_after(); + } + private: + Visitor* visitor_; ///< visitor + }; + /** + * Dump the magic data into the file. + * @return true on success, or false on failure. + */ + bool dump_magic() { + _assert_(true); + const std::string& buf = format_magic(count_, size_); + if (!file_.write(0, buf.c_str(), buf.size())) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + return true; + } + /** + * Format the magic data. + * @return the result string. + */ + std::string format_magic(int64_t count, int64_t size) { + return strprintf("%lld\n%lld\n%s\n", (long long)count, (long long)size, KCDDBMAGICEOF); + } + /** + * Load the magic data from the file. + * @return true on success, or false on failure. + */ + bool load_magic() { + _assert_(true); + char buf[NUMBUFSIZ*3]; + size_t len = file_.size(); + if (len > sizeof(buf) - 1) len = sizeof(buf) - 1; + if (!file_.read(0, buf, len)) return false; + buf[len] = '\0'; + char* rp = buf; + int64_t count = atoi(rp); + char* pv = std::strchr(rp, '\n'); + if (!pv) return false; + rp = pv + 1; + int64_t size = atoi(rp); + pv = std::strchr(rp, '\n'); + if (!pv) return false; + rp = pv + 1; + if (std::strlen(rp) < sizeof(KCDDBMAGICEOF) - 1 || + std::memcmp(rp, KCDDBMAGICEOF, sizeof(KCDDBMAGICEOF) - 1)) return false; + flags_ = 0; + count_ = count; + size_ = size; + return true; + } + /** + * Calculate magic data. + * @param cpath the path of the database file. + * @return true on success, or false on failure. + */ + bool calc_magic(const std::string& cpath) { + _assert_(true); + count_ = 0; + size_ = 0; + DirStream dir; + if (!dir.open(cpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + return false; + } + bool err = false; + std::string name; + while (dir.read(&name)) { + if (*name.c_str() == *KCDDBMAGICFILE) continue; + const std::string& rpath = cpath + File::PATHCHR + name; + File::Status sbuf; + if (File::status(rpath, &sbuf)) { + if (sbuf.size >= 4) { + count_ += 1; + size_ += sbuf.size - 4; + } else { + File::remove(rpath); + } + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "checking the status of a file failed"); + err = true; + } + } + if (!dir.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); + err = true; + } + return !err; + } + /** + * Calculate the module checksum. + * @return the module checksum. + */ + uint8_t calc_checksum() { + _assert_(true); + const char* kbuf = KCDDBCHKSUMSEED; + size_t ksiz = sizeof(KCDDBCHKSUMSEED) - 1; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp_) { + zbuf = comp_->compress(kbuf, ksiz, &zsiz); + if (!zbuf) return 0; + kbuf = zbuf; + ksiz = zsiz; + } + char name[NUMBUFSIZ]; + uint32_t hash = hashpath(kbuf, ksiz, name); + hash += hashmurmur(name, std::strlen(name)); + delete[] zbuf; + return hash; + } + /** + * Dump the meta data into the file. + * @param metapath the path of the meta data file. + * @return true on success, or false on failure. + */ + bool dump_meta(const std::string& metapath) { + _assert_(true); + bool err = false; + char buf[METABUFSIZ]; + char* wp = buf; + wp += std::sprintf(wp, "%u\n", libver_); + wp += std::sprintf(wp, "%u\n", librev_); + wp += std::sprintf(wp, "%u\n", fmtver_); + wp += std::sprintf(wp, "%u\n", chksum_); + wp += std::sprintf(wp, "%u\n", type_); + wp += std::sprintf(wp, "%u\n", opts_); + wp += std::sprintf(wp, "%s\n", KCDDBMAGICEOF); + if (!File::write_file(metapath, buf, wp - buf)) { + set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); + err = true; + } + return !err; + } + /** + * Load the meta data from the file. + * @param metapath the path of the meta data file. + * @return true on success, or false on failure. + */ + bool load_meta(const std::string& metapath) { + _assert_(true); + int64_t size; + char* buf = File::read_file(metapath, &size, METABUFSIZ); + if (!buf) { + set_error(_KCCODELINE_, Error::SYSTEM, "reading a file failed"); + return false; + } + std::string str(buf, size); + delete[] buf; + std::vector<std::string> elems; + if (strsplit(str, '\n', &elems) < 7 || elems[6] != KCDDBMAGICEOF) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data file"); + return false; + } + libver_ = atoi(elems[0].c_str()); + librev_ = atoi(elems[1].c_str()); + fmtver_ = atoi(elems[2].c_str()); + chksum_ = atoi(elems[3].c_str()); + type_ = atoi(elems[4].c_str()); + opts_ = atoi(elems[5].c_str()); + return true; + } + /** + * Dump the opaque data into the file. + * @return true on success, or false on failure. + */ + bool dump_opaque() { + _assert_(true); + bool err = false; + const std::string& opath = path_ + File::PATHCHR + KCDDBOPAQUEFILE; + if (!File::write_file(opath, opaque_, sizeof(opaque_))) { + set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); + err = true; + } + return !err; + } + /** + * Load the opaque data from the file. + * @return true on success, or false on failure. + */ + void load_opaque() { + _assert_(true); + std::memset(opaque_, 0, sizeof(opaque_)); + const std::string& opath = path_ + File::PATHCHR + KCDDBOPAQUEFILE; + int64_t size; + char* buf = File::read_file(opath, &size, sizeof(opaque_)); + if (buf) { + std::memcpy(opaque_, buf, size); + delete[] buf; + } + } + /** + * Remove inner files. + * @param cpath the path of the database file. + * @return true on success, or false on failure. + */ + bool remove_files(const std::string& cpath) { + _assert_(true); + DirStream dir; + if (!dir.open(cpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + return false; + } + bool err = false; + std::string name; + while (dir.read(&name)) { + if (*name.c_str() == *KCDDBMAGICFILE) continue; + const std::string& rpath = cpath + File::PATHCHR + name; + if (!File::remove(rpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + err = true; + } + } + if (!dir.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); + err = true; + } + return !err; + } + /** + * Read a record. + * @param rpath the path of the record. + * @param rec the record structure. + * @return true on success, or false on failure. + */ + bool read_record(const std::string& rpath, Record* rec) { + _assert_(rec); + int64_t rsiz; + char* rbuf = File::read_file(rpath, &rsiz); + if (!rbuf) return false; + rec->rsiz = rsiz; + if (comp_) { + size_t zsiz; + char* zbuf = comp_->decompress(rbuf, rsiz, &zsiz); + if (!zbuf) { + set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed"); + delete[] rbuf; + return false; + } + delete[] rbuf; + rbuf = zbuf; + rsiz = zsiz; + } + const char* rp = rbuf; + if (rsiz < 4 || *(const unsigned char*)rp != RECMAGIC) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a record"); + report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str()); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + delete[] rbuf; + return false; + } + rp++; + uint64_t num; + size_t step = readvarnum(rp, rsiz, &num); + rp += step; + rsiz -= step; + size_t ksiz = num; + if (rsiz < 2) { + report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str()); + delete[] rbuf; + return false; + } + step = readvarnum(rp, rsiz, &num); + rp += step; + rsiz -= step; + size_t vsiz = num; + if (rsiz < 1 + (int64_t)ksiz + (int64_t)vsiz || + ((const unsigned char*)rp)[ksiz+vsiz] != RECMAGIC) { + set_error(_KCCODELINE_, Error::BROKEN, "too short record"); + report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str()); + delete[] rbuf; + return false; + } + rec->rbuf = rbuf; + rec->kbuf = rp; + rec->ksiz = ksiz; + rec->vbuf = rp + ksiz; + rec->vsiz = vsiz; + return true; + } + /** + * Write a record. + * @param rpath the path of the record. + * @param name the file name of the record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @param wsp the pointer to the variable into which the size of the written record is + * assigned. + * @return true on success, or false on failure. + */ + bool write_record(const std::string& rpath, const char* name, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* wsp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && wsp); + bool err = false; + char* rbuf = new char[NUMBUFSIZ*2+ksiz+vsiz]; + char* wp = rbuf; + *(wp++) = RECMAGIC; + wp += writevarnum(wp, ksiz); + wp += writevarnum(wp, vsiz); + std::memcpy(wp, kbuf, ksiz); + wp += ksiz; + std::memcpy(wp, vbuf, vsiz); + wp += vsiz; + *(wp++) = RECMAGIC; + size_t rsiz = wp - rbuf; + if (comp_) { + size_t zsiz; + char* zbuf = comp_->compress(rbuf, rsiz, &zsiz); + if (!zbuf) { + set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed"); + delete[] rbuf; + *wsp = 0; + return false; + } + delete[] rbuf; + rbuf = zbuf; + rsiz = zsiz; + } + if (autotran_ && !tran_) { + const std::string& tpath = path_ + File::PATHCHR + KCDDBATRANPREFIX + name; + if (!File::write_file(tpath, rbuf, rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); + err = true; + } + if (!File::rename(tpath, rpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed"); + err = true; + File::remove(tpath); + } + } else { + if (!File::write_file(rpath, rbuf, rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); + err = true; + } + } + delete[] rbuf; + *wsp = rsiz; + return !err; + } + /** + * Disable all cursors. + * @return true on success, or false on failure. + */ + bool disable_cursors() { + _assert_(true); + if (curs_.empty()) return true; + bool err = false; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->alive_ && !cur->disable()) err = true; + ++cit; + } + return !err; + } + /** + * Escape cursors on a free block. + * @param rpath the file path of the record. + * @param name the file name of the record. + * @return true on success, or false on failure. + */ + bool escape_cursors(const std::string& rpath, const char* name) { + bool err = false; + if (curs_.empty()) return true; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->alive_ && cur->name_ == name) { + do { + if (!cur->dir_.read(&cur->name_)) { + if (!cur->disable()) err = true; + break; + } + } while (*cur->name_.c_str() == *KCDDBMAGICFILE); + } + ++cit; + } + return !err; + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param name the encoded key. + * @return true on success, or false on failure. + */ + bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, const char* name) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && name); + bool err = false; + const std::string& rpath = path_ + File::PATHCHR + name; + Record rec; + if (read_record(rpath, &rec)) { + if (rec.ksiz == ksiz || !std::memcmp(rec.kbuf, kbuf, ksiz)) { + if (!accept_visit_full(kbuf, ksiz, rec.vbuf, rec.vsiz, rec.rsiz, + visitor, rpath, name)) err = true; + } else { + set_error(_KCCODELINE_, Error::LOGIC, "collision of the hash values"); + err = true; + } + delete[] rec.rbuf; + } else { + if (!accept_visit_empty(kbuf, ksiz, visitor, rpath, name)) err = true; + } + return !err; + } + /** + * Accept the visit_full method. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @param osiz the old size of the record. + * @param visitor a visitor object. + * @param rpath the file path of the record. + * @param name the file name of the record. + * @return true on success, or false on failure. + */ + bool accept_visit_full(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz, + size_t osiz, Visitor *visitor, const std::string& rpath, + const char* name) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && visitor); + bool err = false; + size_t rsiz; + const char* rbuf = visitor->visit_full(kbuf, ksiz, vbuf, vsiz, &rsiz); + if (rbuf == Visitor::REMOVE) { + if (tran_) { + const std::string& walpath = walpath_ + File::PATHCHR + name; + if (File::status(walpath)) { + if (!File::remove(rpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + err = true; + } + } else if (!File::rename(rpath, walpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed"); + err = true; + } + } else { + if (!File::remove(rpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + err = true; + } + } + if (!escape_cursors(rpath, name)) err = true; + count_ -= 1; + size_ -= osiz; + if (autosync_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + err = true; + } + } else if (rbuf != Visitor::NOP) { + if (tran_) { + const std::string& walpath = walpath_ + File::PATHCHR + name; + if (!File::status(walpath) && !File::rename(rpath, walpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed"); + err = true; + } + } + size_t wsiz; + if (!write_record(rpath, name, kbuf, ksiz, rbuf, rsiz, &wsiz)) err = true; + size_ += (int64_t)wsiz - (int64_t)osiz; + if (autosync_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + err = true; + } + } + return !err; + } + /** + * Accept the visit_empty method. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param rpath the file path of the record. + * @param name the file name of the record. + * @return true on success, or false on failure. + */ + bool accept_visit_empty(const char* kbuf, size_t ksiz, + Visitor *visitor, const std::string& rpath, const char* name) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + bool err = false; + size_t rsiz; + const char* rbuf = visitor->visit_empty(kbuf, ksiz, &rsiz); + if (rbuf != Visitor::NOP && rbuf != Visitor::REMOVE) { + if (tran_) { + const std::string& walpath = walpath_ + File::PATHCHR + name; + if (!File::status(walpath) && !File::write_file(walpath, "", 0)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed"); + err = true; + } + } + size_t wsiz; + if (!write_record(rpath, name, kbuf, ksiz, rbuf, rsiz, &wsiz)) err = true; + count_ += 1; + size_ += wsiz; + if (autosync_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + err = true; + } + } + return !err; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool iterate_impl(Visitor* visitor, ProgressChecker* checker) { + _assert_(visitor); + int64_t allcnt = count_; + if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + DirStream dir; + if (!dir.open(path_)) { + set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + return false; + } + bool err = false; + std::string name; + int64_t curcnt = 0; + while (dir.read(&name)) { + if (*name.c_str() == *KCDDBMAGICFILE) continue; + const std::string& rpath = path_ + File::PATHCHR + name; + Record rec; + if (read_record(rpath, &rec)) { + if (!accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec.rsiz, + visitor, rpath, name.c_str())) err = true; + delete[] rec.rbuf; + } else { + set_error(_KCCODELINE_, Error::BROKEN, "missing record"); + err = true; + } + curcnt++; + if (checker && !checker->check("iterate", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + break; + } + } + if (!dir.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); + err = true; + } + if (checker && !checker->check("iterate", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + return !err; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool scan_parallel_impl(Visitor *visitor, size_t thnum, ProgressChecker* checker) { + _assert_(visitor && thnum <= MEMMAXSIZ); + int64_t allcnt = count_; + if (checker && !checker->check("scan_parallel", "beginning", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + DirStream dir; + if (!dir.open(path_)) { + set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + return false; + } + class ThreadImpl : public Thread { + public: + explicit ThreadImpl() : + db_(NULL), visitor_(NULL), checker_(NULL), allcnt_(0), + dir_(NULL), itmtx_(NULL), error_() {} + void init(DirDB* db, Visitor* visitor, ProgressChecker* checker, int64_t allcnt, + DirStream* dir, Mutex* itmtx) { + db_ = db; + visitor_ = visitor; + checker_ = checker; + allcnt_ = allcnt; + dir_ = dir; + itmtx_ = itmtx; + } + const Error& error() { + return error_; + } + private: + void run() { + DirDB* db = db_; + Visitor* visitor = visitor_; + ProgressChecker* checker = checker_; + int64_t allcnt = allcnt_; + DirStream* dir = dir_; + Mutex* itmtx = itmtx_; + const std::string& path = db->path_; + while (true) { + itmtx->lock(); + std::string name; + if (!dir->read(&name)) { + itmtx->unlock(); + break; + } + itmtx->unlock(); + if (*name.c_str() == *KCDDBMAGICFILE) continue; + const std::string& rpath = path + File::PATHCHR + name; + Record rec; + if (db->read_record(rpath, &rec)) { + size_t vsiz; + visitor->visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, &vsiz); + delete[] rec.rbuf; + } else { + error_ = db->error(); + break; + } + if (checker && !checker->check("scan_parallel", "processing", -1, allcnt)) { + db->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + error_ = db->error(); + break; + } + } + } + DirDB* db_; + Visitor* visitor_; + ProgressChecker* checker_; + int64_t allcnt_; + DirStream* dir_; + Mutex* itmtx_; + Error error_; + }; + bool err = false; + Mutex itmtx; + ThreadImpl* threads = new ThreadImpl[thnum]; + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->init(this, visitor, checker, allcnt, &dir, &itmtx); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->start(); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->join(); + if (thread->error() != Error::SUCCESS) { + *error_ = thread->error(); + err = true; + } + } + delete[] threads; + if (!dir.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); + err = true; + } + if (checker && !checker->check("scan_parallel", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool synchronize_impl(bool hard, FileProcessor* proc, ProgressChecker* checker) { + _assert_(true); + bool err = false; + if (writer_) { + if (checker && !checker->check("synchronize", "dumping the magic data", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!dump_magic()) err = true; + if (checker && !checker->check("synchronize", "synchronizing the directory", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (hard && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + err = true; + } + } + if (proc) { + if (checker && !checker->check("synchronize", "running the post processor", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!proc->process(path_, count_, size_impl())) { + set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed"); + err = true; + } + } + if (writer_ && !file_.truncate(0)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + return !err; + } + /** + * Begin transaction. + * @return true on success, or false on failure. + */ + bool begin_transaction_impl() { + _assert_(true); + if (!File::make_directory(walpath_)) { + set_error(_KCCODELINE_, Error::SYSTEM, "making a directory failed"); + return false; + } + if (trhard_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + return false; + } + trcount_ = count_; + trsize_ = size_; + return true; + } + /** + * Commit transaction. + * @return true on success, or false on failure. + */ + bool commit_transaction() { + _assert_(true); + bool err = false; + if (!File::rename(walpath_, tmppath_)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming a directory failed"); + err = true; + } + if (!remove_files(tmppath_)) err = true; + if (!File::remove_directory(tmppath_)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a directory failed"); + return false; + } + if (trhard_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + err = true; + } + return !err; + } + /** + * Abort transaction. + * @return true on success, or false on failure. + */ + bool abort_transaction() { + _assert_(true); + bool err = false; + if (!disable_cursors()) err = true; + DirStream dir; + if (dir.open(walpath_)) { + std::string name; + while (dir.read(&name)) { + const std::string& srcpath = walpath_ + File::PATHCHR + name; + const std::string& destpath = path_ + File::PATHCHR + name; + File::Status sbuf; + if (File::status(srcpath, &sbuf)) { + if (sbuf.size > 1) { + if (!File::rename(srcpath, destpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed"); + err = true; + } + } else { + if (File::remove(destpath) || !File::status(destpath)) { + if (!File::remove(srcpath)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + err = true; + } + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); + err = true; + } + } + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "checking a file failed"); + err = true; + } + } + if (!dir.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); + err = true; + } + if (!File::remove_directory(walpath_)) { + set_error(_KCCODELINE_, Error::SYSTEM, "removing a directory failed"); + err = true; + } + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); + err = true; + } + count_ = trcount_; + size_ = trsize_; + if (trhard_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + err = true; + } + return !err; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes. + */ + int64_t size_impl() { + return size_ + count_ * RECUNITSIZ; + } + /** Dummy constructor to forbid the use. */ + DirDB(const DirDB&); + /** Dummy Operator to forbid the use. */ + DirDB& operator =(const DirDB&); + /** The method lock. */ + RWLock mlock_; + /** The record locks. */ + SlottedRWLock rlock_; + /** The last happened error. */ + TSD<Error> error_; + /** The internal logger. */ + Logger* logger_; + /** The kinds of logged messages. */ + uint32_t logkinds_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The open mode. */ + uint32_t omode_; + /** The flag for writer. */ + bool writer_; + /** The flag for auto transaction. */ + bool autotran_; + /** The flag for auto synchronization. */ + bool autosync_; + /** The flag for recovered. */ + bool recov_; + /** The flag for reorganized. */ + bool reorg_; + /** The file for magic data. */ + File file_; + /** The cursor objects. */ + CursorList curs_; + /** The path of the database file. */ + std::string path_; + /** The library version. */ + uint8_t libver_; + /** The library revision. */ + uint8_t librev_; + /** The format revision. */ + uint8_t fmtver_; + /** The module checksum. */ + uint8_t chksum_; + /** The database type. */ + uint8_t type_; + /** The status flags. */ + uint8_t flags_; + /** The options. */ + uint8_t opts_; + /** The record number. */ + AtomicInt64 count_; + /** The total size of records. */ + AtomicInt64 size_; + /** The opaque data. */ + char opaque_[OPAQUESIZ]; + /** The embedded data compressor. */ + Compressor* embcomp_; + /** The data compressor. */ + Compressor* comp_; + /** The compression checksum. */ + bool tran_; + /** The flag whether hard transaction. */ + bool trhard_; + /** The old count before transaction. */ + int64_t trcount_; + /** The old size before transaction. */ + int64_t trsize_; + /** The WAL directory for transaction. */ + std::string walpath_; + /** The temporary directory. */ + std::string tmppath_; +}; + + +/** An alias of the directory tree database. */ +typedef PlantDB<DirDB, BasicDB::TYPEFOREST> ForestDB; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdirmgr.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirmgr.cc new file mode 100644 index 0000000000..6544c077dd --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirmgr.cc @@ -0,0 +1,1411 @@ +/************************************************************************************************* + * The command line utility of the directory hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcdirdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, const char* info); +static int32_t runcreate(int argc, char** argv); +static int32_t runinform(int argc, char** argv); +static int32_t runset(int argc, char** argv); +static int32_t runremove(int argc, char** argv); +static int32_t runget(int argc, char** argv); +static int32_t runlist(int argc, char** argv); +static int32_t runclear(int argc, char** argv); +static int32_t runimport(int argc, char** argv); +static int32_t runcopy(int argc, char** argv); +static int32_t rundump(int argc, char** argv); +static int32_t runload(int argc, char** argv); +static int32_t runsetbulk(int argc, char** argv); +static int32_t runremovebulk(int argc, char** argv); +static int32_t rungetbulk(int argc, char** argv); +static int32_t runcheck(int argc, char** argv); +static int32_t proccreate(const char* path, int32_t oflags, int32_t opts); +static int32_t procinform(const char* path, int32_t oflags, bool st); +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode); +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags); +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz); +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + int64_t max, bool rm, bool pv, bool px); +static int32_t procclear(const char* path, int32_t oflags); +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx); +static int32_t proccopy(const char* path, const char* file, int32_t oflags); +static int32_t procdump(const char* path, const char* file, int32_t oflags); +static int32_t procload(const char* path, const char* file, int32_t oflags); +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs); +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys); +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px); +static int32_t proccheck(const char* path, int32_t oflags); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "create")) { + rv = runcreate(argc, argv); + } else if (!std::strcmp(argv[1], "inform")) { + rv = runinform(argc, argv); + } else if (!std::strcmp(argv[1], "set")) { + rv = runset(argc, argv); + } else if (!std::strcmp(argv[1], "remove")) { + rv = runremove(argc, argv); + } else if (!std::strcmp(argv[1], "get")) { + rv = runget(argc, argv); + } else if (!std::strcmp(argv[1], "list")) { + rv = runlist(argc, argv); + } else if (!std::strcmp(argv[1], "clear")) { + rv = runclear(argc, argv); + } else if (!std::strcmp(argv[1], "import")) { + rv = runimport(argc, argv); + } else if (!std::strcmp(argv[1], "copy")) { + rv = runcopy(argc, argv); + } else if (!std::strcmp(argv[1], "dump")) { + rv = rundump(argc, argv); + } else if (!std::strcmp(argv[1], "load")) { + rv = runload(argc, argv); + } else if (!std::strcmp(argv[1], "setbulk")) { + rv = runsetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "removebulk")) { + rv = runremovebulk(argc, argv); + } else if (!std::strcmp(argv[1], "getbulk")) { + rv = rungetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "check")) { + rv = runcheck(argc, argv); + } else if (!std::strcmp(argv[1], "version") || !std::strcmp(argv[1], "--version")) { + printversion(); + } else { + usage(); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: the command line utility of the directory hash database of Kyoto Cabinet\n", + g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s create [-otr] [-onl|-otl|-onr] [-tc] path\n", g_progname); + eprintf(" %s inform [-onl|-otl|-onr] [-st] path\n", g_progname); + eprintf(" %s set [-onl|-otl|-onr] [-add|-rep|-app|-inci|-incd] [-sx] path key value\n", + g_progname); + eprintf(" %s remove [-onl|-otl|-onr] [-sx] path key\n", g_progname); + eprintf(" %s get [-onl|-otl|-onr] [-rm] [-sx] [-px] [-pz] path key\n", g_progname); + eprintf(" %s list [-onl|-otl|-onr] [-max num] [-rm] [-sx] [-pv] [-px] path [key]\n", + g_progname); + eprintf(" %s clear [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s import [-onl|-otl|-onr] [-sx] path [file]\n", g_progname); + eprintf(" %s copy [-onl|-otl|-onr] path file\n", g_progname); + eprintf(" %s dump [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s load [-otr] [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s setbulk [-onl|-otl|-onr] [-sx] path key value ...\n", g_progname); + eprintf(" %s removebulk [-onl|-otl|-onr] [-sx] path key ...\n", g_progname); + eprintf(" %s getbulk [-onl|-otl|-onr] [-sx] [-px] path key ...\n", g_progname); + eprintf(" %s check [-onl|-otl|-onr] path\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print error message of database +static void dberrprint(kc::BasicDB* db, const char* info) { + const kc::BasicDB::Error& err = db->error(); + eprintf("%s: %s: %s: %d: %s: %s\n", + g_progname, info, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// parse arguments of create command +static int32_t runcreate(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + int32_t opts = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::DirDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::DirDB::TCOMPRESS; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccreate(path, oflags, opts); + return rv; +} + + +// parse arguments of inform command +static int32_t runinform(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + bool st = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-st")) { + st = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procinform(path, oflags, st); + return rv; +} + + +// parse arguments of set command +static int32_t runset(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + const char* vstr = NULL; + int32_t oflags = 0; + int32_t mode = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-add")) { + mode = 'a'; + } else if (!std::strcmp(argv[i], "-rep")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-app")) { + mode = 'c'; + } else if (!std::strcmp(argv[i], "-inci")) { + mode = 'i'; + } else if (!std::strcmp(argv[i], "-incd")) { + mode = 'd'; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else if (!vstr) { + vstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr || !vstr) usage(); + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + int32_t rv = procset(path, kstr, ksiz, vstr, vsiz, oflags, mode); + delete[] kbuf; + delete[] vbuf; + return rv; +} + + +// parse arguments of remove command +static int32_t runremove(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procremove(path, kstr, ksiz, oflags); + delete[] kbuf; + return rv; +} + + +// parse arguments of get command +static int32_t runget(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool rm = false; + bool sx = false; + bool px = false; + bool pz = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else if (!std::strcmp(argv[i], "-pz")) { + pz = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procget(path, kstr, ksiz, oflags, rm, px, pz); + delete[] kbuf; + return rv; +} + + +// parse arguments of list command +static int32_t runlist(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + int64_t max = -1; + bool rm = false; + bool sx = false; + bool pv = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-max")) { + if (++i >= argc) usage(); + max = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-pv")) { + pv = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + char* kbuf = NULL; + size_t ksiz = 0; + if (kstr) { + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = new char[ksiz+1]; + std::memcpy(kbuf, kstr, ksiz); + kbuf[ksiz] = '\0'; + } + } + int32_t rv = proclist(path, kbuf, ksiz, oflags, max, rm, pv, px); + delete[] kbuf; + return rv; +} + + +// parse arguments of clear command +static int32_t runclear(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procclear(path, oflags); + return rv; +} + + +// parse arguments of import command +static int32_t runimport(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procimport(path, file, oflags, sx); + return rv; +} + + +// parse arguments of copy command +static int32_t runcopy(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path || !file) usage(); + int32_t rv = proccopy(path, file, oflags); + return rv; +} + + +// parse arguments of dump command +static int32_t rundump(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procdump(path, file, oflags); + return rv; +} + + +// parse arguments of load command +static int32_t runload(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::DirDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procload(path, file, oflags); + return rv; +} + + +// parse arguments of setbulk command +static int32_t runsetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::map<std::string, std::string> recs; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + if (++i >= argc) usage(); + const char* vstr = argv[i]; + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + std::string key(kstr, ksiz); + std::string value(vstr, vsiz); + recs[key] = value; + delete[] kbuf; + delete[] vbuf; + } + } + if (!path) usage(); + int32_t rv = procsetbulk(path, oflags, recs); + return rv; +} + + +// parse arguments of removebulk command +static int32_t runremovebulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procremovebulk(path, oflags, keys); + return rv; +} + + +// parse arguments of getbulk command +static int32_t rungetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procgetbulk(path, oflags, keys, px); + return rv; +} + + +// parse arguments of check command +static int32_t runcheck(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccheck(path, oflags); + return rv; +} + + +// perform create command +static int32_t proccreate(const char* path, int32_t oflags, int32_t opts) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (opts > 0) db.tune_options(opts); + if (!db.open(path, kc::DirDB::OWRITER | kc::DirDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform inform command +static int32_t procinform(const char* path, int32_t oflags, bool st) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (st) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + if (db.status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t rtype = kc::atoi(status["realtype"].c_str()); + if (rtype > 0 && rtype != type) + oprintf("real type: %s (%s) (realtype=0x%02X)\n", + kc::BasicDB::typecname(rtype), kc::BasicDB::typestring(rtype), rtype); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::DirDB::FOPEN) oprintf(" open"); + if (flags & kc::DirDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + oprintf("\n", flags); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::DirDB::TSMALL) oprintf(" small"); + if (opts & kc::DirDB::TLINEAR) oprintf(" linear"); + if (opts & kc::DirDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t count = kc::atoi(status["count"].c_str()); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s)\n", size, sizestr.c_str()); + } else { + dberrprint(&db, "DB::status failed"); + err = true; + } + } else { + oprintf("count: %lld\n", (long long)db.count()); + oprintf("size: %lld\n", (long long)db.size()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform set command +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + switch (mode) { + default: { + if (!db.set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 'a': { + if (!db.add(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::add failed"); + err = true; + } + break; + } + case 'r': { + if (!db.replace(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::replace failed"); + err = true; + } + break; + } + case 'c': { + if (!db.append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::append failed"); + err = true; + } + break; + } + case 'i': { + int64_t onum = db.increment(kbuf, ksiz, kc::atoi(vbuf)); + if (onum == kc::INT64MIN) { + dberrprint(&db, "DB::increment failed"); + err = true; + } else { + oprintf("%lld\n", (long long)onum); + } + break; + } + case 'd': { + double onum = db.increment_double(kbuf, ksiz, kc::atof(vbuf)); + if (kc::chknan(onum)) { + dberrprint(&db, "DB::increment_double failed"); + err = true; + } else { + oprintf("%f\n", onum); + } + break; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform remove command +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.remove(kbuf, ksiz)) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform get command +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::DirDB::OWRITER : kc::DirDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + char* vbuf; + size_t vsiz; + if (rm) { + vbuf = db.seize(kbuf, ksiz, &vsiz); + } else { + vbuf = db.get(kbuf, ksiz, &vsiz); + } + if (vbuf) { + printdata(vbuf, vsiz, px); + if (!pz) oprintf("\n"); + delete[] vbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform list command +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + int64_t max, bool rm, bool pv, bool px) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::DirDB::OWRITER : kc::DirDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(bool rm, bool pv, bool px) : rm_(rm), pv_(pv), px_(px) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + printdata(kbuf, ksiz, px_); + if (pv_) { + oprintf("\t"); + printdata(vbuf, vsiz, px_); + } + oprintf("\n"); + return rm_ ? REMOVE : NOP; + } + bool rm_; + bool pv_; + bool px_; + } visitor(rm, pv, px); + if (kbuf || max >= 0) { + if (max < 0) max = kc::INT64MAX; + kc::DirDB::Cursor cur(&db); + if (kbuf) { + if (!cur.jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } else { + if (!db.iterate(&visitor, rm)) { + dberrprint(&db, "DB::iterate failed"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform clear command +static int32_t procclear(const char* path, int32_t oflags) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.clear()) { + dberrprint(&db, "DB::clear failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform import command +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx) { + std::istream *is = &std::cin; + std::ifstream ifs; + if (file) { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OWRITER | kc::DirDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + int64_t cnt = 0; + std::string line; + std::vector<std::string> fields; + while (!err && mygetline(is, &line)) { + cnt++; + kc::strsplit(line, '\t', &fields); + if (sx) { + std::vector<std::string>::iterator it = fields.begin(); + std::vector<std::string>::iterator itend = fields.end(); + while (it != itend) { + size_t esiz; + char* ebuf = kc::hexdecode(it->c_str(), &esiz); + it->clear(); + it->append(ebuf, esiz); + delete[] ebuf; + ++it; + } + } + switch (fields.size()) { + case 2: { + if (!db.set(fields[0], fields[1])) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 1: { + if (!db.remove(fields[0]) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + break; + } + } + oputchar('.'); + if (cnt % 50 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + if (cnt % 50 > 0) oprintf(" (%lld)\n", (long long)cnt); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform copy command +static int32_t proccopy(const char* path, const char* file, int32_t oflags) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + DotChecker checker(&std::cout, -100); + if (!db.copy(file, &checker)) { + dberrprint(&db, "DB::copy failed"); + err = true; + } + oprintf(" (end)\n"); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld blocks were copied successfully\n", (long long)checker.count()); + return err ? 1 : 0; +} + + +// perform dump command +static int32_t procdump(const char* path, const char* file, int32_t oflags) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, 1000); + if (!db.dump_snapshot(file)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were dumped successfully\n", (long long)checker.count()); + } else { + if (!db.dump_snapshot(&std::cout)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform load command +static int32_t procload(const char* path, const char* file, int32_t oflags) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OWRITER | kc::DirDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(file)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } else { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(&std::cin)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform setbulk command +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.set_bulk(recs) != (int64_t)recs.size()) { + dberrprint(&db, "DB::set_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform removebulk command +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.remove_bulk(keys) < 0) { + dberrprint(&db, "DB::remove_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform getbulk command +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + std::map<std::string, std::string> recs; + if (db.get_bulk(keys, &recs) >= 0) { + std::map<std::string, std::string>::iterator it = recs.begin(); + std::map<std::string, std::string>::iterator itend = recs.end(); + while (it != itend) { + printdata(it->first.data(), it->first.size(), px); + oprintf("\t"); + printdata(it->second.data(), it->second.size(), px); + oprintf("\n"); + ++it; + } + } else { + dberrprint(&db, "DB::get_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform check command +static int32_t proccheck(const char* path, int32_t oflags) { + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::DirDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + kc::DirDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::jump failed"); + err = true; + } + int64_t cnt = 0; + while (!err) { + size_t ksiz; + const char* vbuf; + size_t vsiz; + char* kbuf = cur.get(&ksiz, &vbuf, &vsiz); + if (kbuf) { + cnt++; + size_t rsiz; + char* rbuf = db.get(kbuf, ksiz, &rsiz); + if (rbuf) { + if (rsiz != vsiz || std::memcmp(rbuf, vbuf, rsiz)) { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] rbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] kbuf; + if (cnt % 1000 == 0) { + oputchar('.'); + if (cnt % 50000 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + } else { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::get failed"); + err = true; + } + break; + } + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::step failed"); + err = true; + } + } + oprintf(" (end)\n"); + if (db.count() != cnt) { + dberrprint(&db, "DB::count failed"); + err = true; + } + if (db.flags() & kc::DirDB::FFATAL) { + dberrprint(&db, "DB::flags indicated fatal error"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld records were checked successfully\n", (long long)cnt); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcdirtest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirtest.cc new file mode 100644 index 0000000000..36d01724f1 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcdirtest.cc @@ -0,0 +1,2245 @@ +/************************************************************************************************* + * The test cases of the directory hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcdirdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t opts, bool lv); +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + bool rnd, int32_t oflags, int32_t opts, bool lv); +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t opts, bool lv); +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t opts, bool lv); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the directory hash database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-set|-get|-getw|-rem|-etc] [-tran]" + " [-oat|-oas|-onl|-otl|-onr] [-tc] [-lv] path rnum\n", g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-oat|-oas|-onl|-otl|-onr] [-tc] [-lv]" + " path rnum\n", g_progname); + eprintf(" %s wicked [-th num] [-it num] [-oat|-oas|-onl|-otl|-onr] [-tc] [-lv]" + " path rnum\n", g_progname); + eprintf(" %s tran [-th num] [-it num] [-hard] [-oat|-oas|-onl|-otl|-onr] [-tc] [-lv]" + " path rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t rtype = kc::atoi(status["realtype"].c_str()); + if (rtype > 0 && rtype != type) + oprintf("real type: %s (%s) (realtype=0x%02X)\n", + kc::BasicDB::typecname(rtype), kc::BasicDB::typestring(rtype), rtype); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::DirDB::FOPEN) oprintf(" open"); + if (flags & kc::DirDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + oprintf("\n", flags); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::DirDB::TSMALL) oprintf(" small"); + if (opts & kc::DirDB::TLINEAR) oprintf(" linear"); + if (opts & kc::DirDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + if (status.find("opaque") != status.end()) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t count = kc::atoi(status["count"].c_str()); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s)\n", size, sizestr.c_str()); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + int32_t mode = 0; + bool tran = false; + int32_t oflags = 0; + int32_t opts = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-set")) { + mode = 's'; + } else if (!std::strcmp(argv[i], "-get")) { + mode = 'g'; + } else if (!std::strcmp(argv[i], "-getw")) { + mode = 'w'; + } else if (!std::strcmp(argv[i], "-rem")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-etc")) { + mode = 'e'; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::DirDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::DirDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::DirDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(path, rnum, thnum, rnd, mode, tran, oflags, opts, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int32_t oflags = 0; + int32_t opts = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::DirDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::DirDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::DirDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(path, rnum, thnum, itnum, rnd, oflags, opts, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t oflags = 0; + int32_t opts = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::DirDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::DirDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::DirDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(path, rnum, thnum, itnum, oflags, opts, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool hard = false; + int32_t oflags = 0; + int32_t opts = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-hard")) { + hard = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::DirDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::DirDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::DirDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::DirDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::DirDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::DirDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(path, rnum, thnum, itnum, hard, oflags, opts, lv); + return rv; +} + + +// perform order command +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t opts, bool lv) { + oprintf("<In-order Test>\n seed=%u path=%s rnum=%lld thnum=%d rnd=%d mode=%d tran=%d" + " oflags=%d opts=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, rnd, mode, tran, oflags, opts, lv); + bool err = false; + kc::DirDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + uint32_t omode = kc::DirDB::OWRITER | kc::DirDB::OCREATE | kc::DirDB::OTRUNCATE; + if (mode == 'r') { + omode = kc::DirDB::OWRITER | kc::DirDB::OCREATE; + } else if (mode == 'g' || mode == 'w') { + omode = kc::DirDB::OREADER; + } + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (mode == 0 || mode == 's' || mode == 'e') { + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 's'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + if (mode == 0 || mode == 'g' || mode == 'e') { + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'g'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'w' || mode == 'e') { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'w'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::DirDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size) : + rnum_(rnum), rnd_(rnd), size_(size) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + kc::File::Status sbuf; + if (!kc::File::status(path, &sbuf)) return false; + if (!sbuf.isdir) return false; + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + } syncprocessor(rnum, rnd, db.size()); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e' && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'r' || mode == 'e') { + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && mode_ != 'e') || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, mode, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, mode, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'r' || mode == 'e'); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + bool rnd, int32_t oflags, int32_t opts, bool lv) { + oprintf("<Queue Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d rnd=%d" + " oflags=%d opts=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, rnd, oflags, opts, lv); + bool err = false; + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::DirDB::OWRITER | kc::DirDB::OCREATE; + if (itcnt == 1) omode |= kc::DirDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::DirDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::DirDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t opts, bool lv) { + oprintf("<Wicked Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d" + " oflags=%d opts=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, oflags, opts, lv); + bool err = false; + kc::DirDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::DirDB::OWRITER | kc::DirDB::OCREATE; + if (itcnt == 1) omode |= kc::DirDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::DirDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(5) > 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::DirDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t opts, bool lv) { + oprintf("<Transaction Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d hard=%d" + " oflags=%d opts=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, hard, oflags, opts, lv); + bool err = false; + kc::DirDB db; + kc::DirDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::DirDB::OWRITER | kc::DirDB::OCREATE; + if (itcnt == 1) omode |= kc::DirDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + std::string parapath = db.path() + "-para"; + if (!paradb.open(parapath, omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::DirDB* db, kc::DirDB* paradb, int64_t rnum, + int32_t thnum, bool hard, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + hard_ = hard; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::DirDB* db_; + kc::DirDB* paradb_; + int64_t rnum_; + int32_t thnum_; + bool hard_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, hard, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, hard, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcfile.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcfile.cc new file mode 100644 index 0000000000..ce7f93583f --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcfile.cc @@ -0,0 +1,2712 @@ +/************************************************************************************************* + * Filesystem abstraction + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcfile.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +/** + * Constants for implementation. + */ +namespace { +const int32_t FILEPERM = 00644; ///< default permission of a new file +const int32_t DIRPERM = 00755; ///< default permission of a new directory +const int32_t PATHBUFSIZ = 8192; ///< size of the path buffer +const int32_t IOBUFSIZ = 16384; ///< size of the IO buffer +const int64_t FILEMAXSIZ = INT64MAX - INT32MAX; // maximum size of a file +const char* const WALPATHEXT = "wal"; ///< extension of the WAL file +const char WALMAGICDATA[] = "KW\n"; ///< magic data of the WAL file +const uint8_t WALMSGMAGIC = 0xee; ///< magic data for WAL record +} + + +/** + * File internal. + */ +struct FileCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + Mutex alock; ///< attribute lock + TSDKey errmsg; ///< error message + ::HANDLE fh; ///< file handle + ::HANDLE mh; ///< map view handle + char* map; ///< mapped memory + int64_t msiz; ///< map size + int64_t lsiz; ///< logical size + int64_t psiz; ///< physical size + std::string path; ///< file path + bool recov; ///< flag of recovery + uint32_t omode; ///< open mode + ::HANDLE walfh; ///< file handle for WAL + int64_t walsiz; ///< size of WAL + bool tran; ///< whether in transaction + bool trhard; ///< whether hard transaction + int64_t trbase; ///< base offset of guarded region + int64_t trmsiz; ///< minimum size during transaction +#else + Mutex alock; ///< attribute lock + TSDKey errmsg; ///< error message + int32_t fd; ///< file descriptor + char* map; ///< mapped memory + int64_t msiz; ///< map size + int64_t lsiz; ///< logical size + int64_t psiz; ///< physical size + std::string path; ///< file path + bool recov; ///< flag of recovery + uint32_t omode; ///< open mode + int32_t walfd; ///< file descriptor for WAL + int64_t walsiz; ///< size of WAL + bool tran; ///< whether in transaction + bool trhard; ///< whether hard transaction + int64_t trbase; ///< base offset of guarded region + int64_t trmsiz; ///< minimum size during transaction +#endif +}; + + +/** + * WAL message. + */ +struct WALMessage { + int64_t off; ///< offset of the region + std::string body; ///< body data +}; + + +/** + * DirStream internal. + */ +struct DirStreamCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + Mutex alock; ///< attribute lock + ::HANDLE dh; ///< directory handle + std::string cur; ///< current file +#else + Mutex alock; ///< attribute lock + ::DIR* dh; ///< directory handle +#endif +}; + + +/** + * Set the error message. + * @param core the inner condition. + * @param msg the error message. + */ +static void seterrmsg(FileCore* core, const char* msg); + + +/** + * Get the path of the WAL file. + * @param path the path of the destination file. + * @return the path of the WAL file. + */ +static std::string walpath(const std::string& path); + + +/** + * Write a log message into the WAL file. + * @param core the inner condition. + * @param off the offset of the destination. + * @param size the size of the data region. + * @param base the base offset. + * @return true on success, or false on failure. + */ +static bool walwrite(FileCore *core, int64_t off, size_t size, int64_t base); + + +/** + * Apply log messages in the WAL file. + * @param core the inner condition. + * @return true on success, or false on failure. + */ +static bool walapply(FileCore* core); + + +/** + * Write data into a file. + * @param fd the file descriptor. + * @param off the offset of the destination. + * @param buf the pointer to the data region. + * @param size the size of the data region. + * @return true on success, or false on failure. + */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static bool mywrite(::HANDLE fh, int64_t off, const void* buf, size_t size); +#else +static bool mywrite(int32_t fd, int64_t off, const void* buf, size_t size); +#endif + + +/** + * Read data from a file. + * @param fd the file descriptor. + * @param buf the pointer to the destination region. + * @param size the size of the data to be read. + * @return true on success, or false on failure. + */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static size_t myread(::HANDLE fh, void* buf, size_t size); +#else +static size_t myread(int32_t fd, void* buf, size_t count); +#endif + + +/** + * System call emulation for Win32. + */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static int64_t win_pwrite(::HANDLE fh, const void* buf, size_t count, int64_t offset); +static int64_t win_pread(::HANDLE fh, void* buf, size_t count, int64_t offset); +static int64_t win_write(::HANDLE fh, const void* buf, size_t count); +static int64_t win_read(::HANDLE fh, void* buf, size_t count); +static int32_t win_ftruncate(::HANDLE fh, int64_t length); +#endif + + +/** Path delimiter character. */ +const char File::PATHCHR = MYPATHCHR; + + +/** Path delimiter string. */ +const char* const File::PATHSTR = MYPATHSTR; + + +/** Extension delimiter character. */ +const char File::EXTCHR = MYEXTCHR; + + +/** Extension delimiter string. */ +const char* const File::EXTSTR = MYEXTSTR; + + +/** Current directory string. */ +const char* const File::CDIRSTR = MYCDIRSTR; + + +/** Parent directory string. */ +const char* const File::PDIRSTR = MYPDIRSTR; + + +/** + * Default constructor. + */ +File::File() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + FileCore* core = new FileCore; + core->fh = NULL; + core->mh = NULL; + core->map = NULL; + core->msiz = 0; + core->lsiz = 0; + core->psiz = 0; + core->recov = false; + core->omode = 0; + core->walfh = NULL; + core->walsiz = 0; + core->tran = false; + core->trhard = false; + core->trmsiz = 0; + opq_ = core; +#else + _assert_(true); + FileCore* core = new FileCore; + core->fd = -1; + core->map = NULL; + core->msiz = 0; + core->lsiz = 0; + core->psiz = 0; + core->recov = false; + core->omode = 0; + core->walfd = -1; + core->walsiz = 0; + core->tran = false; + core->trhard = false; + core->trmsiz = 0; + opq_ = core; +#endif +} + + +/** + * Destructor. + */ +File::~File() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + FileCore* core = (FileCore*)opq_; + if (core->fh) close(); + delete core; +#else + _assert_(true); + FileCore* core = (FileCore*)opq_; + if (core->fd >= 0) close(); + delete core; +#endif +} + + +/** + * Get the last happened error information. + */ +const char* File::error() const { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + FileCore* core = (FileCore*)opq_; + const char* msg = (const char*)core->errmsg.get(); + if (!msg) msg = "no error"; + return msg; +#else + _assert_(true); + FileCore* core = (FileCore*)opq_; + const char* msg = (const char*)core->errmsg.get(); + if (!msg) msg = "no error"; + return msg; +#endif +} + + +/** + * Open a file. + */ +bool File::open(const std::string& path, uint32_t mode, int64_t msiz) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(msiz >= 0 && msiz <= FILEMAXSIZ); + FileCore* core = (FileCore*)opq_; + ::DWORD amode = GENERIC_READ; + ::DWORD smode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + ::DWORD cmode = OPEN_EXISTING; + if (mode & OWRITER) { + amode |= GENERIC_WRITE; + if (mode & OCREATE) { + cmode = OPEN_ALWAYS; + if (mode & OTRUNCATE) cmode = CREATE_ALWAYS; + } else { + if (mode & OTRUNCATE) cmode = TRUNCATE_EXISTING; + } + } + ::HANDLE fh = ::CreateFileA(path.c_str(), amode, smode, NULL, cmode, + FILE_ATTRIBUTE_NORMAL, NULL); + if (!fh || fh == INVALID_HANDLE_VALUE) { + seterrmsg(core, "CreateFile failed"); + return false; + } + if (!(mode & ONOLOCK)) { + ::DWORD lmode = mode & OWRITER ? LOCKFILE_EXCLUSIVE_LOCK : 0; + if (mode & OTRYLOCK) lmode |= LOCKFILE_FAIL_IMMEDIATELY; + OVERLAPPED ol; + ol.Offset = INT32MAX; + ol.OffsetHigh = 0; + ol.hEvent = 0; + if (!::LockFileEx(fh, lmode, 0, 1, 0, &ol)) { + seterrmsg(core, "LockFileEx failed"); + ::CloseHandle(fh); + return false; + } + } + ::LARGE_INTEGER sbuf; + if (!::GetFileSizeEx(fh, &sbuf)) { + seterrmsg(core, "GetFileSizeEx failed"); + ::CloseHandle(fh); + return false; + } + bool recov = false; + if ((!(mode & OWRITER) || !(mode & OTRUNCATE)) && !(mode & ONOLOCK)) { + const std::string& wpath = walpath(path); + ::HANDLE walfh = ::CreateFileA(wpath.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (walfh && walfh != INVALID_HANDLE_VALUE) { + recov = true; + ::LARGE_INTEGER li; + if (::GetFileSizeEx(walfh, &li) && li.QuadPart >= (int64_t)sizeof(WALMAGICDATA)) { + char mbuf[sizeof(WALMAGICDATA)]; + if (myread(walfh, mbuf, sizeof(mbuf)) && + !std::memcmp(mbuf, WALMAGICDATA, sizeof(WALMAGICDATA))) { + ::HANDLE ofh = fh; + if (!(mode & OWRITER)) ofh = ::CreateFileA(wpath.c_str(), GENERIC_WRITE, 0, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (ofh && ofh != INVALID_HANDLE_VALUE) { + core->fh = ofh; + core->walfh = walfh; + walapply(core); + if (ofh != fh && !::CloseHandle(ofh)) seterrmsg(core, "CloseHandle failed"); + li.QuadPart = 0; + if (win_ftruncate(walfh, 0) != 0) seterrmsg(core, "win_ftruncate failed"); + core->fh = NULL; + core->walfh = NULL; + if (!::GetFileSizeEx(fh, &sbuf)) { + seterrmsg(core, "GetFileSizeEx failed"); + ::CloseHandle(fh); + return false; + } + } else { + seterrmsg(core, "CreateFile failed"); + } + } + } + if (!::CloseHandle(walfh)) seterrmsg(core, "CloseHandle failed"); + ::DeleteFileA(wpath.c_str()); + } + } + int64_t lsiz = sbuf.QuadPart; + int64_t psiz = lsiz; + int64_t diff = msiz % PAGESIZ; + if (diff > 0) msiz += PAGESIZ - diff; + ::DWORD mprot = PAGE_READONLY; + ::DWORD vmode = FILE_MAP_READ; + if (mode & OWRITER) { + mprot = PAGE_READWRITE; + vmode = FILE_MAP_WRITE; + } else if (msiz > lsiz) { + msiz = lsiz; + } + sbuf.QuadPart = msiz; + ::HANDLE mh = NULL; + void* map = NULL; + if (msiz > 0) { + mh = ::CreateFileMapping(fh, NULL, mprot, sbuf.HighPart, sbuf.LowPart, NULL); + if (!mh || mh == INVALID_HANDLE_VALUE) { + seterrmsg(core, "CreateFileMapping failed"); + ::CloseHandle(fh); + return false; + } + map = ::MapViewOfFile(mh, vmode, 0, 0, 0); + if (!map) { + seterrmsg(core, "MapViewOfFile failed"); + ::CloseHandle(mh); + ::CloseHandle(fh); + return false; + } + if (psiz < msiz) psiz = msiz; + } + core->fh = fh; + core->mh = mh; + core->map = (char*)map; + core->msiz = msiz; + core->lsiz = lsiz; + core->psiz = psiz; + core->recov = recov; + core->omode = mode; + core->path.append(path); + return true; +#else + _assert_(msiz >= 0 && msiz <= FILEMAXSIZ); + FileCore* core = (FileCore*)opq_; + int32_t oflags = O_RDONLY; + if (mode & OWRITER) { + oflags = O_RDWR; + if (mode & OCREATE) oflags |= O_CREAT; + if (mode & OTRUNCATE) oflags |= O_TRUNC; + } + int32_t fd = ::open(path.c_str(), oflags, FILEPERM); + if (fd < 0) { + switch (errno) { + case EACCES: seterrmsg(core, "open failed (permission denied)"); break; + case EISDIR: seterrmsg(core, "open failed (directory)"); break; + case ENOENT: seterrmsg(core, "open failed (file not found)"); break; + case ENOTDIR: seterrmsg(core, "open failed (invalid path)"); break; + case ENOSPC: seterrmsg(core, "open failed (no space)"); break; + default: seterrmsg(core, "open failed"); break; + } + return false; + } + if (!(mode & ONOLOCK)) { + struct flock flbuf; + std::memset(&flbuf, 0, sizeof(flbuf)); + flbuf.l_type = mode & OWRITER ? F_WRLCK : F_RDLCK; + flbuf.l_whence = SEEK_SET; + flbuf.l_start = 0; + flbuf.l_len = 0; + flbuf.l_pid = 0; + int32_t cmd = mode & OTRYLOCK ? F_SETLK : F_SETLKW; + while (::fcntl(fd, cmd, &flbuf) != 0) { + if (errno != EINTR) { + seterrmsg(core, "fcntl failed"); + ::close(fd); + return false; + } + } + } + struct ::stat sbuf; + if (::fstat(fd, &sbuf) != 0) { + seterrmsg(core, "fstat failed"); + ::close(fd); + return false; + } + if (!S_ISREG(sbuf.st_mode)) { + seterrmsg(core, "not a regular file"); + ::close(fd); + return false; + } + bool recov = false; + if ((!(mode & OWRITER) || !(mode & OTRUNCATE)) && !(mode & ONOLOCK)) { + const std::string& wpath = walpath(path); + int32_t walfd = ::open(wpath.c_str(), O_RDWR, FILEPERM); + if (walfd >= 0) { + struct ::stat wsbuf; + if (::fstat(walfd, &wsbuf) == 0 && wsbuf.st_uid == sbuf.st_uid) { + recov = true; + if (wsbuf.st_size >= (int64_t)sizeof(WALMAGICDATA)) { + char mbuf[sizeof(WALMAGICDATA)]; + if (myread(walfd, mbuf, sizeof(mbuf)) && + !std::memcmp(mbuf, WALMAGICDATA, sizeof(WALMAGICDATA))) { + int32_t ofd = mode & OWRITER ? fd : ::open(path.c_str(), O_WRONLY, FILEPERM); + if (ofd >= 0) { + core->fd = ofd; + core->walfd = walfd; + walapply(core); + if (ofd != fd && ::close(ofd) != 0) seterrmsg(core, "close failed"); + if (::ftruncate(walfd, 0) != 0) seterrmsg(core, "ftruncate failed"); + core->fd = -1; + core->walfd = -1; + if (::fstat(fd, &sbuf) != 0) { + seterrmsg(core, "fstat failed"); + ::close(fd); + return false; + } + } else { + seterrmsg(core, "open failed"); + } + } + } + } + if (::close(walfd) != 0) seterrmsg(core, "close failed"); + if (::unlink(wpath.c_str()) != 0) seterrmsg(core, "unlink failed"); + } + } + int64_t lsiz = sbuf.st_size; + int64_t psiz = lsiz; + int64_t diff = msiz % PAGESIZ; + if (diff > 0) msiz += PAGESIZ - diff; + int32_t mprot = PROT_READ; + if (mode & OWRITER) { + mprot |= PROT_WRITE; + } else if (msiz > lsiz) { + msiz = lsiz; + } + void* map = NULL; + if (msiz > 0) { + map = ::mmap(0, msiz, mprot, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + seterrmsg(core, "mmap failed"); + ::close(fd); + return false; + } + } + core->fd = fd; + core->map = (char*)map; + core->msiz = msiz; + core->lsiz = lsiz; + core->psiz = psiz; + core->recov = recov; + core->omode = mode; + core->path.append(path); + return true; +#endif +} + + +/** + * Close the file. + */ +bool File::close() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + FileCore* core = (FileCore*)opq_; + bool err = false; + if (core->tran && !end_transaction(false)) err = true; + if (core->walfh) { + if (!::CloseHandle(core->walfh)) { + seterrmsg(core, "CloseHandle failed"); + err = true; + } + const std::string& wpath = walpath(core->path); + ::DeleteFileA(wpath.c_str()); + } + if (core->msiz > 0) { + if (!::UnmapViewOfFile(core->map)) { + seterrmsg(core, "UnmapViewOfFile failed"); + err = true; + } + if (!::CloseHandle(core->mh)) { + seterrmsg(core, "CloseHandle failed"); + err = true; + } + } + ::LARGE_INTEGER li; + if (::GetFileSizeEx(core->fh, &li)) { + if ((li.QuadPart != core->lsiz || core->psiz != core->lsiz) && + win_ftruncate(core->fh, core->lsiz) != 0) { + seterrmsg(core, "win_ftruncate failed"); + err = true; + } + } else { + seterrmsg(core, "GetFileSizeEx failed"); + err = true; + } + if (!(core->omode & ONOLOCK)) { + OVERLAPPED ol; + ol.Offset = INT32MAX; + ol.OffsetHigh = 0; + ol.hEvent = 0; + if (!::UnlockFileEx(core->fh, 0, 1, 0, &ol)) { + seterrmsg(core, "UnlockFileEx failed"); + err = true; + } + } + if (!::CloseHandle(core->fh)) { + seterrmsg(core, "CloseHandle failed"); + err = true; + } + core->fh = NULL; + core->mh = NULL; + core->map = NULL; + core->msiz = 0; + core->lsiz = 0; + core->psiz = 0; + core->path.clear(); + core->walfh = NULL; + core->walsiz = 0; + core->tran = false; + core->trhard = false; + core->trmsiz = 0; + return !err; +#else + _assert_(true); + FileCore* core = (FileCore*)opq_; + bool err = false; + if (core->tran && !end_transaction(false)) err = true; + if (core->walfd >= 0) { + if (::close(core->walfd) != 0) { + seterrmsg(core, "close failed"); + err = true; + } + const std::string& wpath = walpath(core->path); + struct ::stat sbuf; + if (::lstat(wpath.c_str(), &sbuf) == 0 && S_ISREG(sbuf.st_mode) && + ::unlink(wpath.c_str()) != 0) { + seterrmsg(core, "unlink failed"); + err = true; + } + } + if (core->msiz > 0 && ::munmap(core->map, core->msiz) != 0) { + seterrmsg(core, "munmap failed"); + err = true; + } + if (core->psiz != core->lsiz && ::ftruncate(core->fd, core->lsiz) != 0) { + seterrmsg(core, "ftruncate failed"); + err = true; + } + if (!(core->omode & ONOLOCK)) { + struct flock flbuf; + std::memset(&flbuf, 0, sizeof(flbuf)); + flbuf.l_type = F_UNLCK; + flbuf.l_whence = SEEK_SET; + flbuf.l_start = 0; + flbuf.l_len = 0; + flbuf.l_pid = 0; + while (::fcntl(core->fd, F_SETLKW, &flbuf) != 0) { + if (errno != EINTR) { + seterrmsg(core, "fcntl failed"); + err = true; + break; + } + } + } + if (::close(core->fd) != 0) { + seterrmsg(core, "close failed"); + err = true; + } + core->fd = -1; + core->map = NULL; + core->msiz = 0; + core->lsiz = 0; + core->psiz = 0; + core->path.clear(); + core->walfd = -1; + core->walsiz = 0; + core->tran = false; + core->trhard = false; + core->trmsiz = 0; + return !err; +#endif +} + + +/** + * Write data. + */ +bool File::write(int64_t off, const void* buf, size_t size) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + if (size < 1) return true; + FileCore* core = (FileCore*)opq_; + if (core->tran && !walwrite(core, off, size, core->trbase)) return false; + int64_t end = off + size; + core->alock.lock(); + if (end <= core->msiz) { + if (end > core->psiz) { + int64_t psiz = end + core->psiz / 2; + int64_t diff = psiz % PAGESIZ; + if (diff > 0) psiz += PAGESIZ - diff; + if (psiz > core->msiz) psiz = core->msiz; + if (win_ftruncate(core->fh, psiz) != 0) { + seterrmsg(core, "win_ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = psiz; + } + if (end > core->lsiz) core->lsiz = end; + core->alock.unlock(); + std::memcpy(core->map + off, buf, size); + return true; + } + if (off < core->msiz) { + if (end > core->psiz) { + if (win_ftruncate(core->fh, end) != 0) { + seterrmsg(core, "win_ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = end; + } + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, buf, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + if (end > core->lsiz) core->lsiz = end; + if (end > core->psiz) { + if (core->psiz < core->msiz && win_ftruncate(core->fh, core->msiz) != 0) { + seterrmsg(core, "win_ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = end; + } + core->alock.unlock(); + if (!mywrite(core->fh, off, buf, size)) { + seterrmsg(core, "mywrite failed"); + return false; + } + return true; +#else + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + if (size < 1) return true; + FileCore* core = (FileCore*)opq_; + if (core->tran && !walwrite(core, off, size, core->trbase)) return false; + int64_t end = off + size; + core->alock.lock(); + if (end <= core->msiz) { + if (end > core->psiz) { + int64_t psiz = end + core->psiz / 2; + int64_t diff = psiz % PAGESIZ; + if (diff > 0) psiz += PAGESIZ - diff; + if (psiz > core->msiz) psiz = core->msiz; + if (::ftruncate(core->fd, psiz) != 0) { + seterrmsg(core, "ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = psiz; + } + if (end > core->lsiz) core->lsiz = end; + core->alock.unlock(); + std::memcpy(core->map + off, buf, size); + return true; + } + if (off < core->msiz) { + if (end > core->psiz) { + if (::ftruncate(core->fd, end) != 0) { + seterrmsg(core, "ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = end; + } + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, buf, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + if (end > core->lsiz) core->lsiz = end; + if (end > core->psiz) { + if (core->psiz < core->msiz && ::ftruncate(core->fd, core->msiz) != 0) { + seterrmsg(core, "ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = end; + } + core->alock.unlock(); + if (!mywrite(core->fd, off, buf, size)) { + seterrmsg(core, "mywrite failed"); + return false; + } + return true; +#endif +} + + +/** + * Write data with assuring the region does not spill from the file size. + */ +bool File::write_fast(int64_t off, const void* buf, size_t size) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + FileCore* core = (FileCore*)opq_; + if (core->tran && !walwrite(core, off, size, core->trbase)) return false; + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(core->map + off, buf, size); + return true; + } + if (off < core->msiz) { + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, buf, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + if (!mywrite(core->fh, off, buf, size)) { + seterrmsg(core, "mywrite failed"); + return false; + } + return true; +#else + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + FileCore* core = (FileCore*)opq_; + if (core->tran && !walwrite(core, off, size, core->trbase)) return false; + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(core->map + off, buf, size); + return true; + } + if (off < core->msiz) { + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, buf, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + if (!mywrite(core->fd, off, buf, size)) { + seterrmsg(core, "mywrite failed"); + return false; + } + return true; +#endif +} + + +/** + * Write data at the end of the file. + */ +bool File::append(const void* buf, size_t size) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(buf && size <= MEMMAXSIZ); + if (size < 1) return true; + FileCore* core = (FileCore*)opq_; + core->alock.lock(); + int64_t off = core->lsiz; + int64_t end = off + size; + if (end <= core->msiz) { + if (end > core->psiz) { + int64_t psiz = end + core->psiz / 2; + int64_t diff = psiz % PAGESIZ; + if (diff > 0) psiz += PAGESIZ - diff; + if (psiz > core->msiz) psiz = core->msiz; + if (win_ftruncate(core->fh, psiz) != 0) { + seterrmsg(core, "win_ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = psiz; + } + core->lsiz = end; + core->alock.unlock(); + std::memcpy(core->map + off, buf, size); + return true; + } + if (off < core->msiz) { + if (end > core->psiz) { + if (win_ftruncate(core->fh, end) != 0) { + seterrmsg(core, "win_ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = end; + } + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, buf, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + core->lsiz = end; + core->psiz = end; + core->alock.unlock(); + while (true) { + int64_t wb = win_pwrite(core->fh, buf, size, off); + if (wb >= (int64_t)size) { + return true; + } else if (wb > 0) { + buf = (char*)buf + wb; + size -= wb; + off += wb; + } else if (wb == -1) { + seterrmsg(core, "win_pwrite failed"); + return false; + } else if (size > 0) { + seterrmsg(core, "win_pwrite failed"); + return false; + } + } + return true; +#else + _assert_(buf && size <= MEMMAXSIZ); + if (size < 1) return true; + FileCore* core = (FileCore*)opq_; + core->alock.lock(); + int64_t off = core->lsiz; + int64_t end = off + size; + if (end <= core->msiz) { + if (end > core->psiz) { + int64_t psiz = end + core->psiz / 2; + int64_t diff = psiz % PAGESIZ; + if (diff > 0) psiz += PAGESIZ - diff; + if (psiz > core->msiz) psiz = core->msiz; + if (::ftruncate(core->fd, psiz) != 0) { + seterrmsg(core, "ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = psiz; + } + core->lsiz = end; + core->alock.unlock(); + std::memcpy(core->map + off, buf, size); + return true; + } + if (off < core->msiz) { + if (end > core->psiz) { + if (::ftruncate(core->fd, end) != 0) { + seterrmsg(core, "ftruncate failed"); + core->alock.unlock(); + return false; + } + core->psiz = end; + } + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, buf, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + core->lsiz = end; + core->psiz = end; + core->alock.unlock(); + while (true) { + ssize_t wb = ::pwrite(core->fd, buf, size, off); + if (wb >= (ssize_t)size) { + return true; + } else if (wb > 0) { + buf = (char*)buf + wb; + size -= wb; + off += wb; + } else if (wb == -1) { + if (errno != EINTR) { + seterrmsg(core, "pwrite failed"); + return false; + } + } else if (size > 0) { + seterrmsg(core, "pwrite failed"); + return false; + } + } + return true; +#endif +} + + +/** + * Read data. + */ +bool File::read(int64_t off, void* buf, size_t size) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + if (size < 1) return true; + FileCore* core = (FileCore*)opq_; + int64_t end = off + size; + core->alock.lock(); + if (end > core->lsiz) { + seterrmsg(core, "out of bounds"); + core->alock.unlock(); + return false; + } + core->alock.unlock(); + if (end <= core->msiz) { + std::memcpy(buf, core->map + off, size); + return true; + } + if (off < core->msiz) { + int64_t hsiz = core->msiz - off; + std::memcpy(buf, core->map + off, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + while (true) { + int64_t rb = win_pread(core->fh, buf, size, off); + if (rb >= (int64_t)size) { + break; + } else if (rb > 0) { + buf = (char*)buf + rb; + size -= rb; + off += rb; + } else if (rb == -1) { + seterrmsg(core, "win_pread failed"); + return false; + } else if (size > 0) { + Thread::yield(); + } + } + return true; +#else + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + if (size < 1) return true; + FileCore* core = (FileCore*)opq_; + int64_t end = off + size; + core->alock.lock(); + if (end > core->lsiz) { + seterrmsg(core, "out of bounds"); + core->alock.unlock(); + return false; + } + core->alock.unlock(); + if (end <= core->msiz) { + std::memcpy(buf, core->map + off, size); + return true; + } + if (off < core->msiz) { + int64_t hsiz = core->msiz - off; + std::memcpy(buf, core->map + off, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + while (true) { + ssize_t rb = ::pread(core->fd, buf, size, off); + if (rb >= (ssize_t)size) { + break; + } else if (rb > 0) { + buf = (char*)buf + rb; + size -= rb; + off += rb; + } else if (rb == -1) { + if (errno != EINTR) { + seterrmsg(core, "pread failed"); + return false; + } + } else if (size > 0) { + Thread::yield(); + } + } + return true; +#endif +} + + +/** + * Read data with assuring the region does not spill from the file size. + */ +bool File::read_fast(int64_t off, void* buf, size_t size) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + FileCore* core = (FileCore*)opq_; + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(buf, core->map + off, size); + return true; + } + if (off < core->msiz) { + int64_t hsiz = core->msiz - off; + std::memcpy(buf, core->map + off, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + while (true) { + int64_t rb = win_pread(core->fh, buf, size, off); + if (rb >= (int64_t)size) { + break; + } else if (rb > 0) { + buf = (char*)buf + rb; + size -= rb; + off += rb; + Thread::yield(); + } else if (rb == -1) { + seterrmsg(core, "win_pread failed"); + return false; + } else if (size > 0) { + if (end > core->lsiz) { + seterrmsg(core, "out of bounds"); + return false; + } + Thread::yield(); + } + } + return true; +#else + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + FileCore* core = (FileCore*)opq_; + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(buf, core->map + off, size); + return true; + } + if (off < core->msiz) { + int64_t hsiz = core->msiz - off; + std::memcpy(buf, core->map + off, hsiz); + off += hsiz; + buf = (char*)buf + hsiz; + size -= hsiz; + } + while (true) { + ssize_t rb = ::pread(core->fd, buf, size, off); + if (rb >= (ssize_t)size) { + break; + } else if (rb > 0) { + buf = (char*)buf + rb; + size -= rb; + off += rb; + Thread::yield(); + } else if (rb == -1) { + if (errno != EINTR) { + seterrmsg(core, "pread failed"); + return false; + } + } else if (size > 0) { + if (end > core->lsiz) { + seterrmsg(core, "out of bounds"); + return false; + } + Thread::yield(); + } + } + return true; +#endif +} + + +/** + * Truncate the file. + */ +bool File::truncate(int64_t size) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(size >= 0 && size <= FILEMAXSIZ); + FileCore* core = (FileCore*)opq_; + if (core->tran && size < core->trmsiz) { + if (!walwrite(core, size, core->trmsiz - size, core->trbase)) return false; + core->trmsiz = size; + } + bool err = false; + core->alock.lock(); + if (core->msiz > 0) { + if (!::UnmapViewOfFile(core->map)) { + seterrmsg(core, "UnmapViewOfFile failed"); + err = true; + } + if (!::CloseHandle(core->mh)) { + seterrmsg(core, "CloseHandle failed"); + err = true; + } + } + if (win_ftruncate(core->fh, size) != 0) { + seterrmsg(core, "win_ftruncate failed"); + err = true; + } + if (core->msiz) { + ::LARGE_INTEGER li; + li.QuadPart = core->msiz; + ::HANDLE mh = ::CreateFileMapping(core->fh, NULL, PAGE_READWRITE, + li.HighPart, li.LowPart, NULL); + if (mh && mh != INVALID_HANDLE_VALUE) { + void* map = ::MapViewOfFile(mh, FILE_MAP_WRITE, 0, 0, 0); + if (map) { + core->mh = mh; + core->map = (char*)map; + } else { + seterrmsg(core, "MapViewOfFile failed"); + ::CloseHandle(mh); + core->mh = NULL; + core->map = NULL; + core->msiz = 0; + err = true; + } + } else { + seterrmsg(core, "CreateFileMapping failed"); + core->mh = NULL; + core->map = NULL; + core->msiz = 0; + err = true; + } + } + core->lsiz = size; + core->psiz = size; + core->alock.unlock(); + return !err; +#else + _assert_(size >= 0 && size <= FILEMAXSIZ); + FileCore* core = (FileCore*)opq_; + if (core->tran && size < core->trmsiz) { + if (!walwrite(core, size, core->trmsiz - size, core->trbase)) return false; + core->trmsiz = size; + } + bool err = false; + core->alock.lock(); + if (::ftruncate(core->fd, size) != 0) { + seterrmsg(core, "ftruncate failed"); + err = true; + } + core->lsiz = size; + core->psiz = size; + core->alock.unlock(); + return !err; +#endif +} + + +/** + * Synchronize updated contents with the file and the device. + */ +bool File::synchronize(bool hard) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + FileCore* core = (FileCore*)opq_; + bool err = false; + core->alock.lock(); + if (hard && core->msiz > 0) { + int64_t msiz = core->msiz; + if (msiz > core->psiz) msiz = core->psiz; + if (msiz > 0 && !::FlushViewOfFile(core->map, msiz)) { + seterrmsg(core, "FlushViewOfFile failed"); + err = true; + } + } + if (win_ftruncate(core->fh, core->lsiz) != 0) { + seterrmsg(core, "win_ftruncate failed"); + err = true; + } + if (core->psiz > core->lsiz) core->psiz = core->lsiz; + if (hard && !::FlushFileBuffers(core->fh)) { + seterrmsg(core, "FlushFileBuffers failed"); + err = true; + } + core->alock.unlock(); + return !err; +#else + _assert_(true); + FileCore* core = (FileCore*)opq_; + bool err = false; + core->alock.lock(); + if (hard && core->msiz > 0) { + int64_t msiz = core->msiz; + if (msiz > core->psiz) msiz = core->psiz; + if (msiz > 0 && ::msync(core->map, msiz, MS_SYNC) != 0) { + seterrmsg(core, "msync failed"); + err = true; + } + } + if (::ftruncate(core->fd, core->lsiz) != 0) { + seterrmsg(core, "ftruncate failed"); + err = true; + } + if (core->psiz > core->lsiz) core->psiz = core->lsiz; + if (hard && ::fsync(core->fd) != 0) { + seterrmsg(core, "fsync failed"); + err = true; + } + core->alock.unlock(); + return !err; +#endif +} + + +/** + * Refresh the internal state for update by others. + */ +bool File::refresh() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + FileCore* core = (FileCore*)opq_; + ::LARGE_INTEGER sbuf; + if (!::GetFileSizeEx(core->fh, &sbuf)) { + seterrmsg(core, "GetFileSizeEx failed"); + return false; + } + core->lsiz = sbuf.QuadPart; + core->psiz = sbuf.QuadPart; + return true; +#else + _assert_(true); + FileCore* core = (FileCore*)opq_; + struct ::stat sbuf; + if (::fstat(core->fd, &sbuf) != 0) { + seterrmsg(core, "fstat failed"); + return false; + } + core->lsiz = sbuf.st_size; + core->psiz = sbuf.st_size; + bool err = false; + int64_t msiz = core->msiz; + if (msiz > core->psiz) msiz = core->psiz; + if (msiz > 0 && ::msync(core->map, msiz, MS_INVALIDATE) != 0) { + seterrmsg(core, "msync failed"); + err = true; + } + return !err; +#endif +} + + +/** + * Begin transaction. + */ +bool File::begin_transaction(bool hard, int64_t off) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(off >= 0 && off <= FILEMAXSIZ); + FileCore* core = (FileCore*)opq_; + core->alock.lock(); + if (!core->walfh) { + const std::string& wpath = walpath(core->path); + ::DWORD amode = GENERIC_READ | GENERIC_WRITE; + ::DWORD smode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + ::HANDLE fh = ::CreateFileA(wpath.c_str(), amode, smode, NULL, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, NULL); + if (!fh || fh == INVALID_HANDLE_VALUE) { + seterrmsg(core, "CreateFile failed"); + core->alock.unlock(); + return false; + } + if (hard && !::FlushFileBuffers(fh)) { + seterrmsg(core, "FlushFileBuffers failed"); + ::CloseHandle(fh); + core->alock.unlock(); + return false; + } + core->walfh = fh; + } + char wbuf[NUMBUFSIZ]; + char* wp = wbuf; + std::memcpy(wp, WALMAGICDATA, sizeof(WALMAGICDATA)); + wp += sizeof(WALMAGICDATA); + int64_t num = hton64(core->lsiz); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + int64_t wsiz = wp - wbuf; + if (!mywrite(core->walfh, 0, wbuf, wsiz)) { + seterrmsg(core, "mywrite failed"); + core->alock.unlock(); + return false; + } + core->walsiz = wsiz; + core->tran = true; + core->trhard = hard; + core->trbase = off; + core->trmsiz = core->lsiz; + core->alock.unlock(); + return true; +#else + _assert_(off >= 0 && off <= FILEMAXSIZ); + FileCore* core = (FileCore*)opq_; + core->alock.lock(); + if (core->walfd < 0) { + const std::string& wpath = walpath(core->path); + int32_t fd = ::open(wpath.c_str(), O_RDWR | O_CREAT | O_TRUNC, FILEPERM); + if (fd < 0) { + switch (errno) { + case EACCES: seterrmsg(core, "open failed (permission denied)"); break; + case ENOENT: seterrmsg(core, "open failed (file not found)"); break; + case ENOTDIR: seterrmsg(core, "open failed (invalid path)"); break; + default: seterrmsg(core, "open failed"); break; + } + core->alock.unlock(); + return false; + } + core->walfd = fd; + } + char wbuf[NUMBUFSIZ]; + char* wp = wbuf; + std::memcpy(wp, WALMAGICDATA, sizeof(WALMAGICDATA)); + wp += sizeof(WALMAGICDATA); + int64_t num = hton64(core->lsiz); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + int64_t wsiz = wp - wbuf; + if (!mywrite(core->walfd, 0, wbuf, wsiz)) { + seterrmsg(core, "mywrite failed"); + core->alock.unlock(); + return false; + } + core->walsiz = wsiz; + core->tran = true; + core->trhard = hard; + core->trbase = off; + core->trmsiz = core->lsiz; + core->alock.unlock(); + return true; +#endif +} + + +/** + * Commit transaction. + */ +bool File::end_transaction(bool commit) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + FileCore* core = (FileCore*)opq_; + bool err = false; + core->alock.lock(); + if (!commit && !walapply(core)) err = true; + if (!err) { + if (core->walsiz <= IOBUFSIZ) { + char mbuf[IOBUFSIZ]; + std::memset(mbuf, 0, core->walsiz); + if (!mywrite(core->walfh, 0, mbuf, core->walsiz)) { + seterrmsg(core, "mywrite failed"); + err = true; + } + } else { + if (win_ftruncate(core->walfh, 0) != 0) { + seterrmsg(core, "win_ftruncate failed"); + err = true; + } + } + } + if (core->trhard) { + int64_t msiz = core->msiz; + if (msiz > core->psiz) msiz = core->psiz; + if (msiz > 0 && !::FlushViewOfFile(core->map, msiz)) { + seterrmsg(core, "FlushViewOfFile failed"); + err = true; + } + if (!::FlushFileBuffers(core->fh)) { + seterrmsg(core, "FlushFileBuffers failed"); + err = true; + } + if (!::FlushFileBuffers(core->walfh)) { + seterrmsg(core, "FlushFileBuffers failed"); + err = true; + } + } + core->tran = false; + core->alock.unlock(); + return !err; +#else + _assert_(true); + FileCore* core = (FileCore*)opq_; + bool err = false; + core->alock.lock(); + if (!commit && !walapply(core)) err = true; + if (!err) { + if (core->walsiz <= IOBUFSIZ) { + char mbuf[IOBUFSIZ]; + std::memset(mbuf, 0, core->walsiz); + if (!mywrite(core->walfd, 0, mbuf, core->walsiz)) { + seterrmsg(core, "mywrite failed"); + err = true; + } + } else { + if (::ftruncate(core->walfd, 0) != 0) { + seterrmsg(core, "ftruncate failed"); + err = true; + } + } + } + if (core->trhard) { + int64_t msiz = core->msiz; + if (msiz > core->psiz) msiz = core->psiz; + if (msiz > 0 && ::msync(core->map, msiz, MS_SYNC) != 0) { + seterrmsg(core, "msync failed"); + err = true; + } + if (::fsync(core->fd) != 0) { + seterrmsg(core, "fsync failed"); + err = true; + } + if (::fsync(core->walfd) != 0) { + seterrmsg(core, "fsync failed"); + err = true; + } + } + core->tran = false; + core->alock.unlock(); + return !err; +#endif +} + + +/** + * Write a WAL message of transaction explicitly. + */ +bool File::write_transaction(int64_t off, size_t size) { + _assert_(off >= 0 && off <= FILEMAXSIZ && size <= MEMMAXSIZ); + FileCore* core = (FileCore*)opq_; + return walwrite(core, off, size, 0); +} + + +/** + * Get the size of the file. + */ +int64_t File::size() const { + _assert_(true); + FileCore* core = (FileCore*)opq_; + return core->lsiz; +} + + +/** + * Get the path of the file. + */ +std::string File::path() const { + _assert_(true); + FileCore* core = (FileCore*)opq_; + return core->path; +} + + +/** + * Check whether the file was recovered or not. + */ +bool File::recovered() const { + _assert_(true); + FileCore* core = (FileCore*)opq_; + return core->recov; +} + + +/** + * Read the whole data from a file. + */ +char* File::read_file(const std::string& path, int64_t* sp, int64_t limit) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(sp); + if (limit < 0) limit = INT64MAX; + ::DWORD amode = GENERIC_READ; + ::DWORD smode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + ::DWORD cmode = OPEN_EXISTING; + ::HANDLE fh = ::CreateFileA(path.c_str(), amode, smode, NULL, cmode, + FILE_ATTRIBUTE_NORMAL, NULL); + if (!fh || fh == INVALID_HANDLE_VALUE) return NULL; + ::LARGE_INTEGER sbuf; + if (!::GetFileSizeEx(fh, &sbuf)) { + ::CloseHandle(fh); + return false; + } + if (limit > (int64_t)sbuf.QuadPart) limit = sbuf.QuadPart; + char* buf = new char[limit+1]; + char* wp = buf; + int64_t rsiz; + while ((rsiz = win_read(fh, wp, limit - (wp - buf))) > 0) { + wp += rsiz; + } + *wp = '\0'; + ::CloseHandle(fh); + *sp = wp - buf; + return buf; +#else + _assert_(sp); + if (limit < 0) limit = INT64MAX; + int32_t fd = ::open(path.c_str(), O_RDONLY, FILEPERM); + if (fd < 0) return NULL; + struct stat sbuf; + if (::fstat(fd, &sbuf) == -1 || !S_ISREG(sbuf.st_mode)) { + ::close(fd); + return NULL; + } + if (limit > (int64_t)sbuf.st_size) limit = sbuf.st_size; + char* buf = new char[limit+1]; + char* wp = buf; + ssize_t rsiz; + while ((rsiz = ::read(fd, wp, limit - (wp - buf))) > 0) { + wp += rsiz; + } + *wp = '\0'; + ::close(fd); + *sp = wp - buf; + return buf; +#endif +} + + +/** + * Write the whole data into a file. + */ +bool File::write_file(const std::string& path, const char* buf, int64_t size) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(buf && size >= 0 && size <= FILEMAXSIZ); + ::DWORD amode = GENERIC_WRITE; + ::DWORD smode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + ::DWORD cmode = CREATE_ALWAYS; + double wsec = 1.0 / CLOCKTICK; + ::HANDLE fh = INVALID_HANDLE_VALUE; + for (int32_t i = 0; i < 10; i++) { + fh = ::CreateFileA(path.c_str(), amode, smode, NULL, cmode, FILE_ATTRIBUTE_NORMAL, NULL); + if (fh && fh != INVALID_HANDLE_VALUE) break; + if (::GetLastError() != ERROR_ACCESS_DENIED) return false; + if (wsec > 1.0) wsec = 1.0; + Thread::sleep(wsec); + wsec *= 2; + } + bool err = false; + const char* rp = buf; + while (!err && size > 0) { + int64_t wb = win_write(fh, rp, size); + switch (wb) { + case -1: { + if (errno != EINTR) { + err = true; + break; + } + } + case 0: { + break; + } + default: { + rp += wb; + size -= wb; + break; + } + } + } + if (!::CloseHandle(fh)) err = true; + return !err; +#else + _assert_(buf && size >= 0 && size <= FILEMAXSIZ); + int32_t fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, FILEPERM); + if (fd < 0) return false; + bool err = false; + const char* rp = buf; + while (!err && size > 0) { + ssize_t wb = ::write(fd, rp, size); + switch (wb) { + case -1: { + if (errno != EINTR) { + err = true; + break; + } + } + case 0: { + break; + } + default: { + rp += wb; + size -= wb; + break; + } + } + } + if (::close(fd) != 0) err = true; + return !err; +#endif +} + + +/** + * Get the status information of a file. + */ +bool File::status(const std::string& path, Status* buf) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::WIN32_FILE_ATTRIBUTE_DATA ibuf; + if (!::GetFileAttributesExA(path.c_str(), GetFileExInfoStandard, &ibuf)) return false; + if (buf) { + buf->isdir = ibuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY; + ::LARGE_INTEGER li; + li.LowPart = ibuf.nFileSizeLow; + li.HighPart = ibuf.nFileSizeHigh; + buf->size = li.QuadPart; + li.LowPart = ibuf.ftLastWriteTime.dwLowDateTime; + li.HighPart = ibuf.ftLastWriteTime.dwHighDateTime; + buf->mtime = li.QuadPart; + } + return true; +#else + _assert_(true); + struct ::stat sbuf; + if (::lstat(path.c_str(), &sbuf) != 0) return false; + if (buf) { + buf->isdir = S_ISDIR(sbuf.st_mode); + buf->size = sbuf.st_size; + buf->mtime = sbuf.st_mtime; + } + return true; +#endif +} + + +/** + * Get the absolute path of a file. + */ +std::string File::absolute_path(const std::string& path) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + char buf[PATHBUFSIZ]; + ::DWORD size = ::GetFullPathNameA(path.c_str(), sizeof(buf), buf, NULL); + if (size < 1) return ""; + if (size < sizeof(buf)) return std::string(buf); + char* lbuf = new char[size]; + ::DWORD nsiz = ::GetFullPathNameA(path.c_str(), size, lbuf, NULL); + if (nsiz < 1 || nsiz >= size) { + delete[] lbuf; + return ""; + } + std::string rbuf(lbuf); + delete[] lbuf; + return rbuf; +#else + _assert_(true); + char buf[PATHBUFSIZ]; + if (!realpath(path.c_str(), buf)) return ""; + return std::string(buf); +#endif +} + + +/** + * Remove a file. + */ +bool File::remove(const std::string& path) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + double wsec = 1.0 / CLOCKTICK; + for (int32_t i = 0; i < 10; i++) { + if (::DeleteFileA(path.c_str())) return true; + if (::GetLastError() != ERROR_ACCESS_DENIED) return false; + if (wsec > 1.0) wsec = 1.0; + Thread::sleep(wsec); + wsec *= 2; + } + std::string tmppath; + strprintf(&tmppath, "%s%ctmp%c%llx", path.c_str(), EXTCHR, EXTCHR, + ((unsigned long long)(time() * UINT16MAX)) % UINT32MAX); + if (::MoveFileExA(path.c_str(), tmppath.c_str(), MOVEFILE_REPLACE_EXISTING)) { + ::DeleteFileA(tmppath.c_str()); + ::DWORD amode = GENERIC_READ | GENERIC_WRITE; + ::DWORD smode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + ::HANDLE fh = ::CreateFileA(tmppath.c_str(), amode, smode, NULL, OPEN_EXISTING, + FILE_FLAG_DELETE_ON_CLOSE, NULL); + if (fh && fh != INVALID_HANDLE_VALUE) ::CloseHandle(fh); + return true; + } + return false; +#else + _assert_(true); + return ::unlink(path.c_str()) == 0; +#endif +} + + +/** + * Change the name or location of a file. + */ +bool File::rename(const std::string& opath, const std::string& npath) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + double wsec = 1.0 / CLOCKTICK; + for (int32_t i = 0; i < 10; i++) { + if (::MoveFileExA(opath.c_str(), npath.c_str(), MOVEFILE_REPLACE_EXISTING)) return true; + if (::GetLastError() != ERROR_ACCESS_DENIED) return false; + if (wsec > 1.0) wsec = 1.0; + Thread::sleep(wsec); + wsec *= 2; + } + std::string tmppath; + strprintf(&tmppath, "%s%ctmp%c%llx", npath.c_str(), EXTCHR, EXTCHR, + ((unsigned long long)(time() * UINT16MAX)) % UINT32MAX); + if (::MoveFileExA(npath.c_str(), tmppath.c_str(), MOVEFILE_REPLACE_EXISTING)) { + if (::MoveFileExA(opath.c_str(), npath.c_str(), MOVEFILE_REPLACE_EXISTING)) { + ::DeleteFileA(tmppath.c_str()); + ::DWORD amode = GENERIC_READ | GENERIC_WRITE; + ::DWORD smode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + ::HANDLE fh = ::CreateFileA(tmppath.c_str(), amode, smode, NULL, OPEN_EXISTING, + FILE_FLAG_DELETE_ON_CLOSE, NULL); + if (fh && fh != INVALID_HANDLE_VALUE) ::CloseHandle(fh); + return true; + } else { + wsec = 1.0 / CLOCKTICK; + for (int32_t i = 0; i < 10; i++) { + if (::MoveFileExA(tmppath.c_str(), npath.c_str(), MOVEFILE_REPLACE_EXISTING)) break; + if (wsec > 1.0) wsec = 1.0; + Thread::sleep(wsec); + wsec *= 2; + } + } + } + return false; +#else + _assert_(true); + return ::rename(opath.c_str(), npath.c_str()) == 0; +#endif +} + + +/** + * Read a directory. + */ +bool File::read_directory(const std::string& path, std::vector<std::string>* strvec) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(strvec); + std::string dpath = path; + size_t plen = path.size(); + if (plen < 1 || path[plen-1] != PATHCHR) dpath.append(PATHSTR); + dpath.append("*"); + ::WIN32_FIND_DATAA fbuf; + ::HANDLE dh = ::FindFirstFileA(dpath.c_str(), &fbuf); + if (!dh || dh == INVALID_HANDLE_VALUE) return false; + if (std::strcmp(fbuf.cFileName, CDIRSTR) && std::strcmp(fbuf.cFileName, PDIRSTR)) + strvec->push_back(fbuf.cFileName); + while (::FindNextFileA(dh, &fbuf)) { + if (std::strcmp(fbuf.cFileName, CDIRSTR) && std::strcmp(fbuf.cFileName, PDIRSTR)) + strvec->push_back(fbuf.cFileName); + } + if (!::FindClose(dh)) return false; + return true; +#else + _assert_(strvec); + ::DIR* dir = ::opendir(path.c_str()); + if (!dir) return false; + struct ::dirent *dp; + while ((dp = ::readdir(dir)) != NULL) { + if (std::strcmp(dp->d_name, CDIRSTR) && std::strcmp(dp->d_name, PDIRSTR)) + strvec->push_back(dp->d_name); + } + if (::closedir(dir) != 0) return false; + return true; +#endif +} + + +/** + * Make a directory. + */ +bool File::make_directory(const std::string& path) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + double wsec = 1.0 / CLOCKTICK; + for (int32_t i = 0; i < 10; i++) { + if (::CreateDirectoryA(path.c_str(), NULL)) return true; + if (::GetLastError() != ERROR_ACCESS_DENIED) return false; + if (wsec > 1.0) wsec = 1.0; + Thread::sleep(wsec); + wsec *= 2; + } + return false; +#else + _assert_(true); + return ::mkdir(path.c_str(), DIRPERM) == 0; +#endif +} + + +/** + * Remove a directory. + */ +bool File::remove_directory(const std::string& path) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + double wsec = 1.0 / CLOCKTICK; + for (int32_t i = 0; i < 10; i++) { + if (::RemoveDirectoryA(path.c_str())) return true; + if (::GetLastError() != ERROR_ACCESS_DENIED) return false; + if (wsec > 1.0) wsec = 1.0; + Thread::sleep(wsec); + wsec *= 2; + } + return false; +#else + _assert_(true); + return ::rmdir(path.c_str()) == 0; +#endif +} + + +/** + * Remove a file or a directory recursively. + */ +bool File::remove_recursively(const std::string& path) { + bool err = false; + std::vector<std::string> list; + list.push_back(path); + while (!list.empty()) { + const std::string& cpath = list.back(); + Status sbuf; + if (status(cpath, &sbuf)) { + if (sbuf.isdir) { + if (remove_directory(cpath)) { + list.pop_back(); + } else { + DirStream dir; + if (dir.open(cpath)) { + std::string ccname; + while (dir.read(&ccname)) { + const std::string& ccpath = cpath + MYPATHCHR + ccname; + if (!remove(ccpath)) list.push_back(ccpath); + } + if (!dir.close()) err = true; + } else { + list.pop_back(); + err = true; + } + } + } else { + if (!remove(cpath)) err = true; + list.pop_back(); + } + } else { + list.pop_back(); + err = true; + } + } + return !err; +} + + +/** + * Get the path of the current working directory. + */ +std::string File::get_current_directory() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + char buf[PATHBUFSIZ]; + ::DWORD size = ::GetCurrentDirectoryA(sizeof(buf), buf); + if (size < 1) return ""; + if (size < sizeof(buf)) return std::string(buf); + char* lbuf = new char[size]; + ::DWORD nsiz = ::GetCurrentDirectoryA(size, lbuf); + if (nsiz < 1 || nsiz >= size) { + delete[] lbuf; + return ""; + } + std::string rbuf(lbuf); + delete[] lbuf; + return rbuf; +#else + _assert_(true); + char buf[PATHBUFSIZ]; + if (!::getcwd(buf, sizeof(buf))) return ""; + return std::string(buf); +#endif +} + + +/** + * Set the current working directory. + */ +bool File::set_current_directory(const std::string& path) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + return ::SetCurrentDirectoryA(path.c_str()); +#else + _assert_(true); + return ::chdir(path.c_str()) == 0; +#endif +} + + +/** + * Synchronize the whole of the file system with the device. + */ +bool File::synchronize_whole() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + return true; +#else + _assert_(true); + ::sync(); + return true; +#endif +} + + + +/** + * Default constructor. + */ +DirStream::DirStream() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + DirStreamCore* core = new DirStreamCore; + core->dh = NULL; + opq_ = core; +#else + _assert_(true); + DirStreamCore* core = new DirStreamCore; + core->dh = NULL; + opq_ = core; +#endif +} + + +/** + * Destructor. + */ +DirStream::~DirStream() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + DirStreamCore* core = (DirStreamCore*)opq_; + if (core->dh) close(); + delete core; +#else + _assert_(true); + DirStreamCore* core = (DirStreamCore*)opq_; + if (core->dh) close(); + delete core; +#endif +} + + +/** + * Open a directory. + */ +bool DirStream::open(const std::string& path) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + DirStreamCore* core = (DirStreamCore*)opq_; + ScopedMutex lock(&core->alock); + if (core->dh) return false; + std::string dpath = path; + size_t plen = path.size(); + if (plen < 1 || path[plen-1] != File::PATHCHR) dpath.append(File::PATHSTR); + dpath.append("*"); + ::WIN32_FIND_DATAA fbuf; + ::HANDLE dh = ::FindFirstFileA(dpath.c_str(), &fbuf); + if (!dh || dh == INVALID_HANDLE_VALUE) return false; + core->dh = dh; + core->cur = fbuf.cFileName; + return true; +#else + _assert_(true); + DirStreamCore* core = (DirStreamCore*)opq_; + ScopedMutex lock(&core->alock); + if (core->dh) return false; + ::DIR* dh = ::opendir(path.c_str()); + if (!dh) return false; + core->dh = dh; + return true; +#endif +} + + +/** + * Close the file. + */ +bool DirStream::close() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + DirStreamCore* core = (DirStreamCore*)opq_; + ScopedMutex lock(&core->alock); + if (!core->dh) return false; + bool err = false; + if (!::FindClose(core->dh)) err = true; + core->dh = NULL; + core->cur.clear(); + return !err; +#else + _assert_(true); + DirStreamCore* core = (DirStreamCore*)opq_; + ScopedMutex lock(&core->alock); + if (!core->dh) return false; + bool err = false; + if (::closedir(core->dh) != 0) err = true; + core->dh = NULL; + return !err; +#endif +} + + +/** + * Read the next file in the directory. + */ +bool DirStream::read(std::string* path) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(path); + DirStreamCore* core = (DirStreamCore*)opq_; + ScopedMutex lock(&core->alock); + if (!core->dh) return false; + while (core->cur == File::CDIRSTR || core->cur == File::PDIRSTR) { + ::WIN32_FIND_DATAA fbuf; + if (::FindNextFileA(core->dh, &fbuf)) { + core->cur = fbuf.cFileName; + } else { + core->cur.clear(); + return false; + } + } + if (core->cur.empty()) return false; + path->clear(); + path->append(core->cur); + ::WIN32_FIND_DATAA fbuf; + if (::FindNextFileA(core->dh, &fbuf)) { + core->cur = fbuf.cFileName; + } else { + core->cur.clear(); + } + return true; +#else + _assert_(path); + DirStreamCore* core = (DirStreamCore*)opq_; + ScopedMutex lock(&core->alock); + if (!core->dh) return false; + struct ::dirent *dp; + do { + dp = ::readdir(core->dh); + if (!dp) return false; + } while (!std::strcmp(dp->d_name, File::CDIRSTR) || !std::strcmp(dp->d_name, File::PDIRSTR)); + path->clear(); + path->append(dp->d_name); + return true; +#endif +} + + +/** + * Set the error message. + */ +static void seterrmsg(FileCore* core, const char* msg) { + _assert_(core && msg); + core->errmsg.set((void*)msg); +} + + +/** + * Get the path of the WAL file. + */ +static std::string walpath(const std::string& path) { + _assert_(true); + return path + File::EXTCHR + WALPATHEXT; +} + + +/** + * Write a log message into the WAL file. + */ +static bool walwrite(FileCore *core, int64_t off, size_t size, int64_t base) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(core && off >= 0 && off <= FILEMAXSIZ && size <= MEMMAXSIZ && base >= 0); + bool err = false; + if (off < base) { + int64_t diff = base - off; + if (diff >= (int64_t)size) return true; + off = base; + size -= diff; + } + int64_t rem = core->trmsiz - off; + if (rem < 1) return true; + if (rem < (int64_t)size) size = rem; + char stack[IOBUFSIZ]; + size_t rsiz = sizeof(int8_t) + sizeof(int64_t) * 2 + size; + char* rbuf = rsiz > sizeof(stack) ? new char[rsiz] : stack; + char* wp = rbuf; + *(wp++) = WALMSGMAGIC; + int64_t num = hton64(off); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(size); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + core->alock.lock(); + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(wp, core->map + off, size); + } else { + if (off < core->msiz) { + int64_t hsiz = core->msiz - off; + std::memcpy(wp, core->map + off, hsiz); + off += hsiz; + wp += hsiz; + size -= hsiz; + } + while (true) { + int64_t rb = win_pread(core->fh, wp, size, off); + if (rb >= (int64_t)size) { + break; + } else if (rb > 0) { + wp += rb; + size -= rb; + off += rb; + } else { + err = true; + break; + } + } + if (err) { + seterrmsg(core, "win_pread failed"); + std::memset(wp, 0, size); + } + } + if (!mywrite(core->walfh, core->walsiz, rbuf, rsiz)) { + seterrmsg(core, "mywrite failed"); + err = true; + } + if (core->trhard && !::FlushFileBuffers(core->walfh)) { + seterrmsg(core, "FlushFileBuffers failed"); + err = true; + } + core->walsiz += rsiz; + if (rbuf != stack) delete[] rbuf; + core->alock.unlock(); + return !err; +#else + _assert_(core && off >= 0 && off <= FILEMAXSIZ && size <= MEMMAXSIZ && base >= 0); + bool err = false; + if (off < base) { + int64_t diff = base - off; + if (diff >= (int64_t)size) return true; + off = base; + size -= diff; + } + int64_t rem = core->trmsiz - off; + if (rem < 1) return true; + if (rem < (int64_t)size) size = rem; + char stack[IOBUFSIZ]; + size_t rsiz = sizeof(int8_t) + sizeof(int64_t) * 2 + size; + char* rbuf = rsiz > sizeof(stack) ? new char[rsiz] : stack; + char* wp = rbuf; + *(wp++) = WALMSGMAGIC; + int64_t num = hton64(off); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(size); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + core->alock.lock(); + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(wp, core->map + off, size); + } else { + if (off < core->msiz) { + int64_t hsiz = core->msiz - off; + std::memcpy(wp, core->map + off, hsiz); + off += hsiz; + wp += hsiz; + size -= hsiz; + } + while (true) { + ssize_t rb = ::pread(core->fd, wp, size, off); + if (rb >= (ssize_t)size) { + break; + } else if (rb > 0) { + wp += rb; + size -= rb; + off += rb; + } else if (rb == -1) { + if (errno != EINTR) { + err = true; + break; + } + } else { + err = true; + break; + } + } + if (err) { + seterrmsg(core, "pread failed"); + std::memset(wp, 0, size); + } + } + if (!mywrite(core->walfd, core->walsiz, rbuf, rsiz)) { + seterrmsg(core, "mywrite failed"); + err = true; + } + if (core->trhard && ::fsync(core->walfd) != 0) { + seterrmsg(core, "fsync failed"); + err = true; + } + core->walsiz += rsiz; + if (rbuf != stack) delete[] rbuf; + core->alock.unlock(); + return !err; +#endif +} + + +/** + * Apply log messages in the WAL file. + */ +static bool walapply(FileCore* core) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(core); + bool err = false; + char buf[IOBUFSIZ]; + int64_t hsiz = sizeof(WALMAGICDATA) + sizeof(int64_t); + ::LARGE_INTEGER li; + if (!::GetFileSizeEx(core->walfh, &li)) { + seterrmsg(core, "GetFileSizeEx failed"); + return false; + } + int64_t rem = li.QuadPart; + if (rem < hsiz) { + seterrmsg(core, "too short WAL file"); + return false; + } + li.QuadPart = 0; + if (!::SetFilePointerEx(core->walfh, li, NULL, FILE_BEGIN)) { + seterrmsg(core, "SetFilePointerEx failed"); + return false; + } + if (!myread(core->walfh, buf, hsiz)) { + seterrmsg(core, "myread failed"); + return false; + } + if (*buf == 0) return true; + if (std::memcmp(buf, WALMAGICDATA, sizeof(WALMAGICDATA))) { + seterrmsg(core, "invalid magic data of WAL"); + return false; + } + int64_t osiz; + std::memcpy(&osiz, buf + sizeof(WALMAGICDATA), sizeof(osiz)); + osiz = ntoh64(osiz); + rem -= hsiz; + hsiz = sizeof(uint8_t) + sizeof(int64_t) * 2; + std::vector<WALMessage> msgs; + int64_t end = 0; + while (rem >= hsiz) { + if (!myread(core->walfh, buf, hsiz)) { + seterrmsg(core, "myread failed"); + err = true; + break; + } + if (*buf == 0) { + rem = 0; + break; + } + rem -= hsiz; + char* rp = buf; + if (*(uint8_t*)(rp++) != WALMSGMAGIC) { + seterrmsg(core, "invalid magic data of WAL message"); + err = true; + break; + } + if (rem > 0) { + int64_t off; + std::memcpy(&off, rp, sizeof(off)); + off = ntoh64(off); + rp += sizeof(off); + int64_t size; + std::memcpy(&size, rp, sizeof(size)); + size = ntoh64(size); + rp += sizeof(size); + if (off < 0 || size < 0) { + seterrmsg(core, "invalid meta data of WAL message"); + err = true; + break; + } + if (rem < size) { + seterrmsg(core, "too short WAL message"); + err = true; + break; + } + char* rbuf = size > (int64_t)sizeof(buf) ? new char[size] : buf; + if (!myread(core->walfh, rbuf, size)) { + seterrmsg(core, "myread failed"); + if (rbuf != buf) delete[] rbuf; + err = true; + break; + } + rem -= size; + WALMessage msg = { off, std::string(rbuf, size) }; + msgs.push_back(msg); + if (off + size > end) end = off + size; + if (rbuf != buf) delete[] rbuf; + } + } + if (rem != 0) { + if (!myread(core->walfh, buf, 1)) { + seterrmsg(core, "myread failed"); + err = true; + } else if (*buf != 0) { + seterrmsg(core, "too few messages of WAL"); + err = true; + } + } + if (end > core->msiz) end = core->msiz; + if (core->psiz < end && win_ftruncate(core->fh, end) != 0) { + seterrmsg(core, "win_ftruncate failed"); + err = true; + } + for (int64_t i = (int64_t)msgs.size() - 1; i >= 0; i--) { + const WALMessage& msg = msgs[i]; + int64_t off = msg.off; + const char* rbuf = msg.body.c_str(); + size_t size = msg.body.size(); + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(core->map + off, rbuf, size); + } else { + if (off < core->msiz) { + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, rbuf, hsiz); + off += hsiz; + rbuf += hsiz; + size -= hsiz; + } + while (true) { + int64_t wb = win_pwrite(core->fh, rbuf, size, off); + if (wb >= (int64_t)size) { + break; + } else if (wb > 0) { + rbuf += wb; + size -= wb; + off += wb; + } else if (wb == -1) { + seterrmsg(core, "win_pwrite failed"); + err = true; + break; + } else if (size > 0) { + seterrmsg(core, "pwrite failed"); + err = true; + break; + } + } + } + } + if (win_ftruncate(core->fh, osiz) == 0) { + core->lsiz = osiz; + core->psiz = osiz; + } else { + seterrmsg(core, "win_ftruncate failed"); + err = true; + } + return !err; +#else + _assert_(core); + bool err = false; + char buf[IOBUFSIZ]; + int64_t hsiz = sizeof(WALMAGICDATA) + sizeof(int64_t); + int64_t rem = ::lseek(core->walfd, 0, SEEK_END); + if (rem < hsiz) { + seterrmsg(core, "lseek failed"); + return false; + } + if (::lseek(core->walfd, 0, SEEK_SET) != 0) { + seterrmsg(core, "lseek failed"); + return false; + } + if (!myread(core->walfd, buf, hsiz)) { + seterrmsg(core, "myread failed"); + return false; + } + if (*buf == 0) return true; + if (std::memcmp(buf, WALMAGICDATA, sizeof(WALMAGICDATA))) { + seterrmsg(core, "invalid magic data of WAL"); + return false; + } + int64_t osiz; + std::memcpy(&osiz, buf + sizeof(WALMAGICDATA), sizeof(osiz)); + osiz = ntoh64(osiz); + rem -= hsiz; + hsiz = sizeof(uint8_t) + sizeof(int64_t) * 2; + std::vector<WALMessage> msgs; + int64_t end = 0; + while (rem >= hsiz) { + if (!myread(core->walfd, buf, hsiz)) { + seterrmsg(core, "myread failed"); + err = true; + break; + } + if (*buf == 0) { + rem = 0; + break; + } + rem -= hsiz; + char* rp = buf; + if (*(uint8_t*)(rp++) != WALMSGMAGIC) { + seterrmsg(core, "invalid magic data of WAL message"); + err = true; + break; + } + if (rem > 0) { + int64_t off; + std::memcpy(&off, rp, sizeof(off)); + off = ntoh64(off); + rp += sizeof(off); + int64_t size; + std::memcpy(&size, rp, sizeof(size)); + size = ntoh64(size); + rp += sizeof(size); + if (off < 0 || size < 0) { + seterrmsg(core, "invalid meta data of WAL message"); + err = true; + break; + } + if (rem < size) { + seterrmsg(core, "too short WAL message"); + err = true; + break; + } + char* rbuf = size > (int64_t)sizeof(buf) ? new char[size] : buf; + if (!myread(core->walfd, rbuf, size)) { + seterrmsg(core, "myread failed"); + if (rbuf != buf) delete[] rbuf; + err = true; + break; + } + rem -= size; + WALMessage msg = { off, std::string(rbuf, size) }; + msgs.push_back(msg); + if (off + size > end) end = off + size; + if (rbuf != buf) delete[] rbuf; + } + } + if (rem != 0) { + if (!myread(core->walfd, buf, 1)) { + seterrmsg(core, "myread failed"); + err = true; + } else if (*buf != 0) { + seterrmsg(core, "too few messages of WAL"); + err = true; + } + } + if (end > core->msiz) end = core->msiz; + if (core->psiz < end && ::ftruncate(core->fd, end) != 0) { + seterrmsg(core, "ftruncate failed"); + err = true; + } + for (int64_t i = (int64_t)msgs.size() - 1; i >= 0; i--) { + const WALMessage& msg = msgs[i]; + int64_t off = msg.off; + const char* rbuf = msg.body.c_str(); + size_t size = msg.body.size(); + int64_t end = off + size; + if (end <= core->msiz) { + std::memcpy(core->map + off, rbuf, size); + } else { + if (off < core->msiz) { + size_t hsiz = core->msiz - off; + std::memcpy(core->map + off, rbuf, hsiz); + off += hsiz; + rbuf += hsiz; + size -= hsiz; + } + while (true) { + ssize_t wb = ::pwrite(core->fd, rbuf, size, off); + if (wb >= (ssize_t)size) { + break; + } else if (wb > 0) { + rbuf += wb; + size -= wb; + off += wb; + } else if (wb == -1) { + if (errno != EINTR) { + seterrmsg(core, "pwrite failed"); + err = true; + break; + } + } else if (size > 0) { + seterrmsg(core, "pwrite failed"); + err = true; + break; + } + } + } + } + if (::ftruncate(core->fd, osiz) == 0) { + core->lsiz = osiz; + core->psiz = osiz; + } else { + seterrmsg(core, "ftruncate failed"); + err = true; + } + return !err; +#endif +} + + +/** + * Write data into a file. + */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static bool mywrite(::HANDLE fh, int64_t off, const void* buf, size_t size) { + _assert_(off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + while (true) { + int64_t wb = win_pwrite(fh, buf, size, off); + if (wb >= (int64_t)size) { + return true; + } else if (wb > 0) { + buf = (char*)buf + wb; + size -= wb; + off += wb; + } else if (wb == -1) { + return false; + } else if (size > 0) { + return false; + } + } + return true; +} +#else +static bool mywrite(int32_t fd, int64_t off, const void* buf, size_t size) { + _assert_(fd >= 0 && off >= 0 && off <= FILEMAXSIZ && buf && size <= MEMMAXSIZ); + while (true) { + ssize_t wb = ::pwrite(fd, buf, size, off); + if (wb >= (ssize_t)size) { + return true; + } else if (wb > 0) { + buf = (char*)buf + wb; + size -= wb; + off += wb; + } else if (wb == -1) { + if (errno != EINTR) return false; + } else if (size > 0) { + return false; + } + } + return true; +} +#endif + + +/** + * Read data from a file. + */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static size_t myread(::HANDLE fh, void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + while (true) { + int64_t rb = win_read(fh, buf, size); + if (rb >= (int64_t)size) { + break; + } else if (rb > 0) { + buf = (char*)buf + rb; + size -= rb; + } else if (rb == -1) { + return false; + } else if (size > 0) { + return false; + } + } + return true; +} +#else +static size_t myread(int32_t fd, void* buf, size_t size) { + _assert_(fd >= 0 && buf && size <= MEMMAXSIZ); + while (true) { + ssize_t rb = ::read(fd, buf, size); + if (rb >= (ssize_t)size) { + break; + } else if (rb > 0) { + buf = (char*)buf + rb; + size -= rb; + } else if (rb == -1) { + if (errno != EINTR) return false; + } else if (size > 0) { + return false; + } + } + return true; +} +#endif + + +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +/** + * Emulate the pwrite call + */ +static int64_t win_pwrite(::HANDLE fh, const void* buf, size_t count, int64_t offset) { + _assert_(buf && count <= MEMMAXSIZ && offset >= 0 && offset <= FILEMAXSIZ); + ::DWORD wb; + ::LARGE_INTEGER li; + li.QuadPart = offset; + ::OVERLAPPED ol; + ol.Offset = li.LowPart; + ol.OffsetHigh = li.HighPart; + ol.hEvent = NULL; + if (!::WriteFile(fh, buf, count, &wb, &ol)) return -1; + return wb; +} +#endif + + +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +/** + * Emulate the pread call + */ +static int64_t win_pread(::HANDLE fh, void* buf, size_t count, int64_t offset) { + _assert_(buf && count <= MEMMAXSIZ && offset >= 0 && offset <= FILEMAXSIZ); + ::DWORD rb; + ::LARGE_INTEGER li; + li.QuadPart = offset; + ::OVERLAPPED ol; + ol.Offset = li.LowPart; + ol.OffsetHigh = li.HighPart; + ol.hEvent = NULL; + if (!::ReadFile(fh, buf, count, &rb, &ol)) return -1; + return rb; +} +#endif + + +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +/** + * Emulate the write call + */ +static int64_t win_write(::HANDLE fh, const void* buf, size_t count) { + _assert_(buf && count <= MEMMAXSIZ); + ::DWORD wb; + if (!::WriteFile(fh, buf, count, &wb, NULL)) return -1; + return wb; +} +#endif + + +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +/** + * Emulate the read call + */ +static int64_t win_read(::HANDLE fh, void* buf, size_t count) { + _assert_(buf && count <= MEMMAXSIZ); + ::DWORD rb; + if (!::ReadFile(fh, buf, count, &rb, NULL)) return -1; + return rb; +} +#endif + + +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +/** + * Emulate the ftruncate call + */ +static int32_t win_ftruncate(::HANDLE fh, int64_t length) { + _assert_(length >= 0 && length <= FILEMAXSIZ); + ::LARGE_INTEGER li; + li.QuadPart = length; + if (!::SetFilePointerEx(fh, li, NULL, FILE_BEGIN)) return -1; + if (!::SetEndOfFile(fh) && ::GetLastError() != 1224) return -1; + return 0; +} +#endif + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcfile.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcfile.h new file mode 100644 index 0000000000..c3436cb14c --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcfile.h @@ -0,0 +1,388 @@ +/************************************************************************************************* + * Filesystem abstraction + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCFILE_H // duplication check +#define _KCFILE_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> + +namespace kyotocabinet { // common namespace + + +/** + * Filesystem abstraction. + */ +class File { + public: + struct Status; + public: + /** Path delimiter character. */ + static const char PATHCHR; + /** Path delimiter string. */ + static const char* const PATHSTR; + /** Extension delimiter character. */ + static const char EXTCHR; + /** Extension delimiter string. */ + static const char* const EXTSTR; + /** Current directory string. */ + static const char* const CDIRSTR; + /** Parent directory string. */ + static const char* const PDIRSTR; + /** + * Status information. + */ + struct Status { + bool isdir; ///< whether directory or not + int64_t size; ///< file size + int64_t mtime; ///< last modified time + }; + /** + * Open modes. + */ + enum OpenMode { + OREADER = 1 << 0, ///< open as a reader + OWRITER = 1 << 1, ///< open as a writer + OCREATE = 1 << 2, ///< writer creating + OTRUNCATE = 1 << 3, ///< writer truncating + ONOLOCK = 1 << 4, ///< open without locking + OTRYLOCK = 1 << 5 ///< lock without blocking + }; + /** + * Default constructor. + */ + explicit File(); + /** + * Destructor. + * @note If the file is not closed, it is closed implicitly. + */ + ~File(); + /** + * Get the last happened error information. + * @return the last happened error information. + */ + const char* error() const; + /** + * Open a file. + * @param path the path of a file. + * @param mode the connection mode. File::OWRITER as a writer, File::OREADER as a reader. + * The following may be added to the writer mode by bitwise-or: File::OCREATE, which means it + * creates a new file if the file does not exist, File::OTRUNCATE, which means it creates a + * new file regardless if the file exists. The following may be added to both of the reader + * mode and the writer mode by bitwise-or: File::ONOLOCK, which means it opens the file + * without file locking, File::TRYLOCK, which means locking is performed without blocking. + * @param msiz the size of the internal memory-mapped region. + * @return true on success, or false on failure. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE, int64_t msiz = 0); + /** + * Close the file. + * @return true on success, or false on failure. + */ + bool close(); + /** + * Write data. + * @param off the offset of the destination. + * @param buf the pointer to the data region. + * @param size the size of the data region. + * @return true on success, or false on failure. + */ + bool write(int64_t off, const void* buf, size_t size); + /** + * Write data. + * @note Equal to the original File::write method except that the sigunature is different. + */ + bool write(int64_t off, const std::string& str) { + _assert_(off >= 0); + return write(off, str.c_str(), str.size()); + } + /** + * Write data with assuring the region does not spill from the file size. + * @param off the offset of the destination. + * @param buf the pointer to the data region. + * @param size the size of the data region. + * @return true on success, or false on failure. + */ + bool write_fast(int64_t off, const void* buf, size_t size); + /** + * Write data with assuring the region does not spill from the file size. + * @note Equal to the original File::write_fast method except that the sigunature is different. + */ + bool write_fast(int64_t off, const std::string& str) { + _assert_(off >= 0); + return write_fast(off, str.c_str(), str.size()); + } + /** + * Write data at the end of the file. + * @param buf the pointer to the data region. + * @param size the size of the data region. + * @return true on success, or false on failure. + */ + bool append(const void* buf, size_t size); + /** + * Write data at the end of the file. + * @note Equal to the original File::append method except that the sigunature is different. + */ + bool append(const std::string& str) { + _assert_(true); + return append(str.c_str(), str.size()); + } + /** + * Read data. + * @param off the offset of the source. + * @param buf the pointer to the destination region. + * @param size the size of the data to be read. + * @return true on success, or false on failure. + */ + bool read(int64_t off, void* buf, size_t size); + /** + * Read data. + * @note Equal to the original File::read method except that the sigunature is different. + */ + bool read(int64_t off, std::string* buf, size_t size) { + _assert_(off >= 0 && buf); + char* tbuf = new char[size]; + if (!read(off, tbuf, size)) { + delete[] tbuf; + return false; + } + buf->append(std::string(tbuf, size)); + delete[] tbuf; + return true; + } + /** + * Read data with assuring the region does not spill from the file size. + * @param off the offset of the source. + * @param buf the pointer to the destination region. + * @param size the size of the data to be read. + * @return true on success, or false on failure. + */ + bool read_fast(int64_t off, void* buf, size_t size); + /** + * Read data. + * @note Equal to the original File::read method except that the sigunature is different. + */ + bool read_fast(int64_t off, std::string* buf, size_t size) { + _assert_(off >= 0 && buf); + char* tbuf = new char[size]; + if (!read_fast(off, tbuf, size)) { + delete[] tbuf; + return false; + } + buf->append(std::string(tbuf, size)); + delete[] tbuf; + return true; + } + /** + * Truncate the file. + * @param size the new size of the file. + * @return true on success, or false on failure. + */ + bool truncate(int64_t size); + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool synchronize(bool hard); + /** + * Refresh the internal state for update by others. + * @return true on success, or false on failure. + */ + bool refresh(); + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param off the beginning offset of the guarded region + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard, int64_t off); + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit); + /** + * Write a WAL message of transaction explicitly. + * @param off the offset of the source. + * @param size the size of the data to be read. + * @return true on success, or false on failure. + */ + bool write_transaction(int64_t off, size_t size); + /** + * Get the size of the file. + * @return the size of the file, or 0 on failure. + */ + int64_t size() const; + /** + * Get the path of the file. + * @return the path of the file in bytes, or an empty string on failure. + */ + std::string path() const; + /** + * Check whether the file was recovered or not. + * @return true if recovered, or false if not. + */ + bool recovered() const; + /** + * Read the whole data from a file. + * @param path the path of a file. + * @param sp the pointer to the variable into which the size of the region of the return value + * is assigned. + * @param limit the limit length to read. If it is nagative, no limit is specified. + * @return the pointer to the region of the read data, or NULL on failure. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a C-style string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the + * delete[] operator when it is no longer in use. + */ + static char* read_file(const std::string& path, int64_t* sp, int64_t limit = -1); + /** + * Write the whole data into a file. + * @param path the path of a file. + * @param buf the data buffer to write. + * @param size the size of the data buffer. + * @return true on success, or false on failure. + * @note The existing file corresponding to the path is overwritten. If no file corresponds + * to the path, a new file is created. + */ + static bool write_file(const std::string& path, const char* buf, int64_t size); + /** + * Get the status information of a file. + * @param path the path of a file. + * @param buf a structure of status information. If it is NULL, it is omitted. + * @return true on success, or false on failure. + */ + static bool status(const std::string& path, Status* buf = NULL); + /** + * Get the absolute path of a file. + * @param path the path of a file. + * @return the absolute path of the file, or an empty string on failure. + */ + static std::string absolute_path(const std::string& path); + /** + * Remove a file. + * @param path the path of a file. + * @return true on success, or false on failure. + */ + static bool remove(const std::string& path); + /** + * Change the name or location of a file. + * @param opath the old path of a file. + * @param npath the new path of a file. + * @return true on success, or false on failure. + */ + static bool rename(const std::string& opath, const std::string& npath); + /** + * Read a directory. + * @param path the path of a directory. + * @param strvec a string list to contain the result. + * @return true on success, or false on failure. + */ + static bool read_directory(const std::string& path, std::vector<std::string>* strvec); + /** + * Make a directory. + * @param path the path of a directory. + * @return true on success, or false on failure. + */ + static bool make_directory(const std::string& path); + /** + * Remove a directory. + * @param path the path of a directory. + * @return true on success, or false on failure. + */ + static bool remove_directory(const std::string& path); + /** + * Remove a file or a directory recursively. + * @param path the path of a file or a directory. + * @return true on success, or false on failure. + */ + static bool remove_recursively(const std::string& path); + /** + * Get the path of the current working directory. + * @return the path of the current working directory, or an empty string on failure. + */ + static std::string get_current_directory(); + /** + * Set the current working directory. + * @param path the path of a directory. + * @return true on success, or false on failure. + */ + static bool set_current_directory(const std::string& path); + /** + * Synchronize the whole of the file system with the device. + * @return true on success, or false on failure. + */ + static bool synchronize_whole(); + private: + /** Dummy constructor to forbid the use. */ + File(const File&); + /** Dummy Operator to forbid the use. */ + File& operator =(const File&); + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Directory stream abstraction. + */ +class DirStream { + public: + /** + * Default constructor. + */ + explicit DirStream(); + /** + * Destructor. + * @note If the file is not closed, it is closed implicitly. + */ + ~DirStream(); + /** + * Open a directory. + * @param path the path of a directory. + * @return true on success, or false on failure. + */ + bool open(const std::string& path); + /** + * Close the file. + * @return true on success, or false on failure. + */ + bool close(); + /** + * Read the next file in the directory. + * @param path a string to store the file path. + * @return true on success, or false on failure. + */ + bool read(std::string* path); + private: + /** Dummy constructor to forbid the use. */ + DirStream(const DirStream&); + /** Dummy Operator to forbid the use. */ + DirStream& operator =(const DirStream&); + /** Opaque pointer. */ + void* opq_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcforestmgr.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcforestmgr.cc new file mode 100644 index 0000000000..32878e0c0a --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcforestmgr.cc @@ -0,0 +1,1497 @@ +/************************************************************************************************* + * The command line utility of the directory tree database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcdirdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, const char* info); +static int32_t runcreate(int argc, char** argv); +static int32_t runinform(int argc, char** argv); +static int32_t runset(int argc, char** argv); +static int32_t runremove(int argc, char** argv); +static int32_t runget(int argc, char** argv); +static int32_t runlist(int argc, char** argv); +static int32_t runclear(int argc, char** argv); +static int32_t runimport(int argc, char** argv); +static int32_t runcopy(int argc, char** argv); +static int32_t rundump(int argc, char** argv); +static int32_t runload(int argc, char** argv); +static int32_t runsetbulk(int argc, char** argv); +static int32_t runremovebulk(int argc, char** argv); +static int32_t rungetbulk(int argc, char** argv); +static int32_t runcheck(int argc, char** argv); +static int32_t proccreate(const char* path, int32_t oflags, int32_t opts, int64_t bnum, + int32_t psiz, kc::Comparator* rcomp); +static int32_t procinform(const char* path, int32_t oflags, bool st); +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode); +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags); +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz); +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + bool des, int64_t max, bool rm, bool pv, bool px); +static int32_t procclear(const char* path, int32_t oflags); +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx); +static int32_t proccopy(const char* path, const char* file, int32_t oflags); +static int32_t procdump(const char* path, const char* file, int32_t oflags); +static int32_t procload(const char* path, const char* file, int32_t oflags); +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs); +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys); +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px); +static int32_t proccheck(const char* path, int32_t oflags); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "create")) { + rv = runcreate(argc, argv); + } else if (!std::strcmp(argv[1], "inform")) { + rv = runinform(argc, argv); + } else if (!std::strcmp(argv[1], "set")) { + rv = runset(argc, argv); + } else if (!std::strcmp(argv[1], "remove")) { + rv = runremove(argc, argv); + } else if (!std::strcmp(argv[1], "get")) { + rv = runget(argc, argv); + } else if (!std::strcmp(argv[1], "list")) { + rv = runlist(argc, argv); + } else if (!std::strcmp(argv[1], "clear")) { + rv = runclear(argc, argv); + } else if (!std::strcmp(argv[1], "import")) { + rv = runimport(argc, argv); + } else if (!std::strcmp(argv[1], "copy")) { + rv = runcopy(argc, argv); + } else if (!std::strcmp(argv[1], "dump")) { + rv = rundump(argc, argv); + } else if (!std::strcmp(argv[1], "load")) { + rv = runload(argc, argv); + } else if (!std::strcmp(argv[1], "setbulk")) { + rv = runsetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "removebulk")) { + rv = runremovebulk(argc, argv); + } else if (!std::strcmp(argv[1], "getbulk")) { + rv = rungetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "check")) { + rv = runcheck(argc, argv); + } else if (!std::strcmp(argv[1], "version") || !std::strcmp(argv[1], "--version")) { + printversion(); + } else { + usage(); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: the command line utility of the directory tree database of Kyoto Cabinet\n", + g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s create [-otr] [-onl|-otl|-onr] [-tc] [-bnum num] [-psiz num] [-rcd|-rcld|-rcdd]" + " path\n", g_progname); + eprintf(" %s inform [-onl|-otl|-onr] [-st] path\n", g_progname); + eprintf(" %s set [-onl|-otl|-onr] [-add|-rep|-app|-inci|-incd] [-sx] path key value\n", + g_progname); + eprintf(" %s remove [-onl|-otl|-onr] [-sx] path key\n", g_progname); + eprintf(" %s get [-onl|-otl|-onr] [-rm] [-sx] [-px] [-pz] path key\n", g_progname); + eprintf(" %s list [-onl|-otl|-onr] [-des] [-max num] [-rm] [-sx] [-pv] [-px] path [key]\n", + g_progname); + eprintf(" %s clear [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s import [-onl|-otl|-onr] [-sx] path [file]\n", g_progname); + eprintf(" %s copy [-onl|-otl|-onr] path file\n", g_progname); + eprintf(" %s dump [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s load [-otr] [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s setbulk [-onl|-otl|-onr] [-sx] path key value ...\n", g_progname); + eprintf(" %s removebulk [-onl|-otl|-onr] [-sx] path key ...\n", g_progname); + eprintf(" %s getbulk [-onl|-otl|-onr] [-sx] [-px] path key ...\n", g_progname); + eprintf(" %s check [-onl|-otl|-onr] path\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print error message of database +static void dberrprint(kc::BasicDB* db, const char* info) { + const kc::BasicDB::Error& err = db->error(); + eprintf("%s: %s: %s: %d: %s: %s\n", + g_progname, info, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// parse arguments of create command +static int32_t runcreate(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + int32_t opts = 0; + int64_t bnum = -1; + int32_t psiz = -1; + kc::Comparator* rcomp = NULL; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::ForestDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::ForestDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccreate(path, oflags, opts, bnum, psiz, rcomp); + return rv; +} + + +// parse arguments of inform command +static int32_t runinform(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + bool st = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-st")) { + st = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procinform(path, oflags, st); + return rv; +} + + +// parse arguments of set command +static int32_t runset(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + const char* vstr = NULL; + int32_t oflags = 0; + int32_t mode = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-add")) { + mode = 'a'; + } else if (!std::strcmp(argv[i], "-rep")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-app")) { + mode = 'c'; + } else if (!std::strcmp(argv[i], "-inci")) { + mode = 'i'; + } else if (!std::strcmp(argv[i], "-incd")) { + mode = 'd'; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else if (!vstr) { + vstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr || !vstr) usage(); + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + int32_t rv = procset(path, kstr, ksiz, vstr, vsiz, oflags, mode); + delete[] kbuf; + delete[] vbuf; + return rv; +} + + +// parse arguments of remove command +static int32_t runremove(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procremove(path, kstr, ksiz, oflags); + delete[] kbuf; + return rv; +} + + +// parse arguments of get command +static int32_t runget(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool rm = false; + bool sx = false; + bool px = false; + bool pz = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else if (!std::strcmp(argv[i], "-pz")) { + pz = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procget(path, kstr, ksiz, oflags, rm, px, pz); + delete[] kbuf; + return rv; +} + + +// parse arguments of list command +static int32_t runlist(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool des = false; + int64_t max = -1; + bool rm = false; + bool sx = false; + bool pv = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-des")) { + des = true; + } else if (!std::strcmp(argv[i], "-max")) { + if (++i >= argc) usage(); + max = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-pv")) { + pv = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + char* kbuf = NULL; + size_t ksiz = 0; + if (kstr) { + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = new char[ksiz+1]; + std::memcpy(kbuf, kstr, ksiz); + kbuf[ksiz] = '\0'; + } + } + int32_t rv = proclist(path, kbuf, ksiz, oflags, des, max, rm, pv, px); + delete[] kbuf; + return rv; +} + + +// parse arguments of clear command +static int32_t runclear(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procclear(path, oflags); + return rv; +} + + +// parse arguments of import command +static int32_t runimport(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procimport(path, file, oflags, sx); + return rv; +} + + +// parse arguments of copy command +static int32_t runcopy(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path || !file) usage(); + int32_t rv = proccopy(path, file, oflags); + return rv; +} + + +// parse arguments of dump command +static int32_t rundump(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procdump(path, file, oflags); + return rv; +} + + +// parse arguments of load command +static int32_t runload(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::ForestDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procload(path, file, oflags); + return rv; +} + + +// parse arguments of setbulk command +static int32_t runsetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::map<std::string, std::string> recs; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + if (++i >= argc) usage(); + const char* vstr = argv[i]; + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + std::string key(kstr, ksiz); + std::string value(vstr, vsiz); + recs[key] = value; + delete[] kbuf; + delete[] vbuf; + } + } + if (!path) usage(); + int32_t rv = procsetbulk(path, oflags, recs); + return rv; +} + + +// parse arguments of removebulk command +static int32_t runremovebulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procremovebulk(path, oflags, keys); + return rv; +} + + +// parse arguments of getbulk command +static int32_t rungetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procgetbulk(path, oflags, keys, px); + return rv; +} + + +// parse arguments of check command +static int32_t runcheck(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccheck(path, oflags); + return rv; +} + + +// perform create command +static int32_t proccreate(const char* path, int32_t oflags, int32_t opts, int64_t bnum, + int32_t psiz, kc::Comparator* rcomp) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (rcomp) db.tune_comparator(rcomp); + if (!db.open(path, kc::ForestDB::OWRITER | kc::ForestDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform inform command +static int32_t procinform(const char* path, int32_t oflags, bool st) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (st) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["cusage_lcnt"] = ""; + status["cusage_lsiz"] = ""; + status["cusage_icnt"] = ""; + status["cusage_isiz"] = ""; + status["tree_level"] = ""; + if (db.status(&status)) { + uint32_t type = kc::atoi(status["realtype"].c_str()); + oprintf("type: %s (type=0x%02X) (%s)\n", + status["type"].c_str(), type, kc::BasicDB::typestring(type)); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::ForestDB::FOPEN) oprintf(" open"); + if (flags & kc::ForestDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + if (kc::atoi(status["trimmed"].c_str()) > 0) oprintf(" (trimmed)"); + oprintf("\n", flags); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::ForestDB::TSMALL) oprintf(" small"); + if (opts & kc::ForestDB::TLINEAR) oprintf(" linear"); + if (opts & kc::ForestDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + oprintf("comparator: %s\n", status["rcomp"].c_str()); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + int64_t pnum = kc::atoi(status["pnum"].c_str()); + int64_t lcnt = kc::atoi(status["lcnt"].c_str()); + int64_t icnt = kc::atoi(status["icnt"].c_str()); + int32_t tlevel = kc::atoi(status["tree_level"].c_str()); + int32_t psiz = kc::atoi(status["psiz"].c_str()); + double load = 0; + if (pnum > 0 && bnum > 0) { + load = (double)pnum / bnum; + if (!(opts & kc::ForestDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (load=%.2f)\n", (long long)bnum, load); + oprintf("pages: %lld (leaf=%lld) (inner=%lld) (level=%d) (psiz=%d)\n", + (long long)pnum, (long long)lcnt, (long long)icnt, tlevel, psiz); + int64_t pccap = kc::atoi(status["pccap"].c_str()); + int64_t cusage = kc::atoi(status["cusage"].c_str()); + int64_t culcnt = kc::atoi(status["cusage_lcnt"].c_str()); + int64_t culsiz = kc::atoi(status["cusage_lsiz"].c_str()); + int64_t cuicnt = kc::atoi(status["cusage_icnt"].c_str()); + int64_t cuisiz = kc::atoi(status["cusage_isiz"].c_str()); + oprintf("cache: %lld (cap=%lld) (ratio=%.2f) (leaf=%lld:%lld) (inner=%lld:%lld)\n", + (long long)cusage, (long long)pccap, (double)cusage / pccap, + (long long)culsiz, (long long)culcnt, (long long)cuisiz, (long long)cuicnt); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s)\n", size, sizestr.c_str()); + } else { + dberrprint(&db, "DB::status failed"); + err = true; + } + } else { + uint8_t flags = db.flags(); + if (flags != 0) { + oprintf("status:"); + if (flags & kc::ForestDB::FOPEN) oprintf(" open"); + if (flags & kc::ForestDB::FFATAL) oprintf(" fatal"); + oprintf("\n"); + } + oprintf("count: %lld\n", (long long)db.count()); + oprintf("size: %lld\n", (long long)db.size()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform set command +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + switch (mode) { + default: { + if (!db.set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 'a': { + if (!db.add(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::add failed"); + err = true; + } + break; + } + case 'r': { + if (!db.replace(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::replace failed"); + err = true; + } + break; + } + case 'c': { + if (!db.append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::append failed"); + err = true; + } + break; + } + case 'i': { + int64_t onum = db.increment(kbuf, ksiz, kc::atoi(vbuf)); + if (onum == kc::INT64MIN) { + dberrprint(&db, "DB::increment failed"); + err = true; + } else { + oprintf("%lld\n", (long long)onum); + } + break; + } + case 'd': { + double onum = db.increment_double(kbuf, ksiz, kc::atof(vbuf)); + if (kc::chknan(onum)) { + dberrprint(&db, "DB::increment_double failed"); + err = true; + } else { + oprintf("%f\n", onum); + } + break; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform remove command +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.remove(kbuf, ksiz)) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform get command +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::ForestDB::OWRITER : kc::ForestDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + char* vbuf; + size_t vsiz; + if (rm) { + vbuf = db.seize(kbuf, ksiz, &vsiz); + } else { + vbuf = db.get(kbuf, ksiz, &vsiz); + } + if (vbuf) { + printdata(vbuf, vsiz, px); + if (!pz) oprintf("\n"); + delete[] vbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform list command +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + bool des, int64_t max, bool rm, bool pv, bool px) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::ForestDB::OWRITER : kc::ForestDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(bool rm, bool pv, bool px) : rm_(rm), pv_(pv), px_(px) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + printdata(kbuf, ksiz, px_); + if (pv_) { + oprintf("\t"); + printdata(vbuf, vsiz, px_); + } + oprintf("\n"); + return rm_ ? REMOVE : NOP; + } + bool rm_; + bool pv_; + bool px_; + } visitor(rm, pv, px); + if (kbuf || des || max >= 0) { + if (max < 0) max = kc::INT64MAX; + kc::ForestDB::Cursor cur(&db); + if (des) { + if (kbuf) { + if (!cur.jump_back(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump_back() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } else { + if (kbuf) { + if (!cur.jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } + } else { + if (!db.iterate(&visitor, rm)) { + dberrprint(&db, "DB::iterate failed"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform clear command +static int32_t procclear(const char* path, int32_t oflags) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.clear()) { + dberrprint(&db, "DB::clear failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform import command +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx) { + std::istream *is = &std::cin; + std::ifstream ifs; + if (file) { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OWRITER | kc::ForestDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + int64_t cnt = 0; + std::string line; + std::vector<std::string> fields; + while (!err && mygetline(is, &line)) { + cnt++; + kc::strsplit(line, '\t', &fields); + if (sx) { + std::vector<std::string>::iterator it = fields.begin(); + std::vector<std::string>::iterator itend = fields.end(); + while (it != itend) { + size_t esiz; + char* ebuf = kc::hexdecode(it->c_str(), &esiz); + it->clear(); + it->append(ebuf, esiz); + delete[] ebuf; + ++it; + } + } + switch (fields.size()) { + case 2: { + if (!db.set(fields[0], fields[1])) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 1: { + if (!db.remove(fields[0]) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + break; + } + } + oputchar('.'); + if (cnt % 50 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + if (cnt % 50 > 0) oprintf(" (%lld)\n", (long long)cnt); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform copy command +static int32_t proccopy(const char* path, const char* file, int32_t oflags) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + DotChecker checker(&std::cout, -100); + if (!db.copy(file, &checker)) { + dberrprint(&db, "DB::copy failed"); + err = true; + } + oprintf(" (end)\n"); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld blocks were copied successfully\n", (long long)checker.count()); + return err ? 1 : 0; +} + + +// perform dump command +static int32_t procdump(const char* path, const char* file, int32_t oflags) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, 1000); + if (!db.dump_snapshot(file)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were dumped successfully\n", (long long)checker.count()); + } else { + if (!db.dump_snapshot(&std::cout)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform load command +static int32_t procload(const char* path, const char* file, int32_t oflags) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OWRITER | kc::ForestDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(file)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } else { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(&std::cin)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform setbulk command +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.set_bulk(recs) != (int64_t)recs.size()) { + dberrprint(&db, "DB::set_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform removebulk command +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.remove_bulk(keys) < 0) { + dberrprint(&db, "DB::remove_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform getbulk command +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + std::map<std::string, std::string> recs; + if (db.get_bulk(keys, &recs) >= 0) { + std::map<std::string, std::string>::iterator it = recs.begin(); + std::map<std::string, std::string>::iterator itend = recs.end(); + while (it != itend) { + printdata(it->first.data(), it->first.size(), px); + oprintf("\t"); + printdata(it->second.data(), it->second.size(), px); + oprintf("\n"); + ++it; + } + } else { + dberrprint(&db, "DB::get_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform check command +static int32_t proccheck(const char* path, int32_t oflags) { + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::ForestDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + kc::ForestDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::jump failed"); + err = true; + } + int64_t cnt = 0; + while (!err) { + size_t ksiz; + const char* vbuf; + size_t vsiz; + char* kbuf = cur.get(&ksiz, &vbuf, &vsiz); + if (kbuf) { + cnt++; + size_t rsiz; + char* rbuf = db.get(kbuf, ksiz, &rsiz); + if (rbuf) { + if (rsiz != vsiz || std::memcmp(rbuf, vbuf, rsiz)) { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] rbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] kbuf; + if (cnt % 1000 == 0) { + oputchar('.'); + if (cnt % 50000 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + } else { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::get failed"); + err = true; + } + break; + } + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::step failed"); + err = true; + } + } + oprintf(" (end)\n"); + if (db.count() != cnt) { + dberrprint(&db, "DB::count failed"); + err = true; + } + kc::File::Status sbuf; + if (!kc::File::status(path, &sbuf) || !sbuf.isdir) { + dberrprint(&db, "File::status failed"); + err = true; + } + if (db.flags() & kc::ForestDB::FFATAL) { + dberrprint(&db, "DB::flags indicated fatal error"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld records were checked successfully\n", (long long)cnt); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcforesttest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcforesttest.cc new file mode 100644 index 0000000000..778915c88b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcforesttest.cc @@ -0,0 +1,2394 @@ +/************************************************************************************************* + * The test cases of the directory tree database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcdirdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, + int64_t pccap, kc::Comparator* rcomp, bool lv); +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv); +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, + int64_t pccap, kc::Comparator* rcomp, bool lv); +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the directory tree database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-set|-get|-getw|-rem|-etc] [-tran]" + " [-oat|-oas|-onl|-otl|-onr] [-tc] [-bnum num] [-psiz num]" + " [-pccap num] [-rcd|-rcld|-rcdd] [-lv] path rnum\n", g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-oat|-oas|-onl|-otl|-onr]" + " [-tc] [-bnum num] [-psiz num] [-pccap num] [-rcd|-rcld|-rcdd] [-lv] path rnum\n", + g_progname); + eprintf(" %s wicked [-th num] [-it num] [-oat|-oas|-onl|-otl|-onr]" + " [-tc] [-bnum num] [-psiz num] [-pccap num] [-rcd|-rcld|-rcdd] [-lv] path rnum\n", + g_progname); + eprintf(" %s tran [-th num] [-it num] [-hard] [-oat|-oas|-onl|-otl|-onr]" + " [-tc] [-bnum num] [-psiz num] [-pccap num] [-rcd|-rcld|-rcdd] [-lv] path rnum\n", + g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["cusage_lcnt"] = ""; + status["cusage_lsiz"] = ""; + status["cusage_icnt"] = ""; + status["cusage_isiz"] = ""; + status["tree_level"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["realtype"].c_str()); + oprintf("type: %s (type=0x%02X) (%s)\n", + status["type"].c_str(), type, kc::BasicDB::typestring(type)); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::ForestDB::FOPEN) oprintf(" open"); + if (flags & kc::ForestDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + if (kc::atoi(status["trimmed"].c_str()) > 0) oprintf(" (trimmed)"); + oprintf("\n", flags); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::ForestDB::TSMALL) oprintf(" small"); + if (opts & kc::ForestDB::TLINEAR) oprintf(" linear"); + if (opts & kc::ForestDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + oprintf("comparator: %s\n", status["rcomp"].c_str()); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + int64_t pnum = kc::atoi(status["pnum"].c_str()); + int64_t lcnt = kc::atoi(status["lcnt"].c_str()); + int64_t icnt = kc::atoi(status["icnt"].c_str()); + int32_t tlevel = kc::atoi(status["tree_level"].c_str()); + int32_t psiz = kc::atoi(status["psiz"].c_str()); + double load = 0; + if (pnum > 0 && bnum > 0) { + load = (double)pnum / bnum; + if (!(opts & kc::ForestDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (load=%.2f)\n", (long long)bnum, load); + oprintf("pages: %lld (leaf=%lld) (inner=%lld) (level=%d) (psiz=%d)\n", + (long long)pnum, (long long)lcnt, (long long)icnt, tlevel, psiz); + int64_t pccap = kc::atoi(status["pccap"].c_str()); + int64_t cusage = kc::atoi(status["cusage"].c_str()); + int64_t culcnt = kc::atoi(status["cusage_lcnt"].c_str()); + int64_t culsiz = kc::atoi(status["cusage_lsiz"].c_str()); + int64_t cuicnt = kc::atoi(status["cusage_icnt"].c_str()); + int64_t cuisiz = kc::atoi(status["cusage_isiz"].c_str()); + oprintf("cache: %lld (cap=%lld) (ratio=%.2f) (leaf=%lld:%lld) (inner=%lld:%lld)\n", + (long long)cusage, (long long)pccap, (double)cusage / pccap, + (long long)culsiz, (long long)culcnt, (long long)cuisiz, (long long)cuicnt); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s)\n", size, sizestr.c_str()); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + int32_t mode = 0; + bool tran = false; + int32_t oflags = 0; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-set")) { + mode = 's'; + } else if (!std::strcmp(argv[i], "-get")) { + mode = 'g'; + } else if (!std::strcmp(argv[i], "-getw")) { + mode = 'w'; + } else if (!std::strcmp(argv[i], "-rem")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-etc")) { + mode = 'e'; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::ForestDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::ForestDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::ForestDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(path, rnum, thnum, rnd, mode, tran, oflags, + opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int32_t oflags = 0; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::ForestDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::ForestDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::ForestDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(path, rnum, thnum, itnum, rnd, oflags, + opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t oflags = 0; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::ForestDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::ForestDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::ForestDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(path, rnum, thnum, itnum, oflags, + opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool hard = false; + int32_t oflags = 0; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-hard")) { + hard = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::ForestDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::ForestDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::ForestDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::ForestDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::ForestDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::ForestDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(path, rnum, thnum, itnum, hard, oflags, + opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// perform order command +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, + int64_t pccap, kc::Comparator* rcomp, bool lv) { + oprintf("<In-order Test>\n seed=%u path=%s rnum=%lld thnum=%d rnd=%d mode=%d tran=%d" + " oflags=%d opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, rnd, mode, tran, oflags, opts, + (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::ForestDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + uint32_t omode = kc::ForestDB::OWRITER | kc::ForestDB::OCREATE | kc::ForestDB::OTRUNCATE; + if (mode == 'r') { + omode = kc::ForestDB::OWRITER | kc::ForestDB::OCREATE; + } else if (mode == 'g' || mode == 'w') { + omode = kc::ForestDB::OREADER; + } + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (mode == 0 || mode == 's' || mode == 'e') { + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 's'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + if (mode == 0 || mode == 'g' || mode == 'e') { + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'g'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'w' || mode == 'e') { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'w'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::ForestDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size) : + rnum_(rnum), rnd_(rnd), size_(size) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + kc::File::Status sbuf; + if (!kc::File::status(path, &sbuf) || !sbuf.isdir) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + } syncprocessor(rnum, rnd, db.size()); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e' && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'r' || mode == 'e') { + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && mode_ != 'e') || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, mode, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, mode, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'r' || mode == 'e'); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Queue Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d rnd=%d" + " oflags=%d opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, rnd, oflags, opts, + (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::ForestDB::OWRITER | kc::ForestDB::OCREATE; + if (itcnt == 1) omode |= kc::ForestDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::ForestDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::ForestDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, + int64_t pccap, kc::Comparator* rcomp, bool lv) { + oprintf("<Wicked Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d" + " oflags=%d opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, oflags, opts, + (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::ForestDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::ForestDB::OWRITER | kc::ForestDB::OCREATE; + if (itcnt == 1) omode |= kc::ForestDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::ForestDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (myrand(4) == 0) { + if (!cur->jump_back(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump_back"); + err_ = true; + } + } else { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(3) == 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + if (myrand(3) == 0 && !cur->step_back() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::ForestDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Transaction Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d hard=%d" + " oflags=%d opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, hard, oflags, opts, + (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::ForestDB db; + kc::ForestDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::ForestDB::OWRITER | kc::ForestDB::OCREATE; + if (itcnt == 1) omode |= kc::ForestDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + std::string parapath = db.path() + "-para"; + if (!paradb.open(parapath, omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::ForestDB* db, kc::ForestDB* paradb, int64_t rnum, + int32_t thnum, bool hard, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + hard_ = hard; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::ForestDB* db_; + kc::ForestDB* paradb_; + int64_t rnum_; + int32_t thnum_; + bool hard_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, hard, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, hard, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcgrasstest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcgrasstest.cc new file mode 100644 index 0000000000..75bc29d4d0 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcgrasstest.cc @@ -0,0 +1,2249 @@ +/************************************************************************************************* + * The test cases of the cache tree database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kccachedb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t procorder(int64_t rnum, int32_t thnum, bool rnd, bool etc, bool tran, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv); +static int32_t procqueue(int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv); +static int32_t procwicked(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv); +static int32_t proctran(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the cache tree database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-etc] [-tran] [-tc] [-bnum num] [-psiz num] [-pccap num]" + " [-rcd|-rcld|-rcdd] [-lv] rnum\n", g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-tc] [-bnum num] [-psiz num] [-pccap num]" + " [-rcd|-rcld|-rcdd] [-lv] rnum\n", g_progname); + eprintf(" %s wicked [-th num] [-it num] [-tc] [-bnum num] [-psiz num] [-pccap num]" + " [-rcd|-rcld|-rcdd] [-lv] rnum\n", g_progname); + eprintf(" %s tran [-th num] [-it num] [-tc] [-bnum num] [-psiz num] [-pccap num]" + " [-rcd|-rcld|-rcdd] [-lv] rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["bnum_used"] = ""; + status["cusage_lcnt"] = ""; + status["cusage_lsiz"] = ""; + status["cusage_icnt"] = ""; + status["cusage_isiz"] = ""; + status["tree_level"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::GrassDB::FOPEN) oprintf(" open"); + if (flags & kc::GrassDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + if (kc::atoi(status["trimmed"].c_str()) > 0) oprintf(" (trimmed)"); + oprintf("\n", flags); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::GrassDB::TSMALL) oprintf(" small"); + if (opts & kc::GrassDB::TLINEAR) oprintf(" linear"); + if (opts & kc::GrassDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + oprintf("comparator: %s\n", status["rcomp"].c_str()); + if (status.find("opaque") != status.end()) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t bnumused = kc::atoi(status["bnum_used"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + int64_t pnum = kc::atoi(status["pnum"].c_str()); + int64_t lcnt = kc::atoi(status["lcnt"].c_str()); + int64_t icnt = kc::atoi(status["icnt"].c_str()); + int32_t tlevel = kc::atoi(status["tree_level"].c_str()); + int32_t psiz = kc::atoi(status["psiz"].c_str()); + double load = 0; + if (pnum > 0 && bnumused > 0) { + load = (double)pnum / bnumused; + if (!(opts & kc::GrassDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (used=%lld) (load=%.2f)\n", + (long long)bnum, (long long)bnumused, load); + oprintf("pages: %lld (leaf=%lld) (inner=%lld) (level=%d) (psiz=%d)\n", + (long long)pnum, (long long)lcnt, (long long)icnt, tlevel, psiz); + int64_t pccap = kc::atoi(status["pccap"].c_str()); + int64_t cusage = kc::atoi(status["cusage"].c_str()); + int64_t culcnt = kc::atoi(status["cusage_lcnt"].c_str()); + int64_t culsiz = kc::atoi(status["cusage_lsiz"].c_str()); + int64_t cuicnt = kc::atoi(status["cusage_icnt"].c_str()); + int64_t cuisiz = kc::atoi(status["cusage_isiz"].c_str()); + oprintf("cache: %lld (cap=%lld) (ratio=%.2f) (leaf=%lld:%lld) (inner=%lld:%lld)\n", + (long long)cusage, (long long)pccap, (double)cusage / pccap, + (long long)culsiz, (long long)culcnt, (long long)cuisiz, (long long)cuicnt); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s)\n", size, sizestr.c_str()); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + bool etc = false; + bool tran = false; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-etc")) { + etc = true; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(rnum, thnum, rnd, etc, tran, opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(rnum, thnum, itnum, rnd, opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(rnum, thnum, itnum, opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::CacheDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(rnum, thnum, itnum, opts, bnum, psiz, pccap, rcomp, lv); + return rv; +} + + +// perform order command +static int32_t procorder(int64_t rnum, int32_t thnum, bool rnd, bool etc, bool tran, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<In-order Test>\n seed=%u rnum=%lld thnum=%d rnd=%d etc=%d tran=%d" + " opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, (long long)rnum, thnum, rnd, etc, tran, + opts, (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::GrassDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + if (!db.open("%", kc::GrassDB::OWRITER | kc::GrassDB::OCREATE | kc::GrassDB::OTRUNCATE)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::GrassDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size) : + rnum_(rnum), rnd_(rnd), size_(size) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + } syncprocessor(rnum, rnd, db.size()); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool etc, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + etc_ = etc; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && !etc_) || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool etc_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, etc, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, etc, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, etc); + oprintf("time: %.3f\n", etime - stime); + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Queue Test>\n seed=%u rnum=%lld thnum=%d itnum=%d rnd=%d" + " opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, rnd, + opts, (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::GrassDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::GrassDB::OWRITER | kc::GrassDB::OCREATE; + if (itcnt == 1) omode |= kc::GrassDB::OTRUNCATE; + if (!db.open("%", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::GrassDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::GrassDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Wicked Test>\n seed=%u rnum=%lld thnum=%d itnum=%d" + " opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, + opts, (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::GrassDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::GrassDB::OWRITER | kc::GrassDB::OCREATE; + if (itcnt == 1) omode |= kc::GrassDB::OTRUNCATE; + if (!db.open("%", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::GrassDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(9)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(3) == 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + if (myrand(3) == 0 && !cur->step_back() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::GrassDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(int64_t rnum, int32_t thnum, int32_t itnum, + int32_t opts, int64_t bnum, int32_t psiz, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Transaction Test>\n seed=%u rnum=%lld thnum=%d itnum=%d" + " opts=%d bnum=%lld psiz=%d pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, + opts, (long long)bnum, psiz, (long long)pccap, rcomp, lv); + bool err = false; + kc::GrassDB db; + kc::GrassDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::GrassDB::OWRITER | kc::GrassDB::OCREATE; + if (itcnt == 1) omode |= kc::GrassDB::OTRUNCATE; + if (!db.open("%", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + if (!paradb.open("para", omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::GrassDB* db, kc::GrassDB* paradb, int64_t rnum, + int32_t thnum, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::GrassDB* db_; + kc::GrassDB* paradb_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kchashdb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kchashdb.cc new file mode 100644 index 0000000000..bb3af3ba35 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kchashdb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * File hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kchashdb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kchashdb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kchashdb.h new file mode 100644 index 0000000000..9370e6ed35 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kchashdb.h @@ -0,0 +1,3865 @@ +/************************************************************************************************* + * File hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCHASHDB_H // duplication check +#define _KCHASHDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> +#include <kcplantdb.h> + +#define KCHDBMAGICDATA "KC\n" ///< magic data of the file +#define KCHDBCHKSUMSEED "__kyotocabinet__" ///< seed of the module checksum +#define KCHDBTMPPATHEXT "tmpkch" ///< extension of the temporary file + +namespace kyotocabinet { // common namespace + + +/** + * File hash database. + * @note This class is a concrete class to operate a hash database on a file. This class can be + * inherited but overwriting methods is forbidden. Before every database operation, it is + * necessary to call the HashDB::open method in order to open a database file and connect the + * database object to it. To avoid data missing or corruption, it is important to close every + * database file by the HashDB::close method when the database is no longer in use. It is + * forbidden for multible database objects in a process to open the same database at the same + * time. It is forbidden to share a database object with child processes. + */ +class HashDB : public BasicDB { + friend class PlantDB<HashDB, BasicDB::TYPETREE>; + public: + class Cursor; + private: + struct Record; + struct FreeBlock; + struct FreeBlockComparator; + class Repeater; + class ScopedVisitor; + /** An alias of set of free blocks. */ + typedef std::set<FreeBlock> FBP; + /** An alias of list of cursors. */ + typedef std::list<Cursor*> CursorList; + /** The offset of the library version. */ + static const int64_t MOFFLIBVER = 4; + /** The offset of the library revision. */ + static const int64_t MOFFLIBREV = 5; + /** The offset of the format revision. */ + static const int64_t MOFFFMTVER = 6; + /** The offset of the module checksum. */ + static const int64_t MOFFCHKSUM = 7; + /** The offset of the database type. */ + static const int64_t MOFFTYPE = 8; + /** The offset of the alignment power. */ + static const int64_t MOFFAPOW = 9; + /** The offset of the free block pool power. */ + static const int64_t MOFFFPOW = 10; + /** The offset of the options. */ + static const int64_t MOFFOPTS = 11; + /** The offset of the bucket number. */ + static const int64_t MOFFBNUM = 16; + /** The offset of the status flags. */ + static const int64_t MOFFFLAGS = 24; + /** The offset of the record number. */ + static const int64_t MOFFCOUNT = 32; + /** The offset of the file size. */ + static const int64_t MOFFSIZE = 40; + /** The offset of the opaque data. */ + static const int64_t MOFFOPAQUE = 48; + /** The size of the header. */ + static const int64_t HEADSIZ = 64; + /** The width of the free block. */ + static const int32_t FBPWIDTH = 6; + /** The large width of the record address. */ + static const int32_t WIDTHLARGE = 6; + /** The small width of the record address. */ + static const int32_t WIDTHSMALL = 4; + /** The size of the record buffer. */ + static const size_t RECBUFSIZ = 48; + /** The size of the IO buffer. */ + static const size_t IOBUFSIZ = 1024; + /** The number of slots of the record lock. */ + static const int32_t RLOCKSLOT = 1024; + /** The default alignment power. */ + static const uint8_t DEFAPOW = 3; + /** The maximum alignment power. */ + static const uint8_t MAXAPOW = 15; + /** The default free block pool power. */ + static const uint8_t DEFFPOW = 10; + /** The maximum free block pool power. */ + static const uint8_t MAXFPOW = 20; + /** The default bucket number. */ + static const int64_t DEFBNUM = 1048583LL; + /** The default size of the memory-mapped region. */ + static const int64_t DEFMSIZ = 64LL << 20; + /** The magic data for record. */ + static const uint8_t RECMAGIC = 0xcc; + /** The magic data for padding. */ + static const uint8_t PADMAGIC = 0xee; + /** The magic data for free block. */ + static const uint8_t FBMAGIC = 0xdd; + /** The maximum unit of auto defragmentation. */ + static const int32_t DFRGMAX = 512; + /** The coefficient of auto defragmentation. */ + static const int32_t DFRGCEF = 2; + /** The checking width for record salvage. */ + static const int64_t SLVGWIDTH = 1LL << 20; + /** The threshold of busy loop and sleep for locking. */ + static const uint32_t LOCKBUSYLOOP = 8192; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class HashDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(HashDB* db) : db_(db), off_(0), end_(0) { + _assert_(db); + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.push_back(this); + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + if (!db_) return; + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.remove(this); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable) { + if (!db_->writer_) { + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + if (!(db_->flags_ & FOPEN) && !db_->autotran_ && !db_->tran_ && + !db_->set_flag(FOPEN, true)) { + return false; + } + } + if (off_ < 1) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + Record rec; + char rbuf[RECBUFSIZ]; + if (!step_impl(&rec, rbuf, 0)) return false; + if (!rec.vbuf && !db_->read_record_body(&rec)) { + delete[] rec.bbuf; + return false; + } + const char* vbuf = rec.vbuf; + size_t vsiz = rec.vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (db_->comp_) { + zbuf = db_->comp_->decompress(vbuf, vsiz, &zsiz); + if (!zbuf) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed"); + delete[] rec.bbuf; + return false; + } + vbuf = zbuf; + vsiz = zsiz; + } + vbuf = visitor->visit_full(rec.kbuf, rec.ksiz, vbuf, vsiz, &vsiz); + delete[] zbuf; + if (vbuf == Visitor::REMOVE) { + uint64_t hash = db_->hash_record(rec.kbuf, rec.ksiz); + uint32_t pivot = db_->fold_hash(hash); + int64_t bidx = hash % db_->bnum_; + Repeater repeater(Visitor::REMOVE, 0); + if (!db_->accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, true)) { + delete[] rec.bbuf; + return false; + } + delete[] rec.bbuf; + } else if (vbuf == Visitor::NOP) { + delete[] rec.bbuf; + if (step) { + if (step_impl(&rec, rbuf, 1)) { + delete[] rec.bbuf; + } else if (db_->error().code() != Error::NOREC) { + return false; + } + } + } else { + zbuf = NULL; + zsiz = 0; + if (db_->comp_) { + zbuf = db_->comp_->compress(vbuf, vsiz, &zsiz); + if (!zbuf) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed"); + delete[] rec.bbuf; + return false; + } + vbuf = zbuf; + vsiz = zsiz; + } + size_t rsiz = db_->calc_record_size(rec.ksiz, vsiz); + if (rsiz <= rec.rsiz) { + rec.psiz = rec.rsiz - rsiz; + rec.vsiz = vsiz; + rec.vbuf = vbuf; + if (!db_->adjust_record(&rec) || !db_->write_record(&rec, true)) { + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + delete[] zbuf; + delete[] rec.bbuf; + if (step) { + if (step_impl(&rec, rbuf, 1)) { + delete[] rec.bbuf; + } else if (db_->error().code() != Error::NOREC) { + return false; + } + } + } else { + uint64_t hash = db_->hash_record(rec.kbuf, rec.ksiz); + uint32_t pivot = db_->fold_hash(hash); + int64_t bidx = hash % db_->bnum_; + Repeater repeater(vbuf, vsiz); + if (!db_->accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, true)) { + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + delete[] zbuf; + delete[] rec.bbuf; + } + } + if (db_->dfunit_ > 0 && db_->frgcnt_ >= db_->dfunit_) { + if (!db_->defrag_impl(db_->dfunit_ * DFRGCEF)) return false; + db_->frgcnt_ -= db_->dfunit_; + } + return true; + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + off_ = 0; + if (db_->lsiz_ <= db_->roff_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + off_ = db_->roff_; + end_ = db_->lsiz_; + return true; + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + off_ = 0; + uint64_t hash = db_->hash_record(kbuf, ksiz); + uint32_t pivot = db_->fold_hash(hash); + int64_t bidx = hash % db_->bnum_; + int64_t off = db_->get_bucket(bidx); + if (off < 0) return false; + Record rec; + char rbuf[RECBUFSIZ]; + while (off > 0) { + rec.off = off; + if (!db_->read_record(&rec, rbuf)) return false; + if (rec.psiz == UINT16MAX) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain"); + db_->report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)db_->psiz_, (long long)rec.off, (long long)db_->file_.size()); + return false; + } + uint32_t tpivot = db_->linear_ ? pivot : + db_->fold_hash(db_->hash_record(rec.kbuf, rec.ksiz)); + if (pivot > tpivot) { + delete[] rec.bbuf; + off = rec.left; + } else if (pivot < tpivot) { + delete[] rec.bbuf; + off = rec.right; + } else { + int32_t kcmp = db_->compare_keys(kbuf, ksiz, rec.kbuf, rec.ksiz); + if (db_->linear_ && kcmp != 0) kcmp = 1; + if (kcmp > 0) { + delete[] rec.bbuf; + off = rec.left; + } else if (kcmp < 0) { + delete[] rec.bbuf; + off = rec.right; + } else { + delete[] rec.bbuf; + off_ = off; + end_ = db_->lsiz_; + return true; + } + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + return jump(key.c_str(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const std::string& key) { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (off_ < 1) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + bool err = false; + Record rec; + char rbuf[RECBUFSIZ]; + if (step_impl(&rec, rbuf, 1)) { + delete[] rec.bbuf; + } else { + err = true; + } + return !err; + } + /** + * Step the cursor to the previous record. + * @note This is a dummy implementation for compatibility. + */ + bool step_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Get the database object. + * @return the database object. + */ + HashDB* db() { + _assert_(true); + return db_; + } + private: + /** + * Step the cursor to the next record. + * @param rec the record structure. + * @param rbuf the working buffer. + * @param skip the number of skipping blocks. + * @return true on success, or false on failure. + */ + bool step_impl(Record* rec, char* rbuf, int64_t skip) { + _assert_(rec && rbuf && skip >= 0); + if (off_ >= end_) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "cursor after the end"); + db_->report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)db_->psiz_, (long long)rec->off, (long long)db_->file_.size()); + return false; + } + while (off_ < end_) { + rec->off = off_; + if (!db_->read_record(rec, rbuf)) return false; + skip--; + if (rec->psiz == UINT16MAX) { + off_ += rec->rsiz; + } else { + if (skip < 0) return true; + delete[] rec->bbuf; + off_ += rec->rsiz; + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + off_ = 0; + return false; + } + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + HashDB* db_; + /** The current offset. */ + int64_t off_; + /** The end offset. */ + int64_t end_; + }; + /** + * Tuning options. + */ + enum Option { + TSMALL = 1 << 0, ///< use 32-bit addressing + TLINEAR = 1 << 1, ///< use linear collision chaining + TCOMPRESS = 1 << 2 ///< compress each record + }; + /** + * Status flags. + */ + enum Flag { + FOPEN = 1 << 0, ///< whether opened + FFATAL = 1 << 1 ///< whether with fatal error + }; + /** + * Default constructor. + */ + explicit HashDB() : + mlock_(), rlock_(RLOCKSLOT), flock_(), atlock_(), error_(), + logger_(NULL), logkinds_(0), mtrigger_(NULL), + omode_(0), writer_(false), autotran_(false), autosync_(false), + reorg_(false), trim_(false), + file_(), fbp_(), curs_(), path_(""), + libver_(0), librev_(0), fmtver_(0), chksum_(0), type_(TYPEHASH), + apow_(DEFAPOW), fpow_(DEFFPOW), opts_(0), bnum_(DEFBNUM), + flags_(0), flagopen_(false), count_(0), lsiz_(0), psiz_(0), opaque_(), + msiz_(DEFMSIZ), dfunit_(0), embcomp_(ZLIBRAWCOMP), + align_(0), fbpnum_(0), width_(0), linear_(false), + comp_(NULL), rhsiz_(0), boff_(0), roff_(0), dfcur_(0), frgcnt_(0), + tran_(false), trhard_(false), trfbp_(), trcount_(0), trsize_(0) { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~HashDB() { + _assert_(true); + if (omode_ != 0) close(); + if (!curs_.empty()) { + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->db_ = NULL; + ++cit; + } + } + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + mlock_.lock_reader(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (writable) { + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, true)) { + mlock_.unlock(); + return false; + } + } + bool err = false; + uint64_t hash = hash_record(kbuf, ksiz); + uint32_t pivot = fold_hash(hash); + int64_t bidx = hash % bnum_; + size_t lidx = bidx % RLOCKSLOT; + if (writable) { + rlock_.lock_writer(lidx); + } else { + rlock_.lock_reader(lidx); + } + if (!accept_impl(kbuf, ksiz, visitor, bidx, pivot, false)) err = true; + rlock_.unlock(lidx); + mlock_.unlock(); + if (!err && dfunit_ > 0 && frgcnt_ >= dfunit_ && mlock_.lock_writer_try()) { + int64_t unit = frgcnt_; + if (unit >= dfunit_) { + if (unit > DFRGMAX) unit = DFRGMAX; + if (!defrag_impl(unit * DFRGCEF)) err = true; + frgcnt_ -= unit; + } + mlock_.unlock(); + } + return !err; + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + mlock_.lock_reader(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (writable) { + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, true)) { + mlock_.unlock(); + return false; + } + } + visitor->visit_before(); + size_t knum = keys.size(); + if (knum < 1) { + visitor->visit_after(); + mlock_.unlock(); + return true; + } + bool err = false; + struct RecordKey { + const char* kbuf; + size_t ksiz; + uint32_t pivot; + uint64_t bidx; + }; + RecordKey* rkeys = new RecordKey[knum]; + std::set<size_t> lidxs; + for (size_t i = 0; i < knum; i++) { + const std::string& key = keys[i]; + RecordKey* rkey = rkeys + i; + rkey->kbuf = key.data(); + rkey->ksiz = key.size(); + uint64_t hash = hash_record(rkey->kbuf, rkey->ksiz); + rkey->pivot = fold_hash(hash); + rkey->bidx = hash % bnum_; + lidxs.insert(rkey->bidx % RLOCKSLOT); + } + std::set<size_t>::iterator lit = lidxs.begin(); + std::set<size_t>::iterator litend = lidxs.end(); + while (lit != litend) { + if (writable) { + rlock_.lock_writer(*lit); + } else { + rlock_.lock_reader(*lit); + } + ++lit; + } + for (size_t i = 0; i < knum; i++) { + RecordKey* rkey = rkeys + i; + if (!accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->bidx, rkey->pivot, false)) { + err = true; + break; + } + } + lit = lidxs.begin(); + litend = lidxs.end(); + while (lit != litend) { + rlock_.unlock(*lit); + ++lit; + } + delete[] rkeys; + visitor->visit_after(); + mlock_.unlock(); + if (!err && dfunit_ > 0 && frgcnt_ >= dfunit_ && mlock_.lock_writer_try()) { + int64_t unit = frgcnt_; + if (unit >= dfunit_) { + if (unit > DFRGMAX) unit = DFRGMAX; + if (!defrag_impl(unit * DFRGCEF)) err = true; + frgcnt_ -= unit; + } + mlock_.unlock(); + } + return !err; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable) { + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, true)) { + mlock_.unlock(); + return false; + } + } + ScopedVisitor svis(visitor); + bool err = false; + if (!iterate_impl(visitor, checker)) err = true; + trigger_meta(MetaTrigger::ITERATE, "iterate"); + return !err; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (thnum < 1) thnum = 1; + if (thnum > (size_t)INT8MAX) thnum = INT8MAX; + if ((int64_t)thnum > bnum_) thnum = bnum_; + ScopedVisitor svis(visitor); + rlock_.lock_reader_all(); + bool err = false; + if (!scan_parallel_impl(visitor, thnum, checker)) err = true; + rlock_.unlock_all(); + trigger_meta(MetaTrigger::ITERATE, "scan_parallel"); + return !err; + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + return error_; + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + error_->set(code, message); + if (code == Error::BROKEN || code == Error::SYSTEM) flags_ |= FFATAL; + if (logger_) { + Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ? + Logger::ERROR : Logger::INFO; + if (kind & logkinds_) + report(file, line, func, kind, "%d: %s: %s", code, Error::codename(code), message); + } + } + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. HashDB::OWRITER as a writer, HashDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: HashDB::OCREATE, + * which means it creates a new database if the file does not exist, HashDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, HashDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, HashDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: HashDB::ONOLOCK, which means it opens the database file without file locking, + * HashDB::OTRYLOCK, which means locking is performed without blocking, HashDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the HashDB::close method when it is no + * longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", path.c_str()); + writer_ = false; + autotran_ = false; + autosync_ = false; + reorg_ = false; + trim_ = false; + uint32_t fmode = File::OREADER; + if (mode & OWRITER) { + writer_ = true; + fmode = File::OWRITER; + if (mode & OCREATE) fmode |= File::OCREATE; + if (mode & OTRUNCATE) fmode |= File::OTRUNCATE; + if (mode & OAUTOTRAN) autotran_ = true; + if (mode & OAUTOSYNC) autosync_ = true; + } + if (mode & ONOLOCK) fmode |= File::ONOLOCK; + if (mode & OTRYLOCK) fmode |= File::OTRYLOCK; + if (!file_.open(path, fmode, msiz_)) { + const char* emsg = file_.error(); + Error::Code code = Error::SYSTEM; + if (std::strstr(emsg, "(permission denied)") || std::strstr(emsg, "(directory)")) { + code = Error::NOPERM; + } else if (std::strstr(emsg, "(file not found)") || std::strstr(emsg, "(invalid path)")) { + code = Error::NOREPOS; + } + set_error(_KCCODELINE_, code, emsg); + return false; + } + if (file_.recovered()) report(_KCCODELINE_, Logger::WARN, "recovered by the WAL file"); + if ((mode & OWRITER) && file_.size() < 1) { + calc_meta(); + libver_ = LIBVER; + librev_ = LIBREV; + fmtver_ = FMTVER; + chksum_ = calc_checksum(); + lsiz_ = roff_; + if (!file_.truncate(lsiz_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + file_.close(); + return false; + } + if (!dump_meta()) { + file_.close(); + return false; + } + if (autosync_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + file_.close(); + return false; + } + } + if (!load_meta()) { + file_.close(); + return false; + } + calc_meta(); + uint8_t chksum = calc_checksum(); + if (chksum != chksum_) { + set_error(_KCCODELINE_, Error::INVALID, "invalid module checksum"); + report(_KCCODELINE_, Logger::WARN, "saved=%02X calculated=%02X", + (unsigned)chksum_, (unsigned)chksum); + file_.close(); + return false; + } + if (((flags_ & FOPEN) || (flags_ & FFATAL)) && !(mode & ONOREPAIR) && !(mode & ONOLOCK)) { + if (!reorganize_file(path)) { + file_.close(); + return false; + } + if (!file_.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + if (!file_.open(path, fmode, msiz_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + if (!load_meta()) { + file_.close(); + return false; + } + calc_meta(); + reorg_ = true; + } + if (type_ == 0 || apow_ > MAXAPOW || fpow_ > MAXFPOW || + bnum_ < 1 || count_ < 0 || lsiz_ < roff_) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data"); + report(_KCCODELINE_, Logger::WARN, "type=0x%02X apow=%d fpow=%d bnum=%lld count=%lld" + " lsiz=%lld fsiz=%lld", (unsigned)type_, (int)apow_, (int)fpow_, (long long)bnum_, + (long long)count_, (long long)lsiz_, (long long)file_.size()); + file_.close(); + return false; + } + if (file_.size() < lsiz_) { + set_error(_KCCODELINE_, Error::BROKEN, "inconsistent file size"); + report(_KCCODELINE_, Logger::WARN, "lsiz=%lld fsiz=%lld", + (long long)lsiz_, (long long)file_.size()); + file_.close(); + return false; + } + if (file_.size() != lsiz_ && !(mode & ONOREPAIR) && !(mode & ONOLOCK) && !trim_file(path)) { + file_.close(); + return false; + } + if (mode & OWRITER) { + if (!(flags_ & FOPEN) && !(flags_ & FFATAL) && !load_free_blocks()) { + file_.close(); + return false; + } + if (!dump_empty_free_blocks()) { + file_.close(); + return false; + } + if (!autotran_ && !set_flag(FOPEN, true)) { + file_.close(); + return false; + } + } + path_.append(path); + omode_ = mode; + trigger_meta(MetaTrigger::OPEN, "open"); + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", path_.c_str()); + bool err = false; + if (tran_ && !abort_transaction()) err = true; + disable_cursors(); + if (writer_) { + if (!dump_free_blocks()) err = true; + if (!dump_meta()) err = true; + } + if (!file_.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + fbp_.clear(); + omode_ = 0; + path_.clear(); + trigger_meta(MetaTrigger::CLOSE, "close"); + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + rlock_.lock_reader_all(); + bool err = false; + if (!synchronize_impl(hard, proc, checker)) err = true; + trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize"); + rlock_.unlock_all(); + return !err; + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, writable); + bool err = false; + if (proc && !proc->process(path_, count_, lsiz_)) { + set_error(_KCCODELINE_, Error::LOGIC, "processing failed"); + err = true; + } + trigger_meta(MetaTrigger::OCCUPY, "occupy"); + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + uint32_t wcnt = 0; + while (true) { + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!tran_) break; + mlock_.unlock(); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + trhard_ = hard; + if (!begin_transaction_impl()) { + mlock_.unlock(); + return false; + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); + mlock_.unlock(); + return true; + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (tran_) { + set_error(_KCCODELINE_, Error::LOGIC, "competition avoided"); + mlock_.unlock(); + return false; + } + trhard_ = hard; + if (!begin_transaction_impl()) { + mlock_.unlock(); + return false; + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try"); + mlock_.unlock(); + return true; + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!tran_) { + set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); + return false; + } + bool err = false; + if (commit) { + if (!commit_transaction()) err = true; + } else { + if (!abort_transaction()) err = true; + } + tran_ = false; + trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction"); + return !err; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + disable_cursors(); + if (!file_.truncate(HEADSIZ)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + fbp_.clear(); + bool err = false; + reorg_ = false; + trim_ = false; + flags_ = 0; + flagopen_ = false; + count_ = 0; + lsiz_ = roff_; + psiz_ = lsiz_; + dfcur_ = roff_; + std::memset(opaque_, 0, sizeof(opaque_)); + if (!file_.truncate(lsiz_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + if (!dump_meta()) err = true; + if (!autotran_ && !set_flag(FOPEN, true)) err = true; + trigger_meta(MetaTrigger::CLEAR, "clear"); + return true; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return count_; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return lsiz_; + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return ""; + } + return path_; + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + (*strmap)["type"] = strprintf("%u", (unsigned)TYPEHASH); + (*strmap)["realtype"] = strprintf("%u", (unsigned)type_); + (*strmap)["path"] = path_; + (*strmap)["libver"] = strprintf("%u", libver_); + (*strmap)["librev"] = strprintf("%u", librev_); + (*strmap)["fmtver"] = strprintf("%u", fmtver_); + (*strmap)["chksum"] = strprintf("%u", chksum_); + (*strmap)["flags"] = strprintf("%u", flags_); + (*strmap)["apow"] = strprintf("%u", apow_); + (*strmap)["fpow"] = strprintf("%u", fpow_); + (*strmap)["opts"] = strprintf("%u", opts_); + (*strmap)["bnum"] = strprintf("%lld", (long long)bnum_); + (*strmap)["msiz"] = strprintf("%lld", (long long)msiz_); + (*strmap)["dfunit"] = strprintf("%lld", (long long)dfunit_); + (*strmap)["frgcnt"] = strprintf("%lld", (long long)(frgcnt_ > 0 ? (int64_t)frgcnt_ : 0)); + (*strmap)["realsize"] = strprintf("%lld", (long long)file_.size()); + (*strmap)["recovered"] = strprintf("%d", file_.recovered()); + (*strmap)["reorganized"] = strprintf("%d", reorg_); + (*strmap)["trimmed"] = strprintf("%d", trim_); + if (strmap->count("opaque") > 0) + (*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_)); + if (strmap->count("fbpnum_used") > 0) { + if (writer_) { + (*strmap)["fbpnum_used"] = strprintf("%lld", (long long)fbp_.size()); + } else { + if (!load_free_blocks()) return false; + (*strmap)["fbpnum_used"] = strprintf("%lld", (long long)fbp_.size()); + fbp_.clear(); + } + } + if (strmap->count("bnum_used") > 0) { + int64_t cnt = 0; + for (int64_t i = 0; i < bnum_; i++) { + if (get_bucket(i) > 0) cnt++; + } + (*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt); + } + (*strmap)["count"] = strprintf("%lld", (long long)count_); + (*strmap)["size"] = strprintf("%lld", (long long)lsiz_); + return true; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + ScopedRWLock lock(&mlock_, false); + if (!logger_) return; + logger_->log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + logger_ = logger; + logkinds_ = kinds; + return true; + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + /** + * Set the power of the alignment of record size. + * @param apow the power of the alignment of record size. + * @return true on success, or false on failure. + */ + bool tune_alignment(int8_t apow) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + apow_ = apow >= 0 ? apow : DEFAPOW; + if (apow_ > MAXAPOW) apow_ = MAXAPOW; + return true; + } + /** + * Set the power of the capacity of the free block pool. + * @param fpow the power of the capacity of the free block pool. + * @return true on success, or false on failure. + */ + bool tune_fbp(int8_t fpow) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + fpow_ = fpow >= 0 ? fpow : DEFFPOW; + if (fpow_ > MAXFPOW) fpow_ = MAXFPOW; + return true; + } + /** + * Set the optional features. + * @param opts the optional features by bitwise-or: HashDB::TSMALL to use 32-bit addressing, + * HashDB::TLINEAR to use linear collision chaining, HashDB::TCOMPRESS to compress each record. + * @return true on success, or false on failure. + */ + bool tune_options(int8_t opts) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + opts_ = opts; + return true; + } + /** + * Set the number of buckets of the hash table. + * @param bnum the number of buckets of the hash table. + * @return true on success, or false on failure. + */ + bool tune_buckets(int64_t bnum) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + bnum_ = bnum > 0 ? bnum : DEFBNUM; + if (bnum_ > INT16MAX) bnum_ = nearbyprime(bnum_); + return true; + } + /** + * Set the size of the internal memory-mapped region. + * @param msiz the size of the internal memory-mapped region. + * @return true on success, or false on failure. + */ + bool tune_map(int64_t msiz) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + msiz_ = msiz >= 0 ? msiz : DEFMSIZ; + return true; + } + /** + * Set the unit step number of auto defragmentation. + * @param dfunit the unit step number of auto defragmentation. + * @return true on success, or false on failure. + */ + bool tune_defrag(int64_t dfunit) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + dfunit_ = dfunit > 0 ? dfunit : 0; + return true; + } + /** + * Set the data compressor. + * @param comp the data compressor object. + * @return true on success, or false on failure. + */ + bool tune_compressor(Compressor* comp) { + _assert_(comp); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + embcomp_ = comp; + return true; + } + /** + * Get the opaque data. + * @return the pointer to the opaque data region, whose size is 16 bytes. + */ + char* opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return opaque_; + } + /** + * Synchronize the opaque data. + * @return true on success, or false on failure. + */ + bool synchronize_opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + bool err = false; + if (!dump_opaque()) err = true; + return !err; + } + /** + * Perform defragmentation of the file. + * @param step the number of steps. If it is not more than 0, the whole region is defraged. + * @return true on success, or false on failure. + */ + bool defrag(int64_t step = 0) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + bool err = false; + if (step > 0) { + if (!defrag_impl(step)) err = true; + } else { + dfcur_ = roff_; + if (!defrag_impl(INT64MAX)) err = true; + } + frgcnt_ = 0; + return !err; + } + /** + * Get the status flags. + * @return the status flags, or 0 on failure. + */ + uint8_t flags() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return flags_; + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, ...) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + va_list ap; + va_start(ap, format); + vstrprintf(&message, format, ap); + va_end(ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report a message for debugging with variable number of arguments. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ap used according to the format string. + */ + void report_valist(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, va_list ap) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + vstrprintf(&message, format, ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report the content of a binary buffer for debugging. + * @param file the file name of the epicenter. + * @param line the line number of the epicenter. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param name the name of the information. + * @param buf the binary buffer. + * @param size the size of the binary buffer + */ + void report_binary(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* name, const char* buf, size_t size) { + _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ); + if (!logger_) return; + char* hex = hexencode(buf, size); + report(file, line, func, kind, "%s=%s", name, hex); + delete[] hex; + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN for beginning + * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaTrigger::ABORTTRAN + * for aborting transaction, and MetaTrigger::MISC for miscellaneous operations. + * @param message the supplement message. + */ + void trigger_meta(MetaTrigger::Kind kind, const char* message) { + _assert_(message); + if (mtrigger_) mtrigger_->trigger(kind, message); + } + /** + * Set the database type. + * @param type the database type. + * @return true on success, or false on failure. + */ + bool tune_type(int8_t type) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + type_ = type; + return true; + } + /** + * Get the library version. + * @return the library version, or 0 on failure. + */ + uint8_t libver() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return libver_; + } + /** + * Get the library revision. + * @return the library revision, or 0 on failure. + */ + uint8_t librev() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return librev_; + } + /** + * Get the format version. + * @return the format version, or 0 on failure. + */ + uint8_t fmtver() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return fmtver_; + } + /** + * Get the module checksum. + * @return the module checksum, or 0 on failure. + */ + uint8_t chksum() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return chksum_; + } + /** + * Get the database type. + * @return the database type, or 0 on failure. + */ + uint8_t type() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return type_; + } + /** + * Get the alignment power. + * @return the alignment power, or 0 on failure. + */ + uint8_t apow() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return apow_; + } + /** + * Get the free block pool power. + * @return the free block pool power, or 0 on failure. + */ + uint8_t fpow() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return fpow_; + } + /** + * Get the options. + * @return the options, or 0 on failure. + */ + uint8_t opts() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return opts_; + } + /** + * Get the bucket number. + * @return the bucket number, or 0 on failure. + */ + int64_t bnum() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return bnum_; + } + /** + * Get the size of the internal memory-mapped region. + * @return the size of the internal memory-mapped region, or 0 on failure. + */ + int64_t msiz() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return msiz_; + } + /** + * Get the unit step number of auto defragmentation. + * @return the unit step number of auto defragmentation, or 0 on failure. + */ + int64_t dfunit() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return dfunit_; + } + /** + * Get the data compressor. + * @return the data compressor, or NULL on failure. + */ + Compressor* comp() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return comp_; + } + /** + * Check whether the database was recovered or not. + * @return true if recovered, or false if not. + */ + bool recovered() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return file_.recovered(); + } + /** + * Check whether the database was reorganized or not. + * @return true if reorganized, or false if not. + */ + bool reorganized() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return reorg_; + } + private: + /** + * Record data. + */ + struct Record { + int64_t off; ///< offset + size_t rsiz; ///< whole size + size_t psiz; ///< size of the padding + size_t ksiz; ///< size of the key + size_t vsiz; ///< size of the value + int64_t left; ///< address of the left child record + int64_t right; ///< address of the right child record + const char* kbuf; ///< pointer to the key + const char* vbuf; ///< pointer to the value + int64_t boff; ///< offset of the body + char* bbuf; ///< buffer of the body + }; + /** + * Free block data. + */ + struct FreeBlock { + int64_t off; ///< offset + size_t rsiz; ///< record size + /** comparing operator */ + bool operator <(const FreeBlock& obj) const { + _assert_(true); + if (rsiz < obj.rsiz) return true; + if (rsiz == obj.rsiz && off > obj.off) return true; + return false; + } + }; + /** + * Comparator for free blocks. + */ + struct FreeBlockComparator { + /** comparing operator */ + bool operator ()(const FreeBlock& a, const FreeBlock& b) const { + _assert_(true); + return a.off < b.off; + } + }; + /** + * Repeating visitor. + */ + class Repeater : public Visitor { + public: + /** constructor */ + explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsiz) { + _assert_(vbuf); + } + private: + /** visit a record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; + size_t vsiz_; + }; + /** + * Scoped visitor. + */ + class ScopedVisitor { + public: + /** constructor */ + explicit ScopedVisitor(Visitor* visitor) : visitor_(visitor) { + _assert_(visitor); + visitor_->visit_before(); + } + /** destructor */ + ~ScopedVisitor() { + _assert_(true); + visitor_->visit_after(); + } + private: + Visitor* visitor_; ///< visitor + }; + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param bidx the bucket index. + * @param pivot the second hash value. + @ @param isiter true for iterator use, or false for direct use. + * @return true on success, or false on failure. + */ + bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, + int64_t bidx, uint32_t pivot, bool isiter) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && bidx >= 0); + int64_t top = get_bucket(bidx); + int64_t off = top; + if (off < 0) return false; + enum { DIREMPTY, DIRLEFT, DIRRIGHT, DIRMIXED } entdir = DIREMPTY; + int64_t entoff = 0; + Record rec; + char rbuf[RECBUFSIZ]; + while (off > 0) { + rec.off = off; + if (!read_record(&rec, rbuf)) return false; + if (rec.psiz == UINT16MAX) { + set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)rec.off, (long long)file_.size()); + return false; + } + uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, rec.ksiz)); + if (pivot > tpivot) { + delete[] rec.bbuf; + off = rec.left; + switch (entdir) { + case DIREMPTY: entdir = DIRLEFT; break; + case DIRRIGHT: entdir = DIRMIXED; break; + default: break; + } + entoff = rec.off + sizeof(uint16_t); + } else if (pivot < tpivot) { + delete[] rec.bbuf; + off = rec.right; + switch (entdir) { + case DIREMPTY: entdir = DIRRIGHT; break; + case DIRLEFT: entdir = DIRMIXED; break; + default: break; + } + entoff = rec.off + sizeof(uint16_t) + width_; + } else { + int32_t kcmp = compare_keys(kbuf, ksiz, rec.kbuf, rec.ksiz); + if (linear_ && kcmp != 0) kcmp = 1; + if (kcmp > 0) { + delete[] rec.bbuf; + off = rec.left; + switch (entdir) { + case DIREMPTY: entdir = DIRLEFT; break; + case DIRRIGHT: entdir = DIRMIXED; break; + default: break; + } + entoff = rec.off + sizeof(uint16_t); + } else if (kcmp < 0) { + delete[] rec.bbuf; + off = rec.right; + switch (entdir) { + case DIREMPTY: entdir = DIRRIGHT; break; + case DIRLEFT: entdir = DIRMIXED; break; + default: break; + } + entoff = rec.off + sizeof(uint16_t) + width_; + } else { + if (!rec.vbuf && !read_record_body(&rec)) { + delete[] rec.bbuf; + return false; + } + const char* vbuf = rec.vbuf; + size_t vsiz = rec.vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp_) { + zbuf = comp_->decompress(vbuf, vsiz, &zsiz); + if (!zbuf) { + set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed"); + delete[] rec.bbuf; + return false; + } + vbuf = zbuf; + vsiz = zsiz; + } + vbuf = visitor->visit_full(kbuf, ksiz, vbuf, vsiz, &vsiz); + delete[] zbuf; + if (vbuf == Visitor::REMOVE) { + bool atran = false; + if (autotran_ && !tran_) { + if (!begin_auto_transaction()) { + delete[] rec.bbuf; + return false; + } + atran = true; + } + if (!write_free_block(rec.off, rec.rsiz, rbuf)) { + if (atran) abort_auto_transaction(); + delete[] rec.bbuf; + return false; + } + insert_free_block(rec.off, rec.rsiz); + frgcnt_ += 1; + delete[] rec.bbuf; + if (!cut_chain(&rec, rbuf, bidx, entoff)) { + if (atran) abort_auto_transaction(); + return false; + } + count_ -= 1; + if (atran) { + if (!commit_auto_transaction()) return false; + } else if (autosync_) { + if (!synchronize_meta()) return false; + } + } else if (vbuf == Visitor::NOP) { + delete[] rec.bbuf; + } else { + zbuf = NULL; + zsiz = 0; + if (comp_ && !isiter) { + zbuf = comp_->compress(vbuf, vsiz, &zsiz); + if (!zbuf) { + set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed"); + delete[] rec.bbuf; + return false; + } + vbuf = zbuf; + vsiz = zsiz; + } + bool atran = false; + if (autotran_ && !tran_) { + if (!begin_auto_transaction()) { + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + atran = true; + } + size_t rsiz = calc_record_size(rec.ksiz, vsiz); + if (rsiz <= rec.rsiz) { + rec.psiz = rec.rsiz - rsiz; + rec.vsiz = vsiz; + rec.vbuf = vbuf; + if (!adjust_record(&rec) || !write_record(&rec, true)) { + if (atran) abort_auto_transaction(); + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + delete[] zbuf; + delete[] rec.bbuf; + } else { + if (!write_free_block(rec.off, rec.rsiz, rbuf)) { + if (atran) abort_auto_transaction(); + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + insert_free_block(rec.off, rec.rsiz); + frgcnt_ += 1; + size_t psiz = calc_record_padding(rsiz); + rec.rsiz = rsiz + psiz; + rec.psiz = psiz; + rec.vsiz = vsiz; + rec.vbuf = vbuf; + bool over = false; + FreeBlock fb; + if (!isiter && fetch_free_block(rec.rsiz, &fb)) { + rec.off = fb.off; + rec.rsiz = fb.rsiz; + rec.psiz = rec.rsiz - rsiz; + over = true; + if (!adjust_record(&rec)) { + if (atran) abort_auto_transaction(); + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + } else { + rec.off = lsiz_.add(rec.rsiz); + } + if (!write_record(&rec, over)) { + if (atran) abort_auto_transaction(); + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + if (!over) psiz_.secure_least(rec.off + rec.rsiz); + delete[] zbuf; + delete[] rec.bbuf; + if (entoff > 0) { + if (!set_chain(entoff, rec.off)) { + if (atran) abort_auto_transaction(); + return false; + } + } else { + if (!set_bucket(bidx, rec.off)) { + if (atran) abort_auto_transaction(); + return false; + } + } + } + if (atran) { + if (!commit_auto_transaction()) return false; + } else if (autosync_) { + if (!synchronize_meta()) return false; + } + } + return true; + } + } + } + size_t vsiz; + const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + char* zbuf = NULL; + size_t zsiz = 0; + if (comp_) { + zbuf = comp_->compress(vbuf, vsiz, &zsiz); + if (!zbuf) { + set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed"); + return false; + } + vbuf = zbuf; + vsiz = zsiz; + } + bool atran = false; + if (autotran_ && !tran_) { + if (!begin_auto_transaction()) { + delete[] zbuf; + return false; + } + atran = true; + } + size_t rsiz = calc_record_size(ksiz, vsiz); + size_t psiz = calc_record_padding(rsiz); + rec.rsiz = rsiz + psiz; + rec.psiz = psiz; + rec.ksiz = ksiz; + rec.vsiz = vsiz; + switch (entdir) { + default: { + rec.left = 0; + rec.right = 0; + break; + } + case DIRLEFT: { + if (linear_) { + rec.left = top; + rec.right = 0; + } else { + rec.left = 0; + rec.right = top; + } + break; + } + case DIRRIGHT: { + rec.left = top; + rec.right = 0; + break; + } + } + rec.kbuf = kbuf; + rec.vbuf = vbuf; + bool over = false; + FreeBlock fb; + if (fetch_free_block(rec.rsiz, &fb)) { + rec.off = fb.off; + rec.rsiz = fb.rsiz; + rec.psiz = rec.rsiz - rsiz; + over = true; + if (!adjust_record(&rec)) { + if (atran) abort_auto_transaction(); + delete[] zbuf; + return false; + } + } else { + rec.off = lsiz_.add(rec.rsiz); + } + if (!write_record(&rec, over)) { + if (atran) abort_auto_transaction(); + delete[] zbuf; + return false; + } + if (!over) psiz_.secure_least(rec.off + rec.rsiz); + delete[] zbuf; + if (entoff < 1 || entdir == DIRLEFT || entdir == DIRRIGHT) { + if (!set_bucket(bidx, rec.off)) { + if (atran) abort_auto_transaction(); + return false; + } + } else { + if (!set_chain(entoff, rec.off)) { + if (atran) abort_auto_transaction(); + return false; + } + } + count_ += 1; + if (atran) { + if (!commit_auto_transaction()) return false; + } else if (autosync_) { + if (!synchronize_meta()) return false; + } + } + return true; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool iterate_impl(Visitor* visitor, ProgressChecker* checker) { + _assert_(visitor); + int64_t allcnt = count_; + if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + int64_t off = roff_; + int64_t end = lsiz_; + Record rec; + char rbuf[RECBUFSIZ]; + int64_t curcnt = 0; + while (off > 0 && off < end) { + rec.off = off; + if (!read_record(&rec, rbuf)) return false; + if (rec.psiz == UINT16MAX) { + off += rec.rsiz; + } else { + if (!rec.vbuf && !read_record_body(&rec)) { + delete[] rec.bbuf; + return false; + } + const char* vbuf = rec.vbuf; + size_t vsiz = rec.vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp_) { + zbuf = comp_->decompress(vbuf, vsiz, &zsiz); + if (!zbuf) { + set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed"); + delete[] rec.bbuf; + return false; + } + vbuf = zbuf; + vsiz = zsiz; + } + vbuf = visitor->visit_full(rec.kbuf, rec.ksiz, vbuf, vsiz, &vsiz); + delete[] zbuf; + if (vbuf == Visitor::REMOVE) { + uint64_t hash = hash_record(rec.kbuf, rec.ksiz); + uint32_t pivot = fold_hash(hash); + int64_t bidx = hash % bnum_; + Repeater repeater(Visitor::REMOVE, 0); + if (!accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, true)) { + delete[] rec.bbuf; + return false; + } + delete[] rec.bbuf; + } else if (vbuf == Visitor::NOP) { + delete[] rec.bbuf; + } else { + zbuf = NULL; + zsiz = 0; + if (comp_) { + zbuf = comp_->compress(vbuf, vsiz, &zsiz); + if (!zbuf) { + set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed"); + delete[] rec.bbuf; + return false; + } + vbuf = zbuf; + vsiz = zsiz; + } + size_t rsiz = calc_record_size(rec.ksiz, vsiz); + if (rsiz <= rec.rsiz) { + rec.psiz = rec.rsiz - rsiz; + rec.vsiz = vsiz; + rec.vbuf = vbuf; + if (!adjust_record(&rec) || !write_record(&rec, true)) { + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + delete[] zbuf; + delete[] rec.bbuf; + } else { + uint64_t hash = hash_record(rec.kbuf, rec.ksiz); + uint32_t pivot = fold_hash(hash); + int64_t bidx = hash % bnum_; + Repeater repeater(vbuf, vsiz); + if (!accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, true)) { + delete[] zbuf; + delete[] rec.bbuf; + return false; + } + delete[] zbuf; + delete[] rec.bbuf; + } + } + off += rec.rsiz; + curcnt++; + if (checker && !checker->check("iterate", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + } + } + if (checker && !checker->check("iterate", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + return true; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool scan_parallel_impl(Visitor *visitor, size_t thnum, ProgressChecker* checker) { + _assert_(visitor && thnum <= MEMMAXSIZ); + int64_t allcnt = count_; + if (checker && !checker->check("scan_parallel", "beginning", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + bool err = false; + std::vector<int64_t> offs; + int64_t bnum = bnum_; + size_t cap = (thnum + 1) * INT8MAX; + for (int64_t bidx = 0; bidx < bnum; bidx++) { + int64_t off = get_bucket(bidx); + if (off > 0) { + offs.push_back(off); + if (offs.size() >= cap) break; + } + } + if (!offs.empty()) { + std::sort(offs.begin(), offs.end()); + if (thnum > offs.size()) thnum = offs.size(); + class ThreadImpl : public Thread { + public: + explicit ThreadImpl() : + db_(NULL), visitor_(NULL), checker_(NULL), allcnt_(0), + begoff_(0), endoff_(0), error_() {} + void init(HashDB* db, Visitor* visitor, ProgressChecker* checker, int64_t allcnt, + int64_t begoff, int64_t endoff) { + db_ = db; + visitor_ = visitor; + checker_ = checker; + allcnt_ = allcnt; + begoff_ = begoff; + endoff_ = endoff; + } + const Error& error() { + return error_; + } + private: + void run() { + HashDB* db = db_; + Visitor* visitor = visitor_; + ProgressChecker* checker = checker_; + int64_t off = begoff_; + int64_t end = endoff_; + int64_t allcnt = allcnt_; + Compressor* comp = db->comp_; + Record rec; + char rbuf[RECBUFSIZ]; + while (off > 0 && off < end) { + rec.off = off; + if (!db->read_record(&rec, rbuf)) { + error_ = db->error(); + break; + } + if (rec.psiz == UINT16MAX) { + off += rec.rsiz; + } else { + if (!rec.vbuf && !db->read_record_body(&rec)) { + delete[] rec.bbuf; + error_ = db->error(); + break; + } + const char* vbuf = rec.vbuf; + size_t vsiz = rec.vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp) { + zbuf = comp->decompress(vbuf, vsiz, &zsiz); + if (!zbuf) { + db->set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed"); + delete[] rec.bbuf; + error_ = db->error(); + break; + } + vbuf = zbuf; + vsiz = zsiz; + } + visitor->visit_full(rec.kbuf, rec.ksiz, vbuf, vsiz, &vsiz); + delete[] zbuf; + delete[] rec.bbuf; + off += rec.rsiz; + if (checker && !checker->check("scan_parallel", "processing", -1, allcnt)) { + db->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + error_ = db->error(); + break; + } + } + } + } + HashDB* db_; + Visitor* visitor_; + ProgressChecker* checker_; + int64_t allcnt_; + int64_t begoff_; + int64_t endoff_; + Error error_; + }; + ThreadImpl* threads = new ThreadImpl[thnum]; + double range = (double)offs.size() / thnum; + for (size_t i = 0; i < thnum; i++) { + int64_t cidx = i * range; + int64_t nidx = (i + 1) * range; + int64_t begoff = i < 1 ? roff_ : offs[cidx]; + int64_t endoff = i < thnum - 1 ? offs[nidx] : (int64_t)lsiz_; + ThreadImpl* thread = threads + i; + thread->init(this, visitor, checker, allcnt, begoff, endoff); + thread->start(); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->join(); + if (thread->error() != Error::SUCCESS) { + *error_ = thread->error(); + err = true; + } + } + delete[] threads; + } + if (checker && !checker->check("scan_parallel", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool synchronize_impl(bool hard, FileProcessor* proc, ProgressChecker* checker) { + _assert_(true); + bool err = false; + if (writer_) { + if (checker && !checker->check("synchronize", "dumping the free blocks", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (hard && !dump_free_blocks()) err = true; + if (checker && !checker->check("synchronize", "dumping the meta data", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!dump_meta()) err = true; + if (checker && !checker->check("synchronize", "synchronizing the file", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!file_.synchronize(hard)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + } + if (proc) { + if (checker && !checker->check("synchronize", "running the post processor", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!proc->process(path_, count_, lsiz_)) { + set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed"); + err = true; + } + } + if (writer_ && !autotran_ && !set_flag(FOPEN, true)) err = true; + return !err; + } + /** + * Synchronize meta data with the file and the device. + * @return true on success, or false on failure. + */ + bool synchronize_meta() { + _assert_(true); + ScopedMutex lock(&flock_); + bool err = false; + if (!dump_meta()) err = true; + if (!file_.synchronize(true)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + return !err; + } + /** + * Perform defragmentation. + * @param step the number of steps. + * @return true on success, or false on failure. + */ + bool defrag_impl(int64_t step) { + _assert_(step >= 0); + int64_t end = lsiz_; + Record rec; + char rbuf[RECBUFSIZ]; + while (true) { + if (dfcur_ >= end) { + dfcur_ = roff_; + return true; + } + if (step-- < 1) return true; + rec.off = dfcur_; + if (!read_record(&rec, rbuf)) return false; + if (rec.psiz == UINT16MAX) break; + delete[] rec.bbuf; + dfcur_ += rec.rsiz; + } + bool atran = false; + if (autotran_ && !tran_) { + if (!begin_auto_transaction()) return false; + atran = true; + } + int64_t base = dfcur_; + int64_t dest = base; + dfcur_ += rec.rsiz; + step++; + while (step-- > 0 && dfcur_ < end) { + rec.off = dfcur_; + if (!read_record(&rec, rbuf)) { + if (atran) abort_auto_transaction(); + return false; + } + escape_cursors(rec.off, dest); + dfcur_ += rec.rsiz; + if (rec.psiz != UINT16MAX) { + if (!rec.vbuf && !read_record_body(&rec)) { + if (atran) abort_auto_transaction(); + delete[] rec.bbuf; + return false; + } + if (rec.psiz >= align_) { + size_t diff = rec.psiz - rec.psiz % align_; + rec.psiz -= diff; + rec.rsiz -= diff; + } + if (!shift_record(&rec, dest)) { + if (atran) abort_auto_transaction(); + delete[] rec.bbuf; + return false; + } + delete[] rec.bbuf; + dest += rec.rsiz; + } + } + trim_free_blocks(base, dfcur_); + if (dfcur_ >= end) { + lsiz_ = dest; + psiz_ = lsiz_; + if (!file_.truncate(lsiz_)) { + if (atran) abort_auto_transaction(); + return false; + } + trim_cursors(); + dfcur_ = roff_; + } else { + size_t rsiz = dfcur_ - dest; + if (!write_free_block(dest, rsiz, rbuf)) { + if (atran) abort_auto_transaction(); + return false; + } + insert_free_block(dest, rsiz); + dfcur_ = dest; + } + if (atran) { + if (!commit_auto_transaction()) return false; + } else if (autosync_) { + if (!synchronize_meta()) return false; + } + return true; + } + /** + * Calculate meta data with saved ones. + */ + void calc_meta() { + _assert_(true); + align_ = 1 << apow_; + fbpnum_ = fpow_ > 0 ? 1 << fpow_ : 0; + width_ = (opts_ & TSMALL) ? sizeof(uint32_t) : sizeof(uint32_t) + 2; + linear_ = (opts_ & TLINEAR) ? true : false; + comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL; + rhsiz_ = sizeof(uint16_t) + sizeof(uint8_t) * 2; + rhsiz_ += linear_ ? width_ : width_ * 2; + boff_ = HEADSIZ + FBPWIDTH * fbpnum_; + if (fbpnum_ > 0) boff_ += width_ * 2 + sizeof(uint8_t) * 2; + roff_ = boff_ + width_ * bnum_; + int64_t rem = roff_ % align_; + if (rem > 0) roff_ += align_ - rem; + dfcur_ = roff_; + frgcnt_ = 0; + tran_ = false; + } + /** + * Calculate the module checksum. + * @return the module checksum. + */ + uint8_t calc_checksum() { + _assert_(true); + const char* kbuf = KCHDBCHKSUMSEED; + size_t ksiz = sizeof(KCHDBCHKSUMSEED) - 1; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp_) { + zbuf = comp_->compress(kbuf, ksiz, &zsiz); + if (!zbuf) return 0; + kbuf = zbuf; + ksiz = zsiz; + } + uint32_t hash = fold_hash(hash_record(kbuf, ksiz)); + delete[] zbuf; + return (hash >> 24) ^ (hash >> 16) ^ (hash >> 8) ^ (hash >> 0); + } + /** + * Dump the meta data into the file. + * @return true on success, or false on failure. + */ + bool dump_meta() { + _assert_(true); + char head[HEADSIZ]; + std::memset(head, 0, sizeof(head)); + std::memcpy(head, KCHDBMAGICDATA, sizeof(KCHDBMAGICDATA)); + std::memcpy(head + MOFFLIBVER, &libver_, sizeof(libver_)); + std::memcpy(head + MOFFLIBREV, &librev_, sizeof(librev_)); + std::memcpy(head + MOFFFMTVER, &fmtver_, sizeof(fmtver_)); + std::memcpy(head + MOFFCHKSUM, &chksum_, sizeof(chksum_)); + std::memcpy(head + MOFFTYPE, &type_, sizeof(type_)); + std::memcpy(head + MOFFAPOW, &apow_, sizeof(apow_)); + std::memcpy(head + MOFFFPOW, &fpow_, sizeof(fpow_)); + std::memcpy(head + MOFFOPTS, &opts_, sizeof(opts_)); + uint64_t num = hton64(bnum_); + std::memcpy(head + MOFFBNUM, &num, sizeof(num)); + if (!flagopen_) flags_ &= ~FOPEN; + std::memcpy(head + MOFFFLAGS, &flags_, sizeof(flags_)); + num = hton64(count_); + std::memcpy(head + MOFFCOUNT, &num, sizeof(num)); + num = hton64(lsiz_); + std::memcpy(head + MOFFSIZE, &num, sizeof(num)); + std::memcpy(head + MOFFOPAQUE, opaque_, sizeof(opaque_)); + if (!file_.write(0, head, sizeof(head))) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + trcount_ = count_; + trsize_ = lsiz_; + return true; + } + /** + * Dump the meta data into the file. + * @return true on success, or false on failure. + */ + bool dump_auto_meta() { + _assert_(true); + const int64_t hsiz = MOFFOPAQUE - MOFFCOUNT; + char head[hsiz]; + std::memset(head, 0, hsiz); + uint64_t num = hton64(count_); + std::memcpy(head, &num, sizeof(num)); + num = hton64(lsiz_); + std::memcpy(head + MOFFSIZE - MOFFCOUNT, &num, sizeof(num)); + if (!file_.write_fast(MOFFCOUNT, head, sizeof(head))) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + trcount_ = count_; + trsize_ = lsiz_; + return true; + } + /** + * Dump the opaque data into the file. + * @return true on success, or false on failure. + */ + bool dump_opaque() { + _assert_(true); + if (!file_.write_fast(MOFFOPAQUE, opaque_, sizeof(opaque_))) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + return true; + } + /** + * Load the meta data from the file. + * @return true on success, or false on failure. + */ + bool load_meta() { + _assert_(true); + char head[HEADSIZ]; + if (file_.size() < (int64_t)sizeof(head)) { + set_error(_KCCODELINE_, Error::INVALID, "missing magic data of the file"); + return false; + } + if (!file_.read(0, head, sizeof(head))) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)0, (long long)file_.size()); + return false; + } + if (std::memcmp(head, KCHDBMAGICDATA, sizeof(KCHDBMAGICDATA))) { + set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of the file"); + return false; + } + std::memcpy(&libver_, head + MOFFLIBVER, sizeof(libver_)); + std::memcpy(&librev_, head + MOFFLIBREV, sizeof(librev_)); + std::memcpy(&fmtver_, head + MOFFFMTVER, sizeof(fmtver_)); + std::memcpy(&chksum_, head + MOFFCHKSUM, sizeof(chksum_)); + std::memcpy(&type_, head + MOFFTYPE, sizeof(type_)); + std::memcpy(&apow_, head + MOFFAPOW, sizeof(apow_)); + std::memcpy(&fpow_, head + MOFFFPOW, sizeof(fpow_)); + std::memcpy(&opts_, head + MOFFOPTS, sizeof(opts_)); + uint64_t num; + std::memcpy(&num, head + MOFFBNUM, sizeof(num)); + bnum_ = ntoh64(num); + std::memcpy(&flags_, head + MOFFFLAGS, sizeof(flags_)); + flagopen_ = flags_ & FOPEN; + std::memcpy(&num, head + MOFFCOUNT, sizeof(num)); + count_ = ntoh64(num); + std::memcpy(&num, head + MOFFSIZE, sizeof(num)); + lsiz_ = ntoh64(num); + psiz_ = lsiz_; + std::memcpy(opaque_, head + MOFFOPAQUE, sizeof(opaque_)); + trcount_ = count_; + trsize_ = lsiz_; + return true; + } + /** + * Set a status flag. + * @param flag the flag kind. + * @param sign whether to set or unset. + * @return true on success, or false on failure. + */ + bool set_flag(uint8_t flag, bool sign) { + _assert_(true); + uint8_t flags; + if (!file_.read(MOFFFLAGS, &flags, sizeof(flags))) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)MOFFFLAGS, (long long)file_.size()); + return false; + } + if (sign) { + flags |= flag; + } else { + flags &= ~flag; + } + if (!file_.write(MOFFFLAGS, &flags, sizeof(flags))) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + flags_ = flags; + return true; + } + /** + * Reorganize the whole file. + * @param path the path of the database file. + * @return true on success, or false on failure. + */ + bool reorganize_file(const std::string& path) { + _assert_(true); + bool err = false; + HashDB db; + db.tune_type(type_); + db.tune_alignment(apow_); + db.tune_fbp(fpow_); + db.tune_options(opts_); + db.tune_buckets(bnum_); + db.tune_map(msiz_); + if (embcomp_) db.tune_compressor(embcomp_); + const std::string& npath = path + File::EXTCHR + KCHDBTMPPATHEXT; + if (db.open(npath, OWRITER | OCREATE | OTRUNCATE)) { + report(_KCCODELINE_, Logger::WARN, "reorganizing the database"); + lsiz_ = file_.size(); + psiz_ = lsiz_; + if (copy_records(&db)) { + if (db.close()) { + if (!File::rename(npath, path)) { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destination failed"); + err = true; + } + } else { + set_error(_KCCODELINE_, db.error().code(), "closing the destination failed"); + err = true; + } + } else { + set_error(_KCCODELINE_, db.error().code(), "record copying failed"); + err = true; + } + File::remove(npath); + } else { + set_error(_KCCODELINE_, db.error().code(), "opening the destination failed"); + err = true; + } + return !err; + } + /** + * Copy all records to another database. + * @param dest the destination database. + * @return true on success, or false on failure. + */ + bool copy_records(HashDB* dest) { + _assert_(dest); + Logger* logger = logger_; + logger_ = NULL; + int64_t off = roff_; + int64_t end = psiz_; + Record rec, nrec; + char rbuf[RECBUFSIZ], nbuf[RECBUFSIZ]; + while (off > 0 && off < end) { + rec.off = off; + if (!read_record(&rec, rbuf)) { + int64_t checkend = off + SLVGWIDTH; + if (checkend > end - (int64_t)rhsiz_) checkend = end - rhsiz_; + bool hit = false; + for (off += rhsiz_; off < checkend; off++) { + rec.off = off; + if (!read_record(&rec, rbuf)) continue; + if ((int64_t)rec.rsiz > SLVGWIDTH || rec.off + (int64_t)rec.rsiz >= checkend) { + delete[] rec.bbuf; + continue; + } + if (rec.psiz != UINT16MAX && !rec.vbuf && !read_record_body(&rec)) { + delete[] rec.bbuf; + continue; + } + delete[] rec.bbuf; + nrec.off = off + rec.rsiz; + if (!read_record(&nrec, nbuf)) continue; + if ((int64_t)nrec.rsiz > SLVGWIDTH || nrec.off + (int64_t)nrec.rsiz >= checkend) { + delete[] nrec.bbuf; + continue; + } + if (nrec.psiz != UINT16MAX && !nrec.vbuf && !read_record_body(&nrec)) { + delete[] nrec.bbuf; + continue; + } + delete[] nrec.bbuf; + hit = true; + break; + } + if (!hit || !read_record(&rec, rbuf)) break; + } + if (rec.psiz == UINT16MAX) { + off += rec.rsiz; + continue; + } + if (!rec.vbuf && !read_record_body(&rec)) { + delete[] rec.bbuf; + bool hit = false; + if (rec.rsiz <= MEMMAXSIZ && off + (int64_t)rec.rsiz < end) { + nrec.off = off + rec.rsiz; + if (read_record(&nrec, nbuf)) { + if (nrec.rsiz > MEMMAXSIZ || nrec.off + (int64_t)nrec.rsiz >= end) { + delete[] nrec.bbuf; + } else if (nrec.psiz != UINT16MAX && !nrec.vbuf && !read_record_body(&nrec)) { + delete[] nrec.bbuf; + } else { + delete[] nrec.bbuf; + hit = true; + } + } + } + if (hit) { + off += rec.rsiz; + continue; + } else { + break; + } + } + const char* vbuf = rec.vbuf; + size_t vsiz = rec.vsiz; + char* zbuf = NULL; + size_t zsiz = 0; + if (comp_) { + zbuf = comp_->decompress(vbuf, vsiz, &zsiz); + if (!zbuf) { + delete[] rec.bbuf; + off += rec.rsiz; + continue; + } + vbuf = zbuf; + vsiz = zsiz; + } + if (!dest->set(rec.kbuf, rec.ksiz, vbuf, vsiz)) { + delete[] zbuf; + delete[] rec.bbuf; + break; + } + delete[] zbuf; + delete[] rec.bbuf; + off += rec.rsiz; + } + logger_ = logger; + return true; + } + /** + * Trim the file size. + * @param path the path of the database file. + * @return true on success, or false on failure. + */ + bool trim_file(const std::string& path) { + _assert_(true); + bool err = false; + report(_KCCODELINE_, Logger::WARN, "trimming the database"); + File* dest = writer_ ? &file_ : new File(); + if (dest == &file_ || dest->open(path, File::OWRITER | File::ONOLOCK, 0)) { + if (!dest->truncate(lsiz_)) { + set_error(_KCCODELINE_, Error::SYSTEM, dest->error()); + err = true; + } + if (dest != &file_) { + if (!dest->close()) { + set_error(_KCCODELINE_, Error::SYSTEM, dest->error()); + err = true; + } + if (!file_.refresh()) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + } + trim_ = true; + } else { + set_error(_KCCODELINE_, Error::SYSTEM, dest->error()); + err = true; + } + if (dest != &file_) delete dest; + return !err; + } + /** + * Get the hash value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the hash value. + */ + uint64_t hash_record(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + return hashmurmur(kbuf, ksiz); + } + /** + * Fold a hash value into a small number. + * @param hash the hash number. + * @return the result number. + */ + uint32_t fold_hash(uint64_t hash) { + _assert_(true); + return (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff00000000ULL) >> 16)) ^ + (((hash & 0x000000000000ffffULL) << 16) | ((hash & 0x00000000ffff0000ULL) >> 16)); + } + /** + * Compare two keys in lexical order. + * @param abuf one key. + * @param asiz the size of the one key. + * @param bbuf the other key. + * @param bsiz the size of the other key. + * @return positive if the former is big, or negative if the latter is big, or 0 if both are + * equivalent. + */ + int32_t compare_keys(const char* abuf, size_t asiz, const char* bbuf, size_t bsiz) { + _assert_(abuf && bbuf); + if (asiz != bsiz) return (int32_t)asiz - (int32_t)bsiz; + return std::memcmp(abuf, bbuf, asiz); + } + /** + * Set an address into a bucket. + * @param bidx the index of the bucket. + * @param off the address. + * @return true on success, or false on failure. + */ + bool set_bucket(int64_t bidx, int64_t off) { + _assert_(bidx >= 0 && off >= 0); + char buf[sizeof(uint64_t)]; + writefixnum(buf, off >> apow_, width_); + if (!file_.write_fast(boff_ + bidx * width_, buf, width_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + return true; + } + /** + * Get an address from a bucket. + * @param bidx the index of the bucket. + * @return the address, or -1 on failure. + */ + int64_t get_bucket(int64_t bidx) { + _assert_(bidx >= 0); + char buf[sizeof(uint64_t)]; + if (!file_.read_fast(boff_ + bidx * width_, buf, width_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)boff_ + bidx * width_, (long long)file_.size()); + return -1; + } + return readfixnum(buf, width_) << apow_; + } + /** + * Set an address into a chain slot. + * @param entoff the address of the chain slot. + * @param off the destination address. + * @return true on success, or false on failure. + */ + bool set_chain(int64_t entoff, int64_t off) { + _assert_(entoff >= 0 && off >= 0); + char buf[sizeof(uint64_t)]; + writefixnum(buf, off >> apow_, width_); + if (!file_.write_fast(entoff, buf, width_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + return true; + } + /** + * Read a record from the file. + * @param rec the record structure. + * @param rbuf the working buffer. + * @return true on success, or false on failure. + */ + bool read_record(Record* rec, char* rbuf) { + _assert_(rec && rbuf); + if (rec->off < roff_) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid record offset"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->off, (long long)file_.size()); + return false; + } + size_t rsiz = psiz_ - rec->off; + if (rsiz > RECBUFSIZ) { + rsiz = RECBUFSIZ; + } else { + if (rsiz < rhsiz_) { + set_error(_KCCODELINE_, Error::BROKEN, "too short record region"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size()); + return false; + } + rsiz = rhsiz_; + } + if (!file_.read_fast(rec->off, rbuf, rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size()); + return false; + } + const char* rp = rbuf; + uint16_t snum; + if (*(uint8_t*)rp == RECMAGIC) { + ((uint8_t*)&snum)[0] = 0; + ((uint8_t*)&snum)[1] = *(uint8_t*)(rp + 1); + } else if (*(uint8_t*)rp >= 0x80) { + if (*(uint8_t*)(rp++) != FBMAGIC || *(uint8_t*)(rp++) != FBMAGIC) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a free block"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size()); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + return false; + } + rec->rsiz = readfixnum(rp, width_) << apow_; + rp += width_; + if (*(uint8_t*)(rp++) != PADMAGIC || *(uint8_t*)(rp++) != PADMAGIC) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a free block"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size()); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + return false; + } + if (rec->rsiz < rhsiz_) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid size of a free block"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size()); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + return false; + } + rec->psiz = UINT16MAX; + rec->ksiz = 0; + rec->vsiz = 0; + rec->left = 0; + rec->right = 0; + rec->kbuf = NULL; + rec->vbuf = NULL; + rec->boff = 0; + rec->bbuf = NULL; + return true; + } else if (*rp == 0) { + set_error(_KCCODELINE_, Error::BROKEN, "nullified region"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size()); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + return false; + } else { + std::memcpy(&snum, rp, sizeof(snum)); + } + rp += sizeof(snum); + rsiz -= sizeof(snum); + rec->psiz = ntoh16(snum); + rec->left = readfixnum(rp, width_) << apow_; + rp += width_; + rsiz -= width_; + if (linear_) { + rec->right = 0; + } else { + rec->right = readfixnum(rp, width_) << apow_; + rp += width_; + rsiz -= width_; + } + uint64_t num; + size_t step = readvarnum(rp, rsiz, &num); + if (step < 1) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid key length"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld snum=%04X", + (long long)psiz_, (long long)rec->off, (long long)rsiz, + (long long)file_.size(), snum); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + return false; + } + rec->ksiz = num; + rp += step; + rsiz -= step; + step = readvarnum(rp, rsiz, &num); + if (step < 1) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid value length"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld snum=%04X", + (long long)psiz_, (long long)rec->off, (long long)rsiz, + (long long)file_.size(), snum); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + return false; + } + rec->vsiz = num; + rp += step; + rsiz -= step; + size_t hsiz = rp - rbuf; + rec->rsiz = hsiz + rec->ksiz + rec->vsiz + rec->psiz; + rec->kbuf = NULL; + rec->vbuf = NULL; + rec->boff = rec->off + hsiz; + rec->bbuf = NULL; + if (rsiz >= rec->ksiz) { + rec->kbuf = rp; + rp += rec->ksiz; + rsiz -= rec->ksiz; + if (rsiz >= rec->vsiz) { + rec->vbuf = rp; + if (rec->psiz > 0) { + rp += rec->vsiz; + rsiz -= rec->vsiz; + if (rsiz > 0 && *(uint8_t*)rp != PADMAGIC) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a record"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld" + " snum=%04X", (long long)psiz_, (long long)rec->off, (long long)rsiz, + (long long)file_.size(), snum); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); + return false; + } + } + } + } else { + if (rec->off + (int64_t)rec->rsiz > psiz_) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid length of a record"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld" + " snum=%04X", (long long)psiz_, (long long)rec->off, (long long)rec->rsiz, + (long long)file_.size(), snum); + return false; + } + if (!read_record_body(rec)) return false; + } + return true; + } + /** + * Read the body of a record from the file. + * @param rec the record structure. + * @return true on success, or false on failure. + */ + bool read_record_body(Record* rec) { + _assert_(rec); + size_t bsiz = rec->ksiz + rec->vsiz; + if (rec->psiz > 0) bsiz++; + char* bbuf = new char[bsiz]; + if (!file_.read_fast(rec->boff, bbuf, bsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)rec->boff, (long long)file_.size()); + delete[] bbuf; + return false; + } + if (rec->psiz > 0 && ((uint8_t*)bbuf)[bsiz-1] != PADMAGIC) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a record"); + report_binary(_KCCODELINE_, Logger::WARN, "bbuf", bbuf, bsiz); + delete[] bbuf; + return false; + } + rec->bbuf = bbuf; + rec->kbuf = rec->bbuf; + rec->vbuf = rec->bbuf + rec->ksiz; + return true; + } + /** + * Write a record into the file. + * @param rec the record structure. + * @param over true for overwriting, or false for new record. + * @return true on success, or false on failure. + */ + bool write_record(Record* rec, bool over) { + _assert_(rec); + char stack[IOBUFSIZ]; + char* rbuf = rec->rsiz > sizeof(stack) ? new char[rec->rsiz] : stack; + char* wp = rbuf; + uint16_t snum = hton16(rec->psiz); + std::memcpy(wp, &snum, sizeof(snum)); + if (rec->psiz < 0x100) *wp = RECMAGIC; + wp += sizeof(snum); + writefixnum(wp, rec->left >> apow_, width_); + wp += width_; + if (!linear_) { + writefixnum(wp, rec->right >> apow_, width_); + wp += width_; + } + wp += writevarnum(wp, rec->ksiz); + wp += writevarnum(wp, rec->vsiz); + std::memcpy(wp, rec->kbuf, rec->ksiz); + wp += rec->ksiz; + std::memcpy(wp, rec->vbuf, rec->vsiz); + wp += rec->vsiz; + if (rec->psiz > 0) { + std::memset(wp, 0, rec->psiz); + *wp = PADMAGIC; + wp += rec->psiz; + } + bool err = false; + if (over) { + if (!file_.write_fast(rec->off, rbuf, rec->rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + } else { + if (!file_.write(rec->off, rbuf, rec->rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + } + if (rbuf != stack) delete[] rbuf; + return !err; + } + /** + * Adjust the padding of a record. + * @param rec the record structure. + * @return true on success, or false on failure. + */ + bool adjust_record(Record* rec) { + _assert_(rec); + if (rec->psiz > (size_t)INT16MAX || rec->psiz > rec->rsiz / 2) { + size_t nsiz = (rec->psiz >> apow_) << apow_; + if (nsiz < rhsiz_) return true; + rec->rsiz -= nsiz; + rec->psiz -= nsiz; + int64_t noff = rec->off + rec->rsiz; + char nbuf[RECBUFSIZ]; + if (!write_free_block(noff, nsiz, nbuf)) return false; + insert_free_block(noff, nsiz); + } + return true; + } + /** + * Calculate the size of a record. + * @param ksiz the size of the key. + * @param vsiz the size of the value. + * @return the size of the record. + */ + size_t calc_record_size(size_t ksiz, size_t vsiz) { + _assert_(true); + size_t rsiz = sizeof(uint16_t) + width_; + if (!linear_) rsiz += width_; + if (ksiz < (1ULL << 7)) { + rsiz += 1; + } else if (ksiz < (1ULL << 14)) { + rsiz += 2; + } else if (ksiz < (1ULL << 21)) { + rsiz += 3; + } else if (ksiz < (1ULL << 28)) { + rsiz += 4; + } else { + rsiz += 5; + } + if (vsiz < (1ULL << 7)) { + rsiz += 1; + } else if (vsiz < (1ULL << 14)) { + rsiz += 2; + } else if (vsiz < (1ULL << 21)) { + rsiz += 3; + } else if (vsiz < (1ULL << 28)) { + rsiz += 4; + } else { + rsiz += 5; + } + rsiz += ksiz; + rsiz += vsiz; + return rsiz; + } + /** + * Calculate the padding size of a record. + * @param rsiz the size of the record. + * @return the size of the padding. + */ + size_t calc_record_padding(size_t rsiz) { + _assert_(true); + size_t diff = rsiz & (align_ - 1); + return diff > 0 ? align_ - diff : 0; + } + /** + * Shift a record to another place. + * @param orec the original record structure. + * @param dest the destination offset. + * @return true on success, or false on failure. + */ + bool shift_record(Record* orec, int64_t dest) { + _assert_(orec && dest >= 0); + uint64_t hash = hash_record(orec->kbuf, orec->ksiz); + uint32_t pivot = fold_hash(hash); + int64_t bidx = hash % bnum_; + int64_t off = get_bucket(bidx); + if (off < 0) return false; + if (off == orec->off) { + orec->off = dest; + if (!write_record(orec, true)) return false; + if (!set_bucket(bidx, dest)) return false; + return true; + } + int64_t entoff = 0; + Record rec; + char rbuf[RECBUFSIZ]; + while (off > 0) { + rec.off = off; + if (!read_record(&rec, rbuf)) return false; + if (rec.psiz == UINT16MAX) { + set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)rec.off, (long long)file_.size()); + return false; + } + uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, rec.ksiz)); + if (pivot > tpivot) { + delete[] rec.bbuf; + off = rec.left; + entoff = rec.off + sizeof(uint16_t); + } else if (pivot < tpivot) { + delete[] rec.bbuf; + off = rec.right; + entoff = rec.off + sizeof(uint16_t) + width_; + } else { + int32_t kcmp = compare_keys(orec->kbuf, orec->ksiz, rec.kbuf, rec.ksiz); + if (linear_ && kcmp != 0) kcmp = 1; + if (kcmp > 0) { + delete[] rec.bbuf; + off = rec.left; + entoff = rec.off + sizeof(uint16_t); + } else if (kcmp < 0) { + delete[] rec.bbuf; + off = rec.right; + entoff = rec.off + sizeof(uint16_t) + width_; + } else { + delete[] rec.bbuf; + orec->off = dest; + if (!write_record(orec, true)) return false; + if (entoff > 0) { + if (!set_chain(entoff, dest)) return false; + } else { + if (!set_bucket(bidx, dest)) return false; + } + return true; + } + } + } + set_error(_KCCODELINE_, Error::BROKEN, "no record to shift"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld fsiz=%lld", + (long long)psiz_, (long long)file_.size()); + return false; + } + /** + * Write a free block into the file. + * @param off the offset of the free block. + * @param rsiz the size of the free block. + * @param rbuf the working buffer. + * @return true on success, or false on failure. + */ + bool write_free_block(int64_t off, size_t rsiz, char* rbuf) { + _assert_(off >= 0 && rbuf); + char* wp = rbuf; + *(wp++) = FBMAGIC; + *(wp++) = FBMAGIC; + writefixnum(wp, rsiz >> apow_, width_); + wp += width_; + *(wp++) = PADMAGIC; + *(wp++) = PADMAGIC; + if (!file_.write_fast(off, rbuf, wp - rbuf)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + return true; + } + /** + * Insert a free block to the free block pool. + * @param off the offset of the free block. + * @param rsiz the size of the free block. + */ + void insert_free_block(int64_t off, size_t rsiz) { + _assert_(off >= 0); + ScopedMutex lock(&flock_); + escape_cursors(off, off + rsiz); + if (fbpnum_ < 1) return; + if (fbp_.size() >= (size_t)fbpnum_) { + FBP::const_iterator it = fbp_.begin(); + if (rsiz <= it->rsiz) return; + fbp_.erase(it); + } + FreeBlock fb = { off, rsiz }; + fbp_.insert(fb); + } + /** + * Fetch the free block pool from a decent sized block. + * @param rsiz the minimum size of the block. + * @param res the structure for the result. + * @return true on success, or false on failure. + */ + bool fetch_free_block(size_t rsiz, FreeBlock* res) { + _assert_(res); + if (fbpnum_ < 1) return false; + ScopedMutex lock(&flock_); + FreeBlock fb = { INT64MAX, rsiz }; + FBP::const_iterator it = fbp_.upper_bound(fb); + if (it == fbp_.end()) return false; + res->off = it->off; + res->rsiz = it->rsiz; + fbp_.erase(it); + escape_cursors(res->off, res->off + res->rsiz); + return true; + } + /** + * Trim invalid free blocks. + * @param begin the beginning offset. + * @param end the end offset. + */ + void trim_free_blocks(int64_t begin, int64_t end) { + _assert_(begin >= 0 && end >= 0); + FBP::const_iterator it = fbp_.begin(); + FBP::const_iterator itend = fbp_.end(); + while (it != itend) { + if (it->off >= begin && it->off < end) { + fbp_.erase(it++); + } else { + ++it; + } + } + } + /** + * Dump all free blocks into the file. + * @return true on success, or false on failure. + */ + bool dump_free_blocks() { + _assert_(true); + if (fbpnum_ < 1) return true; + size_t size = boff_ - HEADSIZ; + char* rbuf = new char[size]; + char* wp = rbuf; + char* end = rbuf + size - width_ * 2 - sizeof(uint8_t) * 2; + size_t num = fbp_.size(); + if (num > 0) { + FreeBlock* blocks = new FreeBlock[num]; + size_t cnt = 0; + FBP::const_iterator it = fbp_.begin(); + FBP::const_iterator itend = fbp_.end(); + while (it != itend) { + blocks[cnt++] = *it; + ++it; + } + std::sort(blocks, blocks + num, FreeBlockComparator()); + for (size_t i = num - 1; i > 0; i--) { + blocks[i].off -= blocks[i-1].off; + } + for (size_t i = 0; wp < end && i < num; i++) { + wp += writevarnum(wp, blocks[i].off >> apow_); + wp += writevarnum(wp, blocks[i].rsiz >> apow_); + } + delete[] blocks; + } + *(wp++) = 0; + *(wp++) = 0; + bool err = false; + if (!file_.write(HEADSIZ, rbuf, wp - rbuf)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + delete[] rbuf; + return !err; + } + /** + * Dump an empty set of free blocks into the file. + * @return true on success, or false on failure. + */ + bool dump_empty_free_blocks() { + _assert_(true); + if (fbpnum_ < 1) return true; + char rbuf[2]; + char* wp = rbuf; + *(wp++) = 0; + *(wp++) = 0; + bool err = false; + if (!file_.write(HEADSIZ, rbuf, wp - rbuf)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + return !err; + } + /** + * Load all free blocks from from the file. + * @return true on success, or false on failure. + */ + bool load_free_blocks() { + _assert_(true); + if (fbpnum_ < 1) return true; + size_t size = boff_ - HEADSIZ; + char* rbuf = new char[size]; + if (!file_.read(HEADSIZ, rbuf, size)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)HEADSIZ, (long long)file_.size()); + delete[] rbuf; + return false; + } + const char* rp = rbuf; + FreeBlock* blocks = new FreeBlock[fbpnum_]; + int32_t num = 0; + while (num < fbpnum_ && size > 1 && *rp != '\0') { + uint64_t off; + size_t step = readvarnum(rp, size, &off); + if (step < 1 || off < 1) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid free block offset"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)off, (long long)file_.size()); + delete[] rbuf; + delete[] blocks; + return false; + } + rp += step; + size -= step; + uint64_t rsiz; + step = readvarnum(rp, size, &rsiz); + if (step < 1 || rsiz < 1) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid free block size"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz=%lld", + (long long)psiz_, (long long)off, (long long)rsiz, (long long)file_.size()); + delete[] rbuf; + delete[] blocks; + return false; + } + rp += step; + size -= step; + blocks[num].off = off << apow_; + blocks[num].rsiz = rsiz << apow_; + num++; + } + for (int32_t i = 1; i < num; i++) { + blocks[i].off += blocks[i-1].off; + } + for (int32_t i = 0; i < num; i++) { + FreeBlock fb = { blocks[i].off, blocks[i].rsiz }; + fbp_.insert(fb); + } + delete[] blocks; + delete[] rbuf; + return true; + } + /** + * Disable all cursors. + */ + void disable_cursors() { + _assert_(true); + if (curs_.empty()) return; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->off_ = 0; + ++cit; + } + } + /** + * Escape cursors on a free block. + * @param off the offset of the free block. + * @param dest the destination offset. + */ + void escape_cursors(int64_t off, int64_t dest) { + _assert_(off >= 0 && dest >= 0); + if (curs_.empty()) return; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->end_ == off) { + cur->end_ = dest; + if (cur->off_ >= cur->end_) cur->off_ = 0; + } + if (cur->off_ == off) { + cur->off_ = dest; + if (cur->off_ >= cur->end_) cur->off_ = 0; + } + ++cit; + } + } + /** + * Trim invalid cursors. + */ + void trim_cursors() { + _assert_(true); + if (curs_.empty()) return; + int64_t end = lsiz_; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->off_ >= end) { + cur->off_ = 0; + } else if (cur->end_ > end) { + cur->end_ = end; + } + ++cit; + } + } + /** + * Remove a record from a bucket chain. + * @param rec the record structure. + * @param rbuf the working buffer. + * @param bidx the bucket index. + * @param entoff the offset of the entry pointer. + * @return true on success, or false on failure. + */ + bool cut_chain(Record* rec, char* rbuf, int64_t bidx, int64_t entoff) { + _assert_(rec && rbuf && bidx >= 0 && entoff >= 0); + int64_t child; + if (rec->left > 0 && rec->right < 1) { + child = rec->left; + } else if (rec->left < 1 && rec->right > 0) { + child = rec->right; + } else if (rec->left < 1) { + child = 0; + } else { + Record prec; + prec.off = rec->left; + if (!read_record(&prec, rbuf)) return false; + if (prec.psiz == UINT16MAX) { + set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)prec.off, (long long)file_.size()); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rhsiz_); + return false; + } + delete[] prec.bbuf; + if (prec.right > 0) { + int64_t off = prec.right; + int64_t pentoff = prec.off + sizeof(uint16_t) + width_; + while (true) { + prec.off = off; + if (!read_record(&prec, rbuf)) return false; + if (prec.psiz == UINT16MAX) { + set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain"); + report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", + (long long)psiz_, (long long)prec.off, (long long)file_.size()); + report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rhsiz_); + return false; + } + delete[] prec.bbuf; + if (prec.right < 1) break; + off = prec.right; + pentoff = prec.off + sizeof(uint16_t) + width_; + } + child = off; + if (!set_chain(pentoff, prec.left)) return false; + if (!set_chain(off + sizeof(uint16_t), rec->left)) return false; + if (!set_chain(off + sizeof(uint16_t) + width_, rec->right)) return false; + } else { + child = prec.off; + if (!set_chain(prec.off + sizeof(uint16_t) + width_, rec->right)) return false; + } + } + if (entoff > 0) { + if (!set_chain(entoff, child)) return false; + } else { + if (!set_bucket(bidx, child)) return false; + } + return true; + } + /** + * Begin transaction. + * @return true on success, or false on failure. + */ + bool begin_transaction_impl() { + _assert_(true); + if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_meta()) return false; + if (!file_.begin_transaction(trhard_, boff_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + if (!file_.write_transaction(MOFFBNUM, HEADSIZ - MOFFBNUM)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + file_.end_transaction(false); + return false; + } + if (fbpnum_ > 0) { + FBP::const_iterator it = fbp_.end(); + FBP::const_iterator itbeg = fbp_.begin(); + for (int32_t cnt = fpow_ * 2 + 1; cnt > 0; cnt--) { + if (it == itbeg) break; + --it; + trfbp_.insert(*it); + } + } + return true; + } + /** + * Begin auto transaction. + * @return true on success, or false on failure. + */ + bool begin_auto_transaction() { + _assert_(true); + atlock_.lock(); + if (!file_.begin_transaction(autosync_, boff_)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + atlock_.unlock(); + return false; + } + if (!file_.write_transaction(MOFFCOUNT, MOFFOPAQUE - MOFFCOUNT)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + file_.end_transaction(false); + atlock_.unlock(); + return false; + } + return true; + } + /** + * Commit transaction. + * @return true on success, or false on failure. + */ + bool commit_transaction() { + _assert_(true); + bool err = false; + if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_auto_meta()) err = true; + if (!file_.end_transaction(true)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + trfbp_.clear(); + return !err; + } + /** + * Commit auto transaction. + * @return true on success, or false on failure. + */ + bool commit_auto_transaction() { + _assert_(true); + bool err = false; + if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_auto_meta()) err = true; + if (!file_.end_transaction(true)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + atlock_.unlock(); + return !err; + } + /** + * Abort transaction. + * @return true on success, or false on failure. + */ + bool abort_transaction() { + _assert_(true); + bool err = false; + if (!file_.end_transaction(false)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + bool flagopen = flagopen_; + if (!load_meta()) err = true; + flagopen_ = flagopen; + calc_meta(); + disable_cursors(); + fbp_.swap(trfbp_); + trfbp_.clear(); + return !err; + } + /** + * Abort auto transaction. + * @return true on success, or false on failure. + */ + bool abort_auto_transaction() { + _assert_(true); + bool err = false; + if (!file_.end_transaction(false)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + if (!load_meta()) err = true; + calc_meta(); + disable_cursors(); + fbp_.clear(); + atlock_.unlock(); + return !err; + } + /** Dummy constructor to forbid the use. */ + HashDB(const HashDB&); + /** Dummy Operator to forbid the use. */ + HashDB& operator =(const HashDB&); + /** The method lock. */ + RWLock mlock_; + /** The record locks. */ + SlottedRWLock rlock_; + /** The file lock. */ + Mutex flock_; + /** The auto transaction lock. */ + Mutex atlock_; + /** The last happened error. */ + TSD<Error> error_; + /** The internal logger. */ + Logger* logger_; + /** The kinds of logged messages. */ + uint32_t logkinds_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The open mode. */ + uint32_t omode_; + /** The flag for writer. */ + bool writer_; + /** The flag for auto transaction. */ + bool autotran_; + /** The flag for auto synchronization. */ + bool autosync_; + /** The flag for reorganized. */ + bool reorg_; + /** The flag for trimmed. */ + bool trim_; + /** The file for data. */ + File file_; + /** The free block pool. */ + FBP fbp_; + /** The cursor objects. */ + CursorList curs_; + /** The path of the database file. */ + std::string path_; + /** The library version. */ + uint8_t libver_; + /** The library revision. */ + uint8_t librev_; + /** The format revision. */ + uint8_t fmtver_; + /** The module checksum. */ + uint8_t chksum_; + /** The database type. */ + uint8_t type_; + /** The alignment power. */ + uint8_t apow_; + /** The free block pool power. */ + uint8_t fpow_; + /** The options. */ + uint8_t opts_; + /** The bucket number. */ + int64_t bnum_; + /** The status flags. */ + uint8_t flags_; + /** The flag for open. */ + bool flagopen_; + /** The record number. */ + AtomicInt64 count_; + /** The logical size of the file. */ + AtomicInt64 lsiz_; + /** The physical size of the file. */ + AtomicInt64 psiz_; + /** The opaque data. */ + char opaque_[HEADSIZ-MOFFOPAQUE]; + /** The size of the internal memory-mapped region. */ + int64_t msiz_; + /** The unit step number of auto defragmentation. */ + int64_t dfunit_; + /** The embedded data compressor. */ + Compressor* embcomp_; + /** The alignment of records. */ + size_t align_; + /** The number of elements of the free block pool. */ + int32_t fbpnum_; + /** The width of record addressing. */ + int32_t width_; + /** The flag for linear collision chaining. */ + bool linear_; + /** The data compressor. */ + Compressor* comp_; + /** The header size of a record. */ + size_t rhsiz_; + /** The offset of the buckets section. */ + int64_t boff_; + /** The offset of the record section. */ + int64_t roff_; + /** The defrag cursor. */ + int64_t dfcur_; + /** The count of fragmentation. */ + AtomicInt64 frgcnt_; + /** The flag whether in transaction. */ + bool tran_; + /** The flag whether hard transaction. */ + bool trhard_; + /** The escaped free block pool for transaction. */ + FBP trfbp_; + /** The count history for transaction. */ + int64_t trcount_; + /** The size history for transaction. */ + int64_t trsize_; +}; + + +/** An alias of the file tree database. */ +typedef PlantDB<HashDB, BasicDB::TYPETREE> TreeDB; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kchashmgr.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kchashmgr.cc new file mode 100644 index 0000000000..adaff1f84c --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kchashmgr.cc @@ -0,0 +1,1531 @@ +/************************************************************************************************* + * The command line utility of the file hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kchashdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, const char* info); +static int32_t runcreate(int argc, char** argv); +static int32_t runinform(int argc, char** argv); +static int32_t runset(int argc, char** argv); +static int32_t runremove(int argc, char** argv); +static int32_t runget(int argc, char** argv); +static int32_t runlist(int argc, char** argv); +static int32_t runclear(int argc, char** argv); +static int32_t runimport(int argc, char** argv); +static int32_t runcopy(int argc, char** argv); +static int32_t rundump(int argc, char** argv); +static int32_t runload(int argc, char** argv); +static int32_t rundefrag(int argc, char** argv); +static int32_t runcheck(int argc, char** argv); +static int32_t runsetbulk(int argc, char** argv); +static int32_t runremovebulk(int argc, char** argv); +static int32_t rungetbulk(int argc, char** argv); +static int32_t proccreate(const char* path, int32_t oflags, + int32_t apow, int32_t fpow, int32_t opts, int64_t bnum); +static int32_t procinform(const char* path, int32_t oflags, bool st); +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode); +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags); +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz); +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + int64_t max, bool rm, bool pv, bool px); +static int32_t procclear(const char* path, int32_t oflags); +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx); +static int32_t proccopy(const char* path, const char* file, int32_t oflags); +static int32_t procdump(const char* path, const char* file, int32_t oflags); +static int32_t procload(const char* path, const char* file, int32_t oflags); +static int32_t procdefrag(const char* path, int32_t oflags); +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs); +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys); +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px); +static int32_t proccheck(const char* path, int32_t oflags); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "create")) { + rv = runcreate(argc, argv); + } else if (!std::strcmp(argv[1], "inform")) { + rv = runinform(argc, argv); + } else if (!std::strcmp(argv[1], "set")) { + rv = runset(argc, argv); + } else if (!std::strcmp(argv[1], "remove")) { + rv = runremove(argc, argv); + } else if (!std::strcmp(argv[1], "get")) { + rv = runget(argc, argv); + } else if (!std::strcmp(argv[1], "list")) { + rv = runlist(argc, argv); + } else if (!std::strcmp(argv[1], "clear")) { + rv = runclear(argc, argv); + } else if (!std::strcmp(argv[1], "import")) { + rv = runimport(argc, argv); + } else if (!std::strcmp(argv[1], "copy")) { + rv = runcopy(argc, argv); + } else if (!std::strcmp(argv[1], "dump")) { + rv = rundump(argc, argv); + } else if (!std::strcmp(argv[1], "load")) { + rv = runload(argc, argv); + } else if (!std::strcmp(argv[1], "defrag")) { + rv = rundefrag(argc, argv); + } else if (!std::strcmp(argv[1], "setbulk")) { + rv = runsetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "removebulk")) { + rv = runremovebulk(argc, argv); + } else if (!std::strcmp(argv[1], "getbulk")) { + rv = rungetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "check")) { + rv = runcheck(argc, argv); + } else if (!std::strcmp(argv[1], "version") || !std::strcmp(argv[1], "--version")) { + printversion(); + } else { + usage(); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: the command line utility of the file hash database of Kyoto Cabinet\n", + g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s create [-otr] [-onl|-otl|-onr] [-apow num] [-fpow num] [-ts] [-tl] [-tc]" + " [-bnum num] path\n", g_progname); + eprintf(" %s inform [-onl|-otl|-onr] [-st] path\n", g_progname); + eprintf(" %s set [-onl|-otl|-onr] [-add|-rep|-app|-inci|-incd] [-sx] path key value\n", + g_progname); + eprintf(" %s remove [-onl|-otl|-onr] [-sx] path key\n", g_progname); + eprintf(" %s get [-onl|-otl|-onr] [-rm] [-sx] [-px] [-pz] path key\n", g_progname); + eprintf(" %s list [-onl|-otl|-onr] [-max num] [-rm] [-sx] [-pv] [-px] path [key]\n", + g_progname); + eprintf(" %s clear [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s import [-onl|-otl|-onr] [-sx] path [file]\n", g_progname); + eprintf(" %s copy [-onl|-otl|-onr] path file\n", g_progname); + eprintf(" %s dump [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s load [-otr] [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s defrag [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s setbulk [-onl|-otl|-onr] [-sx] path key value ...\n", g_progname); + eprintf(" %s removebulk [-onl|-otl|-onr] [-sx] path key ...\n", g_progname); + eprintf(" %s getbulk [-onl|-otl|-onr] [-sx] [-px] path key ...\n", g_progname); + eprintf(" %s check [-onl|-otl|-onr] path\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print error message of database +static void dberrprint(kc::BasicDB* db, const char* info) { + const kc::BasicDB::Error& err = db->error(); + eprintf("%s: %s: %s: %d: %s: %s\n", + g_progname, info, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// parse arguments of create command +static int32_t runcreate(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::HashDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::HashDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::HashDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::HashDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccreate(path, oflags, apow, fpow, opts, bnum); + return rv; +} + + +// parse arguments of inform command +static int32_t runinform(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + bool st = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-st")) { + st = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procinform(path, oflags, st); + return rv; +} + + +// parse arguments of set command +static int32_t runset(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + const char* vstr = NULL; + int32_t oflags = 0; + int32_t mode = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-add")) { + mode = 'a'; + } else if (!std::strcmp(argv[i], "-rep")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-app")) { + mode = 'c'; + } else if (!std::strcmp(argv[i], "-inci")) { + mode = 'i'; + } else if (!std::strcmp(argv[i], "-incd")) { + mode = 'd'; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else if (!vstr) { + vstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr || !vstr) usage(); + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + int32_t rv = procset(path, kstr, ksiz, vstr, vsiz, oflags, mode); + delete[] kbuf; + delete[] vbuf; + return rv; +} + + +// parse arguments of remove command +static int32_t runremove(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procremove(path, kstr, ksiz, oflags); + delete[] kbuf; + return rv; +} + + +// parse arguments of get command +static int32_t runget(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool rm = false; + bool sx = false; + bool px = false; + bool pz = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else if (!std::strcmp(argv[i], "-pz")) { + pz = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procget(path, kstr, ksiz, oflags, rm, px, pz); + delete[] kbuf; + return rv; +} + + +// parse arguments of list command +static int32_t runlist(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + int64_t max = -1; + bool rm = false; + bool sx = false; + bool pv = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-max")) { + if (++i >= argc) usage(); + max = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-pv")) { + pv = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + char* kbuf = NULL; + size_t ksiz = 0; + if (kstr) { + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = new char[ksiz+1]; + std::memcpy(kbuf, kstr, ksiz); + kbuf[ksiz] = '\0'; + } + } + int32_t rv = proclist(path, kbuf, ksiz, oflags, max, rm, pv, px); + delete[] kbuf; + return rv; +} + + +// parse arguments of clear command +static int32_t runclear(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procclear(path, oflags); + return rv; +} + + +// parse arguments of import command +static int32_t runimport(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procimport(path, file, oflags, sx); + return rv; +} + + +// parse arguments of copy command +static int32_t runcopy(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path || !file) usage(); + int32_t rv = proccopy(path, file, oflags); + return rv; +} + + +// parse arguments of dump command +static int32_t rundump(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procdump(path, file, oflags); + return rv; +} + + +// parse arguments of load command +static int32_t runload(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::HashDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procload(path, file, oflags); + return rv; +} + + +// parse arguments of defrag command +static int32_t rundefrag(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procdefrag(path, oflags); + return rv; +} + + +// parse arguments of setbulk command +static int32_t runsetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::map<std::string, std::string> recs; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + if (++i >= argc) usage(); + const char* vstr = argv[i]; + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + std::string key(kstr, ksiz); + std::string value(vstr, vsiz); + recs[key] = value; + delete[] kbuf; + delete[] vbuf; + } + } + if (!path) usage(); + int32_t rv = procsetbulk(path, oflags, recs); + return rv; +} + + +// parse arguments of removebulk command +static int32_t runremovebulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procremovebulk(path, oflags, keys); + return rv; +} + + +// parse arguments of getbulk command +static int32_t rungetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procgetbulk(path, oflags, keys, px); + return rv; +} + + +// parse arguments of check command +static int32_t runcheck(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccheck(path, oflags); + return rv; +} + + +// perform create command +static int32_t proccreate(const char* path, int32_t oflags, + int32_t apow, int32_t fpow, int32_t opts, int64_t bnum) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (!db.open(path, kc::HashDB::OWRITER | kc::HashDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform inform command +static int32_t procinform(const char* path, int32_t oflags, bool st) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (st) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["fbpnum_used"] = ""; + status["bnum_used"] = ""; + if (db.status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t rtype = kc::atoi(status["realtype"].c_str()); + if (rtype > 0 && rtype != type) + oprintf("real type: %s (%s) (realtype=0x%02X)\n", + kc::BasicDB::typecname(rtype), kc::BasicDB::typestring(rtype), rtype); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::HashDB::FOPEN) oprintf(" open"); + if (flags & kc::HashDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + if (kc::atoi(status["trimmed"].c_str()) > 0) oprintf(" (trimmed)"); + oprintf("\n", flags); + int32_t apow = kc::atoi(status["apow"].c_str()); + oprintf("alignment: %d (apow=%d)\n", 1 << apow, apow); + int32_t fpow = kc::atoi(status["fpow"].c_str()); + int32_t fbpnum = fpow > 0 ? 1 << fpow : 0; + int32_t fbpused = kc::atoi(status["fbpnum_used"].c_str()); + int64_t frgcnt = kc::atoi(status["frgcnt"].c_str()); + oprintf("free block pool: %d (fpow=%d) (used=%d) (frg=%lld)\n", + fbpnum, fpow, fbpused, (long long)frgcnt); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::HashDB::TSMALL) oprintf(" small"); + if (opts & kc::HashDB::TLINEAR) oprintf(" linear"); + if (opts & kc::HashDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t bnumused = kc::atoi(status["bnum_used"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + double load = 0; + if (count > 0 && bnumused > 0) { + load = (double)count / bnumused; + if (!(opts & kc::HashDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (used=%lld) (load=%.2f)\n", + (long long)bnum, (long long)bnumused, load); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + int64_t msiz = kc::atoi(status["msiz"].c_str()); + int64_t realsize = kc::atoi(status["realsize"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s) (map=%lld)", size, sizestr.c_str(), (long long)msiz); + if (size != realsize) oprintf(" (gap=%lld)", (long long)(realsize - size)); + oprintf("\n"); + } else { + dberrprint(&db, "DB::status failed"); + err = true; + } + } else { + uint8_t flags = db.flags(); + if (flags != 0) { + oprintf("status:"); + if (flags & kc::HashDB::FOPEN) oprintf(" open"); + if (flags & kc::HashDB::FFATAL) oprintf(" fatal"); + oprintf("\n"); + } + oprintf("count: %lld\n", (long long)db.count()); + oprintf("size: %lld\n", (long long)db.size()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform set command +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + switch (mode) { + default: { + if (!db.set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 'a': { + if (!db.add(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::add failed"); + err = true; + } + break; + } + case 'r': { + if (!db.replace(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::replace failed"); + err = true; + } + break; + } + case 'c': { + if (!db.append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::append failed"); + err = true; + } + break; + } + case 'i': { + int64_t onum = db.increment(kbuf, ksiz, kc::atoi(vbuf)); + if (onum == kc::INT64MIN) { + dberrprint(&db, "DB::increment failed"); + err = true; + } else { + oprintf("%lld\n", (long long)onum); + } + break; + } + case 'd': { + double onum = db.increment_double(kbuf, ksiz, kc::atof(vbuf)); + if (kc::chknan(onum)) { + dberrprint(&db, "DB::increment_double failed"); + err = true; + } else { + oprintf("%f\n", onum); + } + break; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform remove command +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.remove(kbuf, ksiz)) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform get command +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::HashDB::OWRITER : kc::HashDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + char* vbuf; + size_t vsiz; + if (rm) { + vbuf = db.seize(kbuf, ksiz, &vsiz); + } else { + vbuf = db.get(kbuf, ksiz, &vsiz); + } + if (vbuf) { + printdata(vbuf, vsiz, px); + if (!pz) oprintf("\n"); + delete[] vbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform list command +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + int64_t max, bool rm, bool pv, bool px) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::HashDB::OWRITER : kc::HashDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(bool rm, bool pv, bool px) : rm_(rm), pv_(pv), px_(px) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + printdata(kbuf, ksiz, px_); + if (pv_) { + oprintf("\t"); + printdata(vbuf, vsiz, px_); + } + oprintf("\n"); + return rm_ ? REMOVE : NOP; + } + bool rm_; + bool pv_; + bool px_; + } visitor(rm, pv, px); + if (kbuf || max >= 0) { + if (max < 0) max = kc::INT64MAX; + kc::HashDB::Cursor cur(&db); + if (kbuf) { + if (!cur.jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } else { + if (!db.iterate(&visitor, rm)) { + dberrprint(&db, "DB::iterate failed"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform clear command +static int32_t procclear(const char* path, int32_t oflags) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.clear()) { + dberrprint(&db, "DB::clear failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform import command +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx) { + std::istream *is = &std::cin; + std::ifstream ifs; + if (file) { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | kc::HashDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + int64_t cnt = 0; + std::string line; + std::vector<std::string> fields; + while (!err && mygetline(is, &line)) { + cnt++; + kc::strsplit(line, '\t', &fields); + if (sx) { + std::vector<std::string>::iterator it = fields.begin(); + std::vector<std::string>::iterator itend = fields.end(); + while (it != itend) { + size_t esiz; + char* ebuf = kc::hexdecode(it->c_str(), &esiz); + it->clear(); + it->append(ebuf, esiz); + delete[] ebuf; + ++it; + } + } + switch (fields.size()) { + case 2: { + if (!db.set(fields[0], fields[1])) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 1: { + if (!db.remove(fields[0]) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + break; + } + } + oputchar('.'); + if (cnt % 50 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + if (cnt % 50 > 0) oprintf(" (%lld)\n", (long long)cnt); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform copy command +static int32_t proccopy(const char* path, const char* file, int32_t oflags) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + DotChecker checker(&std::cout, -100); + if (!db.copy(file, &checker)) { + dberrprint(&db, "DB::copy failed"); + err = true; + } + oprintf(" (end)\n"); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld blocks were copied successfully\n", (long long)checker.count()); + return err ? 1 : 0; +} + + +// perform dump command +static int32_t procdump(const char* path, const char* file, int32_t oflags) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, 1000); + if (!db.dump_snapshot(file)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were dumped successfully\n", (long long)checker.count()); + } else { + if (!db.dump_snapshot(&std::cout)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform load command +static int32_t procload(const char* path, const char* file, int32_t oflags) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | kc::HashDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(file)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } else { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(&std::cin)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform defrag command +static int32_t procdefrag(const char* path, int32_t oflags) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.defrag(0)) { + dberrprint(&db, "DB::defrag failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform setbulk command +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.set_bulk(recs) != (int64_t)recs.size()) { + dberrprint(&db, "DB::set_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform removebulk command +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.remove_bulk(keys) < 0) { + dberrprint(&db, "DB::remove_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform getbulk command +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + std::map<std::string, std::string> recs; + if (db.get_bulk(keys, &recs) >= 0) { + std::map<std::string, std::string>::iterator it = recs.begin(); + std::map<std::string, std::string>::iterator itend = recs.end(); + while (it != itend) { + printdata(it->first.data(), it->first.size(), px); + oprintf("\t"); + printdata(it->second.data(), it->second.size(), px); + oprintf("\n"); + ++it; + } + } else { + dberrprint(&db, "DB::get_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform check command +static int32_t proccheck(const char* path, int32_t oflags) { + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::HashDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + kc::HashDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::jump failed"); + err = true; + } + int64_t cnt = 0; + while (!err) { + size_t ksiz; + const char* vbuf; + size_t vsiz; + char* kbuf = cur.get(&ksiz, &vbuf, &vsiz); + if (kbuf) { + cnt++; + size_t rsiz; + char* rbuf = db.get(kbuf, ksiz, &rsiz); + if (rbuf) { + if (rsiz != vsiz || std::memcmp(rbuf, vbuf, rsiz)) { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] rbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] kbuf; + if (cnt % 1000 == 0) { + oputchar('.'); + if (cnt % 50000 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + } else { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::get failed"); + err = true; + } + break; + } + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::step failed"); + err = true; + } + } + oprintf(" (end)\n"); + if (db.count() != cnt) { + dberrprint(&db, "DB::count failed"); + err = true; + } + kc::File::Status sbuf; + if (kc::File::status(path, &sbuf)) { + if (db.size() != sbuf.size && sbuf.size % (1 << 20) != 0) { + dberrprint(&db, "DB::size failed"); + err = true; + } + } else { + dberrprint(&db, "File::status failed"); + err = true; + } + if (db.flags() & kc::HashDB::FFATAL) { + dberrprint(&db, "DB::flags indicated fatal error"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld records were checked successfully\n", (long long)cnt); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kchashtest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kchashtest.cc new file mode 100644 index 0000000000..77477196e6 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kchashtest.cc @@ -0,0 +1,2414 @@ +/************************************************************************************************* + * The test cases of the file hash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kchashdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t apow, int32_t fpow, + int32_t opts, int64_t bnum, int64_t msiz, int64_t dfunit, bool lv); +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + bool rnd, int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, + int64_t bnum, int64_t msiz, int64_t dfunit, bool lv); +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, + int64_t bnum, int64_t msiz, int64_t dfunit, bool lv); +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, + int64_t bnum, int64_t msiz, int64_t dfunit, bool lv); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the file hash database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-set|-get|-getw|-rem|-etc] [-tran]" + " [-oat|-oas|-onl|-otl|-onr] [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num]" + " [-msiz num] [-dfunit num] [-lv] path rnum\n", g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-oat|-oas|-onl|-otl|-onr]" + " [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num] [-msiz num] [-dfunit num]" + " [-lv] path rnum\n", g_progname); + eprintf(" %s wicked [-th num] [-it num] [-oat|-oas|-onl|-otl|-onr]" + " [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num] [-msiz num] [-dfunit num]" + " [-lv] path rnum\n", g_progname); + eprintf(" %s tran [-th num] [-it num] [-hard] [-oat|-oas|-onl|-otl|-onr]" + " [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num] [-msiz num] [-dfunit num]" + " [-lv] path rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["fbpnum_used"] = ""; + status["bnum_used"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t rtype = kc::atoi(status["realtype"].c_str()); + if (rtype > 0 && rtype != type) + oprintf("real type: %s (%s) (realtype=0x%02X)\n", + kc::BasicDB::typecname(rtype), kc::BasicDB::typestring(rtype), rtype); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::HashDB::FOPEN) oprintf(" open"); + if (flags & kc::HashDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + if (kc::atoi(status["trimmed"].c_str()) > 0) oprintf(" (trimmed)"); + oprintf("\n", flags); + int32_t apow = kc::atoi(status["apow"].c_str()); + oprintf("alignment: %d (apow=%d)\n", 1 << apow, apow); + int32_t fpow = kc::atoi(status["fpow"].c_str()); + int32_t fbpnum = fpow > 0 ? 1 << fpow : 0; + int32_t fbpused = kc::atoi(status["fbpnum_used"].c_str()); + int64_t frgcnt = kc::atoi(status["frgcnt"].c_str()); + oprintf("free block pool: %d (fpow=%d) (used=%d) (frg=%lld)\n", + fbpnum, fpow, fbpused, (long long)frgcnt); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::HashDB::TSMALL) oprintf(" small"); + if (opts & kc::HashDB::TLINEAR) oprintf(" linear"); + if (opts & kc::HashDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t bnumused = kc::atoi(status["bnum_used"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + double load = 0; + if (count > 0 && bnumused > 0) { + load = (double)count / bnumused; + if (!(opts & kc::HashDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (used=%lld) (load=%.2f)\n", + (long long)bnum, (long long)bnumused, load); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + int64_t msiz = kc::atoi(status["msiz"].c_str()); + int64_t realsize = kc::atoi(status["realsize"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s) (map=%lld)", size, sizestr.c_str(), (long long)msiz); + if (size != realsize) oprintf(" (gap=%lld)", (long long)(realsize - size)); + oprintf("\n"); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + int32_t mode = 0; + bool tran = false; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-set")) { + mode = 's'; + } else if (!std::strcmp(argv[i], "-get")) { + mode = 'g'; + } else if (!std::strcmp(argv[i], "-getw")) { + mode = 'w'; + } else if (!std::strcmp(argv[i], "-rem")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-etc")) { + mode = 'e'; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::HashDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::HashDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::HashDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::HashDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::HashDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(path, rnum, thnum, rnd, mode, tran, oflags, + apow, fpow, opts, bnum, msiz, dfunit, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::HashDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::HashDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::HashDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::HashDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::HashDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(path, rnum, thnum, itnum, rnd, oflags, + apow, fpow, opts, bnum, msiz, dfunit, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::HashDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::HashDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::HashDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::HashDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::HashDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(path, rnum, thnum, itnum, oflags, + apow, fpow, opts, bnum, msiz, dfunit, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool hard = false; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-hard")) { + hard = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::HashDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::HashDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::HashDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::HashDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::HashDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::HashDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::HashDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::HashDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(path, rnum, thnum, itnum, hard, oflags, + apow, fpow, opts, bnum, msiz, dfunit, lv); + return rv; +} + + +// perform order command +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t apow, int32_t fpow, + int32_t opts, int64_t bnum, int64_t msiz, int64_t dfunit, bool lv) { + oprintf("<In-order Test>\n seed=%u path=%s rnum=%lld thnum=%d rnd=%d mode=%d tran=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld msiz=%lld dfunit=%lld" + " lv=%d\n\n", g_randseed, path, (long long)rnum, thnum, rnd, mode, tran, + oflags, apow, fpow, opts, (long long)bnum, (long long)msiz, (long long)dfunit, lv); + bool err = false; + kc::HashDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + uint32_t omode = kc::HashDB::OWRITER | kc::HashDB::OCREATE | kc::HashDB::OTRUNCATE; + if (mode == 'r') { + omode = kc::HashDB::OWRITER | kc::HashDB::OCREATE; + } else if (mode == 'g' || mode == 'w') { + omode = kc::HashDB::OREADER; + } + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (mode == 0 || mode == 's' || mode == 'e') { + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 's'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + if (mode == 0 || mode == 'g' || mode == 'e') { + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'g'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'w' || mode == 'e') { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'w'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::HashDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size, int64_t msiz) : + rnum_(rnum), rnd_(rnd), size_(size), msiz_(msiz) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + kc::File::Status sbuf; + if (!kc::File::status(path, &sbuf)) return false; + if (sbuf.size != size_ && sbuf.size != msiz_ && + sbuf.size % (1 << 20) != 0) return false; + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + int64_t msiz_; + } syncprocessor(rnum, rnd, db.size(), msiz); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e' && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'r' || mode == 'e') { + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && mode_ != 'e') || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, mode, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, mode, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'r' || mode == 'e'); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + bool rnd, int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, + int64_t bnum, int64_t msiz, int64_t dfunit, bool lv) { + oprintf("<Queue Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d rnd=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld msiz=%lld dfunit=%lld" + " lv=%d\n\n", g_randseed, path, (long long)rnum, thnum, itnum, rnd, + oflags, apow, fpow, opts, (long long)bnum, (long long)msiz, (long long)dfunit, lv); + bool err = false; + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::HashDB::OWRITER | kc::HashDB::OCREATE; + if (itcnt == 1) omode |= kc::HashDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::HashDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::HashDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, + int64_t bnum, int64_t msiz, int64_t dfunit, bool lv) { + oprintf("<Wicked Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld msiz=%lld dfunit=%lld" + " lv=%d\n\n", g_randseed, path, (long long)rnum, thnum, itnum, + oflags, apow, fpow, opts, (long long)bnum, (long long)msiz, (long long)dfunit, lv); + bool err = false; + kc::HashDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::HashDB::OWRITER | kc::HashDB::OCREATE; + if (itcnt == 1) omode |= kc::HashDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::HashDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(5) > 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (myrand(2) == 0) { + if (!db_->defrag(0)) { + dberrprint(db_, __LINE__, "DB::defrag"); + err_ = true; + } + } else { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::HashDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, + int64_t bnum, int64_t msiz, int64_t dfunit, bool lv) { + oprintf("<Transaction Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d hard=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld msiz=%lld dfunit=%lld" + " lv=%d\n\n", g_randseed, path, (long long)rnum, thnum, itnum, hard, + oflags, apow, fpow, opts, (long long)bnum, (long long)msiz, (long long)dfunit, lv); + bool err = false; + kc::HashDB db; + kc::HashDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::HashDB::OWRITER | kc::HashDB::OCREATE; + if (itcnt == 1) omode |= kc::HashDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + std::string parapath = db.path() + "-para"; + if (!paradb.open(parapath, omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::HashDB* db, kc::HashDB* paradb, int64_t rnum, + int32_t thnum, bool hard, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + hard_ = hard; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::HashDB* db_; + kc::HashDB* paradb_; + int64_t rnum_; + int32_t thnum_; + bool hard_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, hard, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, hard, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kclangc.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kclangc.cc new file mode 100644 index 0000000000..6b3a894d5c --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kclangc.cc @@ -0,0 +1,1534 @@ +/************************************************************************************************* + * C language binding + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcpolydb.h" +#include "kcdbext.h" +#include "kclangc.h" +#include "myconf.h" + +using namespace kyotocabinet; + +extern "C" { + + +/** The package version. */ +const char* const KCVERSION = VERSION; + + +/** Special pointer for no operation by the visiting function. */ +const char* const KCVISNOP = DB::Visitor::NOP; + + +/** Special pointer to remove the record by the visiting function. */ +const char* const KCVISREMOVE = DB::Visitor::REMOVE; + + +/** + * Allocate a region on memory. + */ +void* kcmalloc(size_t size) { + _assert_(size > 0 && size <= MEMMAXSIZ); + return new char[size]; +} + + +/** + * Release a region allocated in the library. + */ +void kcfree(void* ptr) { + _assert_(true); + delete[] (char*)ptr; +} + + +/** + * Get the time of day in seconds. + */ +double kctime(void) { + _assert_(true); + return kyotocabinet::time(); +} + + +/** + * Convert a string to an integer. + */ +int64_t kcatoi(const char* str) { + _assert_(str); + return kyotocabinet::atoi(str); +} + + +/** + * Convert a string with a metric prefix to an integer. + */ +int64_t kcatoix(const char* str) { + _assert_(str); + return kyotocabinet::atoix(str); +} + + +/** + * Convert a string to a real number. + */ +double kcatof(const char* str) { + _assert_(str); + return kyotocabinet::atof(str); +} + + +/** + * Get the hash value by MurMur hashing. + */ +uint64_t kchashmurmur(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + return kyotocabinet::hashmurmur(buf, size); +} + + +/** + * Get the hash value by FNV hashing. + */ +uint64_t kchashfnv(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + return kyotocabinet::hashfnv(buf, size); +} + + +/** + * Calculate the levenshtein distance of two regions. + */ +size_t kclevdist(const void* abuf, size_t asiz, const void* bbuf, size_t bsiz, int32_t utf) { + _assert_(abuf && asiz <= MEMMAXSIZ && bbuf && bsiz <= MEMMAXSIZ); + size_t dist; + if (utf) { + uint32_t astack[128]; + uint32_t* aary = asiz > sizeof(astack) / sizeof(*astack) ? new uint32_t[asiz] : astack; + size_t anum; + strutftoucs((const char*)abuf, asiz, aary, &anum); + uint32_t bstack[128]; + uint32_t* bary = bsiz > sizeof(bstack) / sizeof(*bstack) ? new uint32_t[bsiz] : bstack; + size_t bnum; + strutftoucs((const char*)bbuf, bsiz, bary, &bnum); + dist = strucsdist(aary, anum, bary, bnum); + if (bary != bstack) delete[] bary; + if (aary != astack) delete[] aary; + } else { + dist = memdist(abuf, asiz, bbuf, bsiz); + } + return dist; +} + + +/** + * Get the quiet Not-a-Number value. + */ +double kcnan() { + _assert_(true); + return kyotocabinet::nan(); +} + + +/** + * Get the positive infinity value. + */ +double kcinf() { + _assert_(true); + return kyotocabinet::inf(); +} + + +/** + * Check a number is a Not-a-Number value. + */ +int32_t kcchknan(double num) { + _assert_(true); + return kyotocabinet::chknan(num); +} + + +/** + * Check a number is an infinity value. + */ +int32_t kcchkinf(double num) { + _assert_(true); + return kyotocabinet::chkinf(num); +} + + +/** + * Get the readable string of an error code. + */ +const char* kcecodename(int32_t code) { + _assert_(true); + return BasicDB::Error::codename((BasicDB::Error::Code)code); +} + + +/** + * Create a database object. + */ +KCDB* kcdbnew(void) { + _assert_(true); + return (KCDB*)new PolyDB; +} + + +/** + * Destroy a database object. + */ +void kcdbdel(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + delete pdb; +} + + +/** + * Open a database file. + */ +int32_t kcdbopen(KCDB* db, const char* path, uint32_t mode) { + _assert_(db && path); + PolyDB* pdb = (PolyDB*)db; + return pdb->open(path, mode); +} + + +/** + * Close the database file. + */ +int32_t kcdbclose(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->close(); +} + + +/** + * Get the code of the last happened error. + */ +int32_t kcdbecode(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->error().code(); +} + + +/** + * Get the supplement message of the last happened error. + */ +const char* kcdbemsg(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->error().message(); +} + + +/** + * Accept a visitor to a record. + */ +int32_t kcdbaccept(KCDB* db, const char* kbuf, size_t ksiz, + KCVISITFULL fullproc, KCVISITEMPTY emptyproc, void* opq, int32_t writable) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl(KCVISITFULL fullproc, KCVISITEMPTY emptyproc, void* opq) : + fullproc_(fullproc), emptyproc_(emptyproc), opq_(opq) {} + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (!fullproc_) return NOP; + return fullproc_(kbuf, ksiz, vbuf, vsiz, sp, opq_); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + if (!emptyproc_) return NOP; + return emptyproc_(kbuf, ksiz, sp, opq_); + } + private: + KCVISITFULL fullproc_; + KCVISITEMPTY emptyproc_; + void* opq_; + }; + VisitorImpl visitor(fullproc, emptyproc, opq); + return pdb->accept(kbuf, ksiz, &visitor, writable); +} + + +/** + * Accept a visitor to multiple records at once. + */ +int32_t kcdbacceptbulk(KCDB* db, const KCSTR* keys, size_t knum, + KCVISITFULL fullproc, KCVISITEMPTY emptyproc, + void* opq, int32_t writable) { + _assert_(db && keys && knum <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + std::vector<std::string> xkeys; + xkeys.reserve(knum); + for (size_t i = 0; i < knum; i++) { + xkeys.push_back(std::string(keys[i].buf, keys[i].size)); + } + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl(KCVISITFULL fullproc, KCVISITEMPTY emptyproc, void* opq) : + fullproc_(fullproc), emptyproc_(emptyproc), opq_(opq) {} + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (!fullproc_) return NOP; + return fullproc_(kbuf, ksiz, vbuf, vsiz, sp, opq_); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + if (!emptyproc_) return NOP; + return emptyproc_(kbuf, ksiz, sp, opq_); + } + private: + KCVISITFULL fullproc_; + KCVISITEMPTY emptyproc_; + void* opq_; + }; + VisitorImpl visitor(fullproc, emptyproc, opq); + return pdb->accept_bulk(xkeys, &visitor, writable); +} + + +/** + * Iterate to accept a visitor for each record. + */ +int32_t kcdbiterate(KCDB* db, KCVISITFULL fullproc, void* opq, int32_t writable) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl(KCVISITFULL fullproc, void* opq) : fullproc_(fullproc), opq_(opq) {} + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (!fullproc_) return NOP; + return fullproc_(kbuf, ksiz, vbuf, vsiz, sp, opq_); + } + private: + KCVISITFULL fullproc_; + void* opq_; + }; + VisitorImpl visitor(fullproc, opq); + return pdb->iterate(&visitor, writable); +} + + +/** + * Scan each record in parallel. + */ +int32_t kcdbscanpara(KCDB* db, KCVISITFULL fullproc, void* opq, size_t thnum) { + _assert_(db && thnum <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl(KCVISITFULL fullproc, void* opq) : fullproc_(fullproc), opq_(opq) {} + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (!fullproc_) return NOP; + return fullproc_(kbuf, ksiz, vbuf, vsiz, sp, opq_); + } + private: + KCVISITFULL fullproc_; + void* opq_; + }; + VisitorImpl visitor(fullproc, opq); + return pdb->scan_parallel(&visitor, thnum); +} + + +/** + * Set the value of a record. + */ +int32_t kcdbset(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->set(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Add a record. + */ +int32_t kcdbadd(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->add(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Replace the value of a record. + */ +int32_t kcdbreplace(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->replace(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Append the value of a record. + */ +int32_t kcdbappend(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->append(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Add a number to the numeric value of a record. + */ +int64_t kcdbincrint(KCDB* db, const char* kbuf, size_t ksiz, int64_t num, int64_t orig) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->increment(kbuf, ksiz, num, orig); +} + + +/** + * Add a number to the numeric value of a record. + */ +double kcdbincrdouble(KCDB* db, const char* kbuf, size_t ksiz, double num, double orig) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->increment_double(kbuf, ksiz, num, orig); +} + + +/** + * Perform compare-and-swap. + */ +int32_t kcdbcas(KCDB* db, const char* kbuf, size_t ksiz, + const char* ovbuf, size_t ovsiz, const char* nvbuf, size_t nvsiz) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->cas(kbuf, ksiz, ovbuf, ovsiz, nvbuf, nvsiz); +} + + +/** + * Remove a record. + */ +int32_t kcdbremove(KCDB* db, const char* kbuf, size_t ksiz) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->remove(kbuf, ksiz); +} + + +/** + * Retrieve the value of a record. + */ +char* kcdbget(KCDB* db, const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ && sp); + PolyDB* pdb = (PolyDB*)db; + return pdb->get(kbuf, ksiz, sp); +} + + +/** + * Check the existence of a record. + */ +int32_t kcdbcheck(KCDB* db, const char* kbuf, size_t ksiz) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->check(kbuf, ksiz); +} + + +/** + * Retrieve the value of a record. + */ +int32_t kcdbgetbuf(KCDB* db, const char* kbuf, size_t ksiz, char* vbuf, size_t max) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ && vbuf); + PolyDB* pdb = (PolyDB*)db; + return pdb->get(kbuf, ksiz, vbuf, max); +} + + +/** + * Retrieve the value of a record and remove it atomically. + */ +char* kcdbseize(KCDB* db, const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(db && kbuf && ksiz <= MEMMAXSIZ && sp); + PolyDB* pdb = (PolyDB*)db; + return pdb->seize(kbuf, ksiz, sp); +} + + +/** + * Store records at once. + */ +int64_t kcdbsetbulk(KCDB* db, const KCREC* recs, size_t rnum, int32_t atomic) { + _assert_(db && recs && rnum <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + std::map<std::string, std::string> xrecs; + for (size_t i = 0; i < rnum; i++) { + const KCREC* rec = recs + i; + xrecs[std::string(rec->key.buf, rec->key.size)] = + std::string(rec->value.buf, rec->value.size); + } + return pdb->set_bulk(xrecs, atomic); +} + + +/** + * Remove records at once. + */ +int64_t kcdbremovebulk(KCDB* db, const KCSTR* keys, size_t knum, int32_t atomic) { + _assert_(db && keys && knum <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + std::vector<std::string> xkeys; + xkeys.reserve(knum); + for (size_t i = 0; i < knum; i++) { + const KCSTR* key = keys + i; + xkeys.push_back(std::string(key->buf, key->size)); + } + return pdb->remove_bulk(xkeys, atomic); +} + + +/** + * Retrieve records at once. + */ +int64_t kcdbgetbulk(KCDB* db, const KCSTR* keys, size_t knum, KCREC* recs, int32_t atomic) { + _assert_(db && keys && knum <= MEMMAXSIZ && recs); + PolyDB* pdb = (PolyDB*)db; + std::vector<std::string> xkeys; + xkeys.reserve(knum); + for (size_t i = 0; i < knum; i++) { + const KCSTR* key = keys + i; + xkeys.push_back(std::string(key->buf, key->size)); + } + std::map<std::string, std::string> xrecs; + if (pdb->get_bulk(xkeys, &xrecs, atomic) < 0) return -1; + std::map<std::string, std::string>::iterator it = xrecs.begin(); + std::map<std::string, std::string>::iterator itend = xrecs.end(); + size_t ridx = 0; + while (ridx < knum && it != itend) { + size_t ksiz = it->first.size(); + char* kbuf = new char[ksiz+1]; + std::memcpy(kbuf, it->first.data(), ksiz); + kbuf[ksiz] = '\0'; + size_t vsiz = it->second.size(); + char* vbuf = new char[vsiz+1]; + std::memcpy(vbuf, it->second.data(), vsiz); + vbuf[vsiz] = '\0'; + KCREC* rec = recs + (ridx++); + rec->key.buf = kbuf; + rec->key.size = ksiz; + rec->value.buf = vbuf; + rec->value.size = vsiz; + ++it; + } + return ridx; +} + + +/** + * Synchronize updated contents with the file and the device. + */ +int32_t kcdbsync(KCDB* db, int32_t hard, KCFILEPROC proc, void* opq) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + class FileProcessorImpl : public BasicDB::FileProcessor { + public: + explicit FileProcessorImpl(KCFILEPROC proc, void* opq) : proc_(proc), opq_(opq) {} + bool process(const std::string& path, int64_t count, int64_t size) { + if (!proc_) return true; + return proc_(path.c_str(), count, size, opq_); + } + private: + KCFILEPROC proc_; + void* opq_; + }; + FileProcessorImpl myproc(proc, opq); + return pdb->synchronize(hard, &myproc); +} + + +/** + * Occupy database by locking and do something meanwhile. + */ +int32_t kcdboccupy(KCDB* db, int32_t writable, KCFILEPROC proc, void* opq) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + class FileProcessorImpl : public BasicDB::FileProcessor { + public: + explicit FileProcessorImpl(KCFILEPROC proc, void* opq) : proc_(proc), opq_(opq) {} + bool process(const std::string& path, int64_t count, int64_t size) { + if (!proc_) return true; + return proc_(path.c_str(), count, size, opq_); + } + private: + KCFILEPROC proc_; + void* opq_; + }; + FileProcessorImpl myproc(proc, opq); + return pdb->occupy(writable, &myproc); +} + + +/** + * Create a copy of the database file. + */ +int32_t kcdbcopy(KCDB* db, const char* dest) { + _assert_(db && dest); + PolyDB* pdb = (PolyDB*)db; + return pdb->copy(dest); +} + + +/** + * Begin transaction. + */ +int32_t kcdbbegintran(KCDB* db, int32_t hard) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->begin_transaction(hard); +} + + +/** + * Try to begin transaction. + */ +int32_t kcdbbegintrantry(KCDB* db, int32_t hard) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->begin_transaction_try(hard); +} + + +/** + * End transaction. + */ +int32_t kcdbendtran(KCDB* db, int32_t commit) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->end_transaction(commit); +} + + +/** + * Remove all records. + */ +int32_t kcdbclear(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->clear(); +} + + +/** + * Dump records into a file. + */ +int32_t kcdbdumpsnap(KCDB* db, const char* dest) { + _assert_(db && dest); + PolyDB* pdb = (PolyDB*)db; + return pdb->dump_snapshot(dest); +} + + +/** + * Load records from a file. + */ +int32_t kcdbloadsnap(KCDB* db, const char* src) { + _assert_(db && src); + PolyDB* pdb = (PolyDB*)db; + return pdb->load_snapshot(src); +} + + +/** + * Get the number of records. + */ +int64_t kcdbcount(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->count(); +} + + +/** + * Get the size of the database file. + */ +int64_t kcdbsize(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return pdb->size(); +} + + +/** + * Get the path of the database file. + */ +char* kcdbpath(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + std::string path = pdb->path(); + size_t psiz = path.size(); + char* pbuf = new char[psiz+1]; + std::memcpy(pbuf, path.c_str(), psiz + 1); + return pbuf; +} + + +/** + * Get the miscellaneous status information. + */ +char* kcdbstatus(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + std::map<std::string, std::string> status; + if (!pdb->status(&status)) return NULL; + std::ostringstream obuf; + std::map<std::string, std::string>::iterator it = status.begin(); + std::map<std::string, std::string>::iterator itend = status.end(); + while (it != itend) { + obuf << it->first << "\t" << it->second << "\n"; + ++it; + } + std::string sstr = obuf.str(); + size_t ssiz = sstr.size(); + char* sbuf = new char[ssiz+1]; + std::memcpy(sbuf, sstr.c_str(), ssiz + 1); + return sbuf; +} + + +/** + * Get keys matching a prefix string. + */ +int64_t kcdbmatchprefix(KCDB* db, const char* prefix, char** strary, size_t max) { + _assert_(db && prefix && strary && max <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + std::vector<std::string> strvec; + if (pdb->match_prefix(prefix, &strvec, max) == -1) return -1; + int64_t cnt = 0; + std::vector<std::string>::iterator it = strvec.begin(); + std::vector<std::string>::iterator itend = strvec.end(); + while (it != itend) { + size_t ksiz = it->size(); + char* kbuf = new char[ksiz+1]; + std::memcpy(kbuf, it->data(), ksiz); + kbuf[ksiz] = '\0'; + strary[cnt++] = kbuf; + ++it; + } + return cnt; +} + + +/** + * Get keys matching a regular expression string. + */ +int64_t kcdbmatchregex(KCDB* db, const char* regex, char** strary, size_t max) { + _assert_(db && regex && strary && max <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + std::vector<std::string> strvec; + if (pdb->match_regex(regex, &strvec, max) == -1) return -1; + int64_t cnt = 0; + std::vector<std::string>::iterator it = strvec.begin(); + std::vector<std::string>::iterator itend = strvec.end(); + while (it != itend) { + size_t ksiz = it->size(); + char* kbuf = new char[ksiz+1]; + std::memcpy(kbuf, it->data(), ksiz); + kbuf[ksiz] = '\0'; + strary[cnt++] = kbuf; + ++it; + } + return cnt; +} + + +/** + * Get keys similar to a string in terms of the levenshtein distance. + */ +int64_t kcdbmatchsimilar(KCDB* db, const char* origin, uint32_t range, int32_t utf, + char** strary, size_t max) { + _assert_(db && origin && strary && max <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + std::vector<std::string> strvec; + if (pdb->match_similar(origin, range, utf, &strvec, max) == -1) return -1; + int64_t cnt = 0; + std::vector<std::string>::iterator it = strvec.begin(); + std::vector<std::string>::iterator itend = strvec.end(); + while (it != itend) { + size_t ksiz = it->size(); + char* kbuf = new char[ksiz+1]; + std::memcpy(kbuf, it->data(), ksiz); + kbuf[ksiz] = '\0'; + strary[cnt++] = kbuf; + ++it; + } + return cnt; +} + + +/** + * Merge records from other databases. + */ +int32_t kcdbmerge(KCDB* db, KCDB** srcary, size_t srcnum, uint32_t mode) { + _assert_(db && srcary && srcnum <= MEMMAXSIZ); + PolyDB* pdb = (PolyDB*)db; + return pdb->merge((BasicDB**)srcary, srcnum, (PolyDB::MergeMode)mode); +} + + +/** + * Create a cursor object. + */ +KCCUR* kcdbcursor(KCDB* db) { + _assert_(db); + PolyDB* pdb = (PolyDB*)db; + return (KCCUR*)pdb->cursor(); +} + + +/** + * Destroy a cursor object. + */ +void kccurdel(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + delete pcur; +} + + +/** + * Accept a visitor to the current record. + */ +int32_t kccuraccept(KCCUR* cur, KCVISITFULL fullproc, void* opq, + int32_t writable, int32_t step) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl(KCVISITFULL fullproc, void* opq) : fullproc_(fullproc), opq_(opq) {} + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (!fullproc_) return NOP; + return fullproc_(kbuf, ksiz, vbuf, vsiz, sp, opq_); + } + private: + KCVISITFULL fullproc_; + void* opq_; + }; + VisitorImpl visitor(fullproc, opq); + return pcur->accept(&visitor, writable, step); +} + + +/** + * Set the value of the current record. + */ +int32_t kccursetvalue(KCCUR* cur, const char* vbuf, size_t vsiz, int32_t step) { + _assert_(cur && vbuf && vsiz <= MEMMAXSIZ); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->set_value(vbuf, vsiz, step); +} + + +/** + * Remove the current record. + */ +int32_t kccurremove(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->remove(); +} + + +/** + * Get the key of the current record. + */ +char* kccurgetkey(KCCUR* cur, size_t* sp, int32_t step) { + _assert_(cur && sp); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->get_key(sp, step); +} + + +/** + * Get the value of the current record. + */ +char* kccurgetvalue(KCCUR* cur, size_t* sp, int32_t step) { + _assert_(cur && sp); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->get_value(sp, step); +} + + +/** + * Get a pair of the key and the value of the current record. + */ +char* kccurget(KCCUR* cur, size_t* ksp, const char** vbp, size_t* vsp, int32_t step) { + _assert_(cur && ksp && vbp && vsp); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->get(ksp, vbp, vsp, step); +} + + +/** + * Get a pair of the key and the value of the current record and remove it atomically. + */ +char* kccurseize(KCCUR* cur, size_t* ksp, const char** vbp, size_t* vsp) { + _assert_(cur && ksp && vbp && vsp); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->seize(ksp, vbp, vsp); +} + + +/** + * Jump the cursor to the first record. + */ +int32_t kccurjump(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->jump(); +} + + +/** + * Jump the cursor to a record. + */ +int32_t kccurjumpkey(KCCUR* cur, const char* kbuf, size_t ksiz) { + _assert_(cur && kbuf && ksiz <= MEMMAXSIZ); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->jump(kbuf, ksiz); +} + + +/** + * Jump the cursor to the last record for backward scan. + */ +int32_t kccurjumpback(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->jump_back(); +} + + +/** + * Jump the cursor to a record for backward scan. + */ +int32_t kccurjumpbackkey(KCCUR* cur, const char* kbuf, size_t ksiz) { + _assert_(cur && kbuf && ksiz <= MEMMAXSIZ); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->jump_back(kbuf, ksiz); +} + + +/** + * Step the cursor to the next record. + */ +int32_t kccurstep(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->step(); +} + + +/** + * Step the cursor to the previous record. + */ +int32_t kccurstepback(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->step_back(); +} + + +/** + * Get the database object. + */ +KCDB* kccurdb(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return (KCDB*)pcur->db(); +} + + +/** + * Get the code of the last happened error. + */ +int32_t kccurecode(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->error().code(); +} + + +/** + * Get the supplement message of the last happened error. + */ +const char* kccuremsg(KCCUR* cur) { + _assert_(cur); + PolyDB::Cursor* pcur = (PolyDB::Cursor*)cur; + return pcur->error().message(); +} + + +/** + * Create an index database object. + */ +KCIDX* kcidxnew(void) { + _assert_(true); + return (KCIDX*)new IndexDB; +} + + +/** + * Destroy a database object. + */ +void kcidxdel(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + delete idb; +} + + +/** + * Open a database file. + */ +int32_t kcidxopen(KCIDX* idx, const char* path, uint32_t mode) { + _assert_(idx && path); + IndexDB* idb = (IndexDB*)idx; + return idb->open(path, mode); +} + + +/** + * Close the database file. + */ +int32_t kcidxclose(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + return idb->close(); +} + + +/** + * Get the code of the last happened error. + */ +int32_t kcidxecode(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + return idb->error().code(); +} + + +/** + * Get the supplement message of the last happened error. + */ +const char* kcidxemsg(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + return idb->error().message(); +} + + +/** + * Set the value of a record. + */ +int32_t kcidxset(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(idx && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + IndexDB* idb = (IndexDB*)idx; + return idb->set(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Add a record. + */ +int32_t kcidxadd(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(idx && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + IndexDB* idb = (IndexDB*)idx; + return idb->add(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Replace the value of a record. + */ +int32_t kcidxreplace(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(idx && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + IndexDB* idb = (IndexDB*)idx; + return idb->replace(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Append the value of a record. + */ +int32_t kcidxappend(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(idx && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + IndexDB* idb = (IndexDB*)idx; + return idb->append(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Remove a record. + */ +int32_t kcidxremove(KCIDX* idx, const char* kbuf, size_t ksiz) { + _assert_(idx && kbuf && ksiz <= MEMMAXSIZ); + IndexDB* idb = (IndexDB*)idx; + return idb->remove(kbuf, ksiz); +} + + +/** + * Retrieve the value of a record. + */ +char* kcidxget(KCIDX* idx, const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(idx && kbuf && ksiz <= MEMMAXSIZ && sp); + IndexDB* idb = (IndexDB*)idx; + return idb->get(kbuf, ksiz, sp); +} + + +/** + * Synchronize updated contents with the file and the device. + */ +int32_t kcidxsync(KCIDX* idx, int32_t hard, KCFILEPROC proc, void* opq) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + class FileProcessorImpl : public BasicDB::FileProcessor { + public: + explicit FileProcessorImpl(KCFILEPROC proc, void* opq) : proc_(proc), opq_(opq) {} + bool process(const std::string& path, int64_t count, int64_t size) { + if (!proc_) return true; + return proc_(path.c_str(), count, size, opq_); + } + private: + KCFILEPROC proc_; + void* opq_; + }; + FileProcessorImpl myproc(proc, opq); + return idb->synchronize(hard, &myproc); +} + + +/** + * Remove all records. + */ +int32_t kcidxclear(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + return idb->clear(); +} + + +/** + * Get the number of records. + */ +int64_t kcidxcount(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + return idb->count(); +} + + +/** + * Get the size of the database file. + */ +int64_t kcidxsize(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + return idb->size(); +} + + +/** + * Get the path of the database file. + */ +char* kcidxpath(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + std::string path = idb->path(); + size_t psiz = path.size(); + char* pbuf = new char[psiz+1]; + std::memcpy(pbuf, path.c_str(), psiz + 1); + return pbuf; +} + + +/** + * Get the miscellaneous status information. + */ +char* kcidxstatus(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + std::map<std::string, std::string> status; + if (!idb->status(&status)) return NULL; + std::ostringstream obuf; + std::map<std::string, std::string>::iterator it = status.begin(); + std::map<std::string, std::string>::iterator itend = status.end(); + while (it != itend) { + obuf << it->first << "\t" << it->second << "\n"; + ++it; + } + std::string sstr = obuf.str(); + size_t ssiz = sstr.size(); + char* sbuf = new char[ssiz+1]; + std::memcpy(sbuf, sstr.c_str(), ssiz + 1); + return sbuf; +} + + +/** + * Reveal the inner database object. + */ +KCDB* kcidxrevealinnerdb(KCIDX* idx) { + _assert_(idx); + IndexDB* idb = (IndexDB*)idx; + return (KCDB*)idb->reveal_inner_db(); +} + + +/** + * Create a string hash map object. + */ +KCMAP* kcmapnew(size_t bnum) { + _assert_(true); + return (KCMAP*)new TinyHashMap(bnum); +} + + +/** + * Destroy a map object. + */ +void kcmapdel(KCMAP* map) { + _assert_(map); + TinyHashMap* thm = (TinyHashMap*)map; + delete thm; +} + + +/** + * Set the value of a record. + */ +void kcmapset(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(map && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + TinyHashMap* thm = (TinyHashMap*)map; + thm->set(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Add a record. + */ +int32_t kcmapadd(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(map && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + TinyHashMap* thm = (TinyHashMap*)map; + return thm->add(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Replace the value of a record. + */ +int32_t kcmapreplace(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(map && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + TinyHashMap* thm = (TinyHashMap*)map; + return thm->replace(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Append the value of a record. + */ +void kcmapappend(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(map && kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + TinyHashMap* thm = (TinyHashMap*)map; + thm->append(kbuf, ksiz, vbuf, vsiz); +} + + +/** + * Remove a record. + */ +int32_t kcmapremove(KCMAP* map, const char* kbuf, size_t ksiz) { + _assert_(map && kbuf && ksiz <= MEMMAXSIZ); + TinyHashMap* thm = (TinyHashMap*)map; + return thm->remove(kbuf, ksiz); +} + + +/** + * Retrieve the value of a record. + */ +const char* kcmapget(KCMAP* map, const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(map && kbuf && ksiz <= MEMMAXSIZ && sp); + TinyHashMap* thm = (TinyHashMap*)map; + return thm->get(kbuf, ksiz, sp); +} + + +/** + * Remove all records. + */ +void kcmapclear(KCMAP* map) { + _assert_(map); + TinyHashMap* thm = (TinyHashMap*)map; + thm->clear(); +} + + +/** + * Get the number of records. + */ +size_t kcmapcount(KCMAP* map) { + _assert_(map); + TinyHashMap* thm = (TinyHashMap*)map; + return thm->count(); +} + + +/** + * Create a string hash map iterator object. + */ +KCMAPITER* kcmapiterator(KCMAP* map) { + _assert_(map); + TinyHashMap* thm = (TinyHashMap*)map; + return (KCMAPITER*)new TinyHashMap::Iterator(thm); +} + + +/** + * Destroy an iterator object. + */ +void kcmapiterdel(KCMAPITER* iter) { + _assert_(iter); + TinyHashMap::Iterator* thmi = (TinyHashMap::Iterator*)iter; + delete thmi; +} + + +/** + * Get the key of the current record. + */ +const char* kcmapitergetkey(KCMAPITER* iter, size_t* sp) { + _assert_(iter && sp); + TinyHashMap::Iterator* thmi = (TinyHashMap::Iterator*)iter; + return thmi->get_key(sp); +} + + +/** + * Get the value of the current record. + */ +const char* kcmapitergetvalue(KCMAPITER* iter, size_t* sp) { + _assert_(iter && sp); + TinyHashMap::Iterator* thmi = (TinyHashMap::Iterator*)iter; + return thmi->get_value(sp); +} + + +/** + * Get a pair of the key and the value of the current record. + */ +const char* kcmapiterget(KCMAPITER* iter, size_t* ksp, const char** vbp, size_t* vsp) { + _assert_(iter && ksp && vbp && vsp); + TinyHashMap::Iterator* thmi = (TinyHashMap::Iterator*)iter; + return thmi->get(ksp, vbp, vsp); +} + + +/** + * Step the cursor to the next record. + */ +void kcmapiterstep(KCMAPITER* iter) { + _assert_(iter); + TinyHashMap::Iterator* thmi = (TinyHashMap::Iterator*)iter; + return thmi->step(); +} + + +/** + * Create a string hash map sorter object. + */ +KCMAPSORT* kcmapsorter(KCMAP* map) { + _assert_(map); + TinyHashMap* thm = (TinyHashMap*)map; + return (KCMAPSORT*)new TinyHashMap::Sorter(thm); +} + + +/** + * Destroy an sorter object. + */ +void kcmapsortdel(KCMAPSORT* sort) { + _assert_(sort); + TinyHashMap::Sorter* thms = (TinyHashMap::Sorter*)sort; + delete thms; +} + + +/** + * Get the key of the current record. + */ +const char* kcmapsortgetkey(KCMAPSORT* sort, size_t* sp) { + _assert_(sort && sp); + TinyHashMap::Sorter* thms = (TinyHashMap::Sorter*)sort; + return thms->get_key(sp); +} + + +/** + * Get the value of the current record. + */ +const char* kcmapsortgetvalue(KCMAPSORT* sort, size_t* sp) { + _assert_(sort && sp); + TinyHashMap::Sorter* thms = (TinyHashMap::Sorter*)sort; + return thms->get_value(sp); +} + + +/** + * Get a pair of the key and the value of the current record. + */ +const char* kcmapsortget(KCMAPSORT* sort, size_t* ksp, const char** vbp, size_t* vsp) { + _assert_(sort && ksp && vbp && vsp); + TinyHashMap::Sorter* thms = (TinyHashMap::Sorter*)sort; + return thms->get(ksp, vbp, vsp); +} + + +/** + * Step the cursor to the next record. + */ +void kcmapsortstep(KCMAPSORT* sort) { + _assert_(sort); + TinyHashMap::Sorter* thms = (TinyHashMap::Sorter*)sort; + return thms->step(); +} + + +/** + * Create a string array list object. + */ +KCLIST* kclistnew() { + _assert_(true); + return (KCLIST*)new TinyArrayList(); +} + + +/** + * Destroy a list object. + */ +void kclistdel(KCLIST* list) { + _assert_(list); + TinyArrayList* tal = (TinyArrayList*)list; + delete tal; +} + + +/** + * Insert a record at the bottom of the list. + */ +void kclistpush(KCLIST* list, const char* buf, size_t size) { + _assert_(list && buf && size <= MEMMAXSIZ); + TinyArrayList* tal = (TinyArrayList*)list; + tal->push(buf, size); +} + + +/** + * Remove a record at the bottom of the list. + */ +int32_t kclistpop(KCLIST* list) { + _assert_(list); + TinyArrayList* tal = (TinyArrayList*)list; + return tal->pop(); +} + + +/** + * Insert a record at the top of the list. + */ +void kclistunshift(KCLIST* list, const char* buf, size_t size) { + _assert_(list && buf && size <= MEMMAXSIZ); + TinyArrayList* tal = (TinyArrayList*)list; + tal->unshift(buf, size); +} + + +/** + * Remove a record at the top of the list. + */ +int32_t kclistshift(KCLIST* list) { + _assert_(list); + TinyArrayList* tal = (TinyArrayList*)list; + return tal->shift(); +} + + +/** + * Insert a record at the position of the given index of the list. + */ +void kclistinsert(KCLIST* list, const char* buf, size_t size, size_t idx) { + _assert_(list && buf && size <= MEMMAXSIZ); + TinyArrayList* tal = (TinyArrayList*)list; + tal->insert(buf, size, idx); +} + + +/** + * Remove a record at the position of the given index of the list. + */ +void kclistremove(KCLIST* list, size_t idx) { + _assert_(list); + TinyArrayList* tal = (TinyArrayList*)list; + return tal->remove(idx); +} + + +/** + * Retrieve a record at the position of the given index of the list. + */ +const char* kclistget(KCLIST* list, size_t idx, size_t* sp) { + _assert_(list && sp); + TinyArrayList* tal = (TinyArrayList*)list; + return tal->get(idx, sp); +} + + +/** + * Remove all records. + */ +void kclistclear(KCLIST* list) { + _assert_(list); + TinyArrayList* tal = (TinyArrayList*)list; + tal->clear(); +} + + +/** + * Get the number of records. + */ +size_t kclistcount(KCLIST* list) { + _assert_(list); + TinyArrayList* tal = (TinyArrayList*)list; + return tal->count(); +} + + +} + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kclangc.h b/plugins/Dbx_kyoto/src/kyotocabinet/kclangc.h new file mode 100644 index 0000000000..9432fd523b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kclangc.h @@ -0,0 +1,1620 @@ +/************************************************************************************************* + * C language binding + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCLANGC_H /* duplication check */ +#define _KCLANGC_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#if !defined(__STDC_LIMIT_MACROS) +#define __STDC_LIMIT_MACROS 1 /**< enable limit macros for C++ */ +#endif + +#include <assert.h> +#include <ctype.h> +#include <errno.h> +#include <float.h> +#include <limits.h> +#include <locale.h> +#include <math.h> +#include <setjmp.h> +#include <stdarg.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> +#include <string.h> +#include <time.h> +#include <stdint.h> + + +/** + * C wrapper of polymorphic database. + */ +typedef struct { + void* db; /**< dummy member */ +} KCDB; + + +/** + * C wrapper of polymorphic cursor. + */ +typedef struct { + void* cur; /**< dummy member */ +} KCCUR; + + +/** + * Binary string of byte array. + */ +typedef struct { + char* buf; /**< pointer to the data region */ + size_t size; /**< size of the data region */ +} KCSTR; + + +/** + * Key-Value record. + */ +typedef struct { + KCSTR key; /**< key string */ + KCSTR value; /**< value string */ +} KCREC; + + +/** + * Error codes. + */ +enum { + KCESUCCESS, /**< success */ + KCENOIMPL, /**< not implemented */ + KCEINVALID, /**< invalid operation */ + KCENOREPOS, /**< no repository */ + KCENOPERM, /**< no permission */ + KCEBROKEN, /**< broken file */ + KCEDUPREC, /**< record duplication */ + KCENOREC, /**< no record */ + KCELOGIC, /**< logical inconsistency */ + KCESYSTEM, /**< system error */ + KCEMISC = 15 /**< miscellaneous error */ +}; + + +/** + * Open modes. + */ +enum { + KCOREADER = 1 << 0, /**< open as a reader */ + KCOWRITER = 1 << 1, /**< open as a writer */ + KCOCREATE = 1 << 2, /**< writer creating */ + KCOTRUNCATE = 1 << 3, /**< writer truncating */ + KCOAUTOTRAN = 1 << 4, /**< auto transaction */ + KCOAUTOSYNC = 1 << 5, /**< auto synchronization */ + KCONOLOCK = 1 << 6, /**< open without locking */ + KCOTRYLOCK = 1 << 7, /**< lock without blocking */ + KCONOREPAIR = 1 << 8 /**< open without auto repair */ +}; + + +/** + * Merge modes. + */ +enum { + KCMSET, /**< overwrite the existing value */ + KCMADD, /**< keep the existing value */ + KCMREPLACE, /**< modify the existing record only */ + KCMAPPEND /**< append the new value */ +}; + + +/** The package version. */ +extern const char* const KCVERSION; + + +/** Special pointer for no operation by the visiting function. */ +extern const char* const KCVISNOP; + + +/** Special pointer to remove the record by the visiting function. */ +extern const char* const KCVISREMOVE; + + +/** + * Call back function to visit a full record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param opq an opaque pointer. + * @return If it is the pointer to a region, the value is replaced by the content. If it + * is KCVISNOP, nothing is modified. If it is KCVISREMOVE, the record is removed. + */ +typedef const char* (*KCVISITFULL)(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp, void* opq); + + +/** + * Call back function to visit an empty record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param opq an opaque pointer. + * @return If it is the pointer to a region, the value is replaced by the content. If it + * is KCVISNOP or KCVISREMOVE, nothing is modified. + */ +typedef const char* (*KCVISITEMPTY)(const char* kbuf, size_t ksiz, size_t* sp, void* opq); + + +/** + * Call back function to process the database file. + * @param path the path of the database file. + * @param count the number of records. + * @param size the size of the available region. + * @param opq an opaque pointer. + * @return true on success, or false on failure. + */ +typedef int32_t (*KCFILEPROC)(const char* path, int64_t count, int64_t size, void* opq); + + +/** + * Allocate a region on memory. + * @param size the size of the region. + * @return the pointer to the allocated region. The region of the return value should be + * released with the kcfree function when it is no longer in use. + */ +void* kcmalloc(size_t size); + + +/** + * Release a region allocated in the library. + * @param ptr the pointer to the region. + */ +void kcfree(void* ptr); + + +/** + * Get the time of day in seconds. + * @return the time of day in seconds. The accuracy is in microseconds. + */ +double kctime(void); + + +/** + * Convert a string to an integer. + * @param str specifies the string. + * @return the integer. If the string does not contain numeric expression, 0 is returned. + */ +int64_t kcatoi(const char* str); + + +/** + * Convert a string with a metric prefix to an integer. + * @param str the string, which can be trailed by a binary metric prefix. "K", "M", "G", "T", + * "P", and "E" are supported. They are case-insensitive. + * @return the integer. If the string does not contain numeric expression, 0 is returned. If + * the integer overflows the domain, INT64_MAX or INT64_MIN is returned according to the + * sign. + */ +int64_t kcatoix(const char* str); + + +/** + * Convert a string to a real number. + * @param str specifies the string. + * @return the real number. If the string does not contain numeric expression, 0.0 is + * returned. + */ +double kcatof(const char* str); + + +/** + * Get the hash value by MurMur hashing. + * @param buf the source buffer. + * @param size the size of the source buffer. + * @return the hash value. + */ +uint64_t kchashmurmur(const void* buf, size_t size); + + +/** + * Get the hash value by FNV hashing. + * @param buf the source buffer. + * @param size the size of the source buffer. + * @return the hash value. + */ +uint64_t kchashfnv(const void* buf, size_t size); + + +/** + * Calculate the levenshtein distance of two regions. + * @param abuf the pointer to the region of one buffer. + * @param asiz the size of the region of one buffer. + * @param bbuf the pointer to the region of the other buffer. + * @param bsiz the size of the region of the other buffer. + * @param utf flag to treat keys as UTF-8 strings. + * @return the levenshtein distance of two regions. + */ +size_t kclevdist(const void* abuf, size_t asiz, const void* bbuf, size_t bsiz, int32_t utf); + + +/** + * Get the quiet Not-a-Number value. + * @return the quiet Not-a-Number value. + */ +double kcnan(); + + +/** + * Get the positive infinity value. + * @return the positive infinity value. + */ +double kcinf(); + + +/** + * Check a number is a Not-a-Number value. + * @return true for the number is a Not-a-Number value, or false if not. + */ +int32_t kcchknan(double num); + + +/** + * Check a number is an infinity value. + * @return true for the number is an infinity value, or false if not. + */ +int32_t kcchkinf(double num); + + +/** + * Get the readable string of an error code. + * @param code the error code. + * @return the readable string of the error code. + */ +const char* kcecodename(int32_t code); + + +/** + * Create a polymorphic database object. + * @return the created database object. + * @note The object of the return value should be released with the kcdbdel function when it is + * no longer in use. + */ +KCDB* kcdbnew(void); + + +/** + * Destroy a database object. + * @param db the database object. + */ +void kcdbdel(KCDB* db); + + +/** + * Open a database file. + * @param db a database object. + * @param path the path of a database file. If it is "-", the database will be a prototype + * hash database. If it is "+", the database will be a prototype tree database. If it is ":", + * the database will be a stash database. If it is "*", the database will be a cache hash + * database. If it is "%", the database will be a cache tree database. If its suffix is + * ".kch", the database will be a file hash database. If its suffix is ".kct", the database + * will be a file tree database. If its suffix is ".kcd", the database will be a directory + * hash database. If its suffix is ".kcf", the database will be a directory tree database. + * If its suffix is ".kcx", the database will be a plain text database. Otherwise, this + * function fails. Tuning parameters can trail the name, separated by "#". Each parameter is + * composed of the name and the value, separated by "=". If the "type" parameter is specified, + * the database type is determined by the value in "-", "+", ":", "*", "%", "kch", "kct", + * "kcd", kcf", and "kcx". All database types support the logging parameters of "log", + * "logkinds", and "logpx". The prototype hash database and the prototype tree database do + * not support any other tuning parameter. The stash database supports "bnum". The cache + * hash database supports "opts", "bnum", "zcomp", "capcnt", "capsiz", and "zkey". The cache + * tree database supports all parameters of the cache hash database except for capacity + * limitation, and supports "psiz", "rcomp", "pccap" in addition. The file hash database + * supports "apow", "fpow", "opts", "bnum", "msiz", "dfunit", "zcomp", and "zkey". The file + * tree database supports all parameters of the file hash database and "psiz", "rcomp", + * "pccap" in addition. The directory hash database supports "opts", "zcomp", and "zkey". + * The directory tree database supports all parameters of the directory hash database and + * "psiz", "rcomp", "pccap" in addition. The plain text database does not support any other + * tuning parameter. + * @param mode the connection mode. KCOWRITER as a writer, KCOREADER as a reader. + * The following may be added to the writer mode by bitwise-or: KCOCREATE, which means + * it creates a new database if the file does not exist, KCOTRUNCATE, which means it + * creates a new database regardless if the file exists, KCOAUTOTRAN, which means each + * updating operation is performed in implicit transaction, KCOAUTOSYNC, which means + * each updating operation is followed by implicit synchronization with the file system. The + * following may be added to both of the reader mode and the writer mode by bitwise-or: + * KCONOLOCK, which means it opens the database file without file locking, + * KCOTRYLOCK, which means locking is performed without blocking, KCONOREPAIR, which + * means the database file is not repaired implicitly even if file destruction is detected. + * @return true on success, or false on failure. + * @note The tuning parameter "log" is for the original "tune_logger" and the value specifies + * the path of the log file, or "-" for the standard output, or "+" for the standard error. + * "logkinds" specifies kinds of logged messages and the value can be "debug", "info", "warn", + * or "error". "logpx" specifies the prefix of each log message. "opts" is for "tune_options" + * and the value can contain "s" for the small option, "l" for the linear option, and "c" for + * the compress option. "bnum" corresponds to "tune_bucket". "zcomp" is for "tune_compressor" + * and the value can be "zlib" for the ZLIB raw compressor, "def" for the ZLIB deflate + * compressor, "gz" for the ZLIB gzip compressor, "lzo" for the LZO compressor, "lzma" for the + * LZMA compressor, or "arc" for the Arcfour cipher. "zkey" specifies the cipher key of the + * compressor. "capcount" is for "cap_count". "capsize" is for "cap_size". "psiz" is for + * "tune_page". "rcomp" is for "tune_comparator" and the value can be "lex" for the lexical + * comparator or "dec" for the decimal comparator. "pccap" is for "tune_page_cache". "apow" + * is for "tune_alignment". "fpow" is for "tune_fbp". "msiz" is for "tune_map". "dfunit" is + * for "tune_defrag". Every opened database must be closed by the kcdbclose method when it is + * no longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ +int32_t kcdbopen(KCDB* db, const char* path, uint32_t mode); + + +/** + * Close the database file. + * @param db a database object. + * @return true on success, or false on failure. + */ +int32_t kcdbclose(KCDB* db); + + +/** + * Get the code of the last happened error. + * @param db a database object. + * @return the code of the last happened error. + */ +int32_t kcdbecode(KCDB* db); + + +/** + * Get the supplement message of the last happened error. + * @param db a database object. + * @return the supplement message of the last happened error. + */ +const char* kcdbemsg(KCDB* db); + + +/** + * Accept a visitor to a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param fullproc a call back function to visit a record. + * @param emptyproc a call back function to visit an empty record space. + * @param opq an opaque pointer to be given to the call back functions. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ +int32_t kcdbaccept(KCDB* db, const char* kbuf, size_t ksiz, + KCVISITFULL fullproc, KCVISITEMPTY emptyproc, void* opq, int32_t writable); + + +/** + * Accept a visitor to multiple records at once. + * @param db a database object. + * @param keys specifies an array of binary strings of the keys. + * @param knum specifies the number of the keys. + * @param fullproc a call back function to visit a record. + * @param emptyproc a call back function to visit an empty record space. + * @param opq an opaque pointer to be given to the call back functions. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ +int32_t kcdbacceptbulk(KCDB* db, const KCSTR* keys, size_t knum, + KCVISITFULL fullproc, KCVISITEMPTY emptyproc, + void* opq, int32_t writable); + + +/** + * Iterate to accept a visitor for each record. + * @param db a database object. + * @param fullproc a call back function to visit a record. + * @param opq an opaque pointer to be given to the call back function. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ +int32_t kcdbiterate(KCDB* db, KCVISITFULL fullproc, void* opq, int32_t writable); + + +/** + * Scan each record in parallel. + * @param db a database object. + * @param fullproc a call back function to visit a record. + * @param opq an opaque pointer to be given to the call back function. + * @param thnum the number of worker threads. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ +int32_t kcdbscanpara(KCDB* db, KCVISITFULL fullproc, void* opq, size_t thnum); + + +/** + * Set the value of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the value is overwritten. + */ +int32_t kcdbset(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Add a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the record is not modified and false is returned. + */ +int32_t kcdbadd(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Replace the value of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, no new record is created and false is returned. + * If the corresponding record exists, the value is modified. + */ +int32_t kcdbreplace(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Append the value of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the given value is appended at the end of the existing value. + */ +int32_t kcdbappend(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Add a number to the numeric value of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param num the additional number. + * @param orig the origin number if no record corresponds to the key. If it is INT64_MIN and + * no record corresponds, this function fails. If it is INT64_MAX, the value is set as the + * additional number regardless of the current value. + * @return the result value, or INT64_MIN on failure. + * @note The value is serialized as an 8-byte binary integer in big-endian order, not a decimal + * string. If existing value is not 8-byte, this function fails. + */ +int64_t kcdbincrint(KCDB* db, const char* kbuf, size_t ksiz, int64_t num, int64_t orig); + + +/** + * Add a number to the numeric value of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param num the additional number. + * @param orig the origin number if no record corresponds to the key. If it is negative + * infinity and no record corresponds, this function fails. If it is positive infinity, the + * value is set as the additional number regardless of the current value. + * @return the result value, or Not-a-number on failure. + * @note The value is serialized as an 16-byte binary fixed-point number in big-endian order, + * not a decimal string. If existing value is not 16-byte, this function fails. + */ +double kcdbincrdouble(KCDB* db, const char* kbuf, size_t ksiz, double num, double orig); + + +/** + * Perform compare-and-swap. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param ovbuf the pointer to the old value region. NULL means that no record corresponds. + * @param ovsiz the size of the old value region. + * @param nvbuf the pointer to the new value region. NULL means that the record is removed. + * @param nvsiz the size of new old value region. + * @return true on success, or false on failure. + */ +int32_t kcdbcas(KCDB* db, const char* kbuf, size_t ksiz, + const char* ovbuf, size_t ovsiz, const char* nvbuf, size_t nvsiz); + + +/** + * Remove a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. + */ +int32_t kcdbremove(KCDB* db, const char* kbuf, size_t ksiz); + + +/** + * Retrieve the value of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + * @note If no record corresponds to the key, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. The region of the return value should be released with the + * kcfree function when it is no longer in use. + */ +char* kcdbget(KCDB* db, const char* kbuf, size_t ksiz, size_t* sp); + + +/** + * Check the existence of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the size of the value, or -1 on failure. + */ +int32_t kcdbcheck(KCDB* db, const char* kbuf, size_t ksiz); + + +/** + * Retrieve the value of a record. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the buffer into which the value of the corresponding record is + * written. + * @param max the size of the buffer. + * @return the size of the value, or -1 on failure. + */ +int32_t kcdbgetbuf(KCDB* db, const char* kbuf, size_t ksiz, char* vbuf, size_t max); + + +/** + * Retrieve the value of a record and remove it atomically. + * @param db a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + * @note If no record corresponds to the key, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. The region of the return value should be released with the + * kcfree function when it is no longer in use. + */ +char* kcdbseize(KCDB* db, const char* kbuf, size_t ksiz, size_t* sp); + + +/** + * Store records at once. + * @param db a database object. + * @param recs the records to store. + * @param rnum specifies the number of the records. + * @param atomic true to perform all operations atomically, or false for non-atomic operations. + * @return the number of stored records, or -1 on failure. + */ +int64_t kcdbsetbulk(KCDB* db, const KCREC* recs, size_t rnum, int32_t atomic); + + +/** + * Remove records at once. + * @param db a database object. + * @param keys the keys of the records to remove. + * @param knum specifies the number of the keys. + * @param atomic true to perform all operations atomically, or false for non-atomic operations. + * @return the number of removed records, or -1 on failure. + */ +int64_t kcdbremovebulk(KCDB* db, const KCSTR* keys, size_t knum, int32_t atomic); + + +/** + * Retrieve records at once. + * @param db a database object. + * @param keys the keys of the records to retrieve. + * @param knum specifies the number of the keys. + * @param recs an array to contain the result. Its size must be sufficient. + * @param atomic true to perform all operations atomically, or false for non-atomic operations. + * @return the number of retrieved records, or -1 on failure. + * @note The regions of the key and the value of each element of the result should be released + * with the kcfree function when it is no longer in use. + */ +int64_t kcdbgetbulk(KCDB* db, const KCSTR* keys, size_t knum, KCREC* recs, int32_t atomic); + + +/** + * Synchronize updated contents with the file and the device. + * @param db a database object. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor call back function. If it is NULL, no postprocessing is + * performed. + * @param opq an opaque pointer to be given to the call back function. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ +int32_t kcdbsync(KCDB* db, int32_t hard, KCFILEPROC proc, void* opq); + + +/** + * Occupy database by locking and do something meanwhile. + * @param db a database object. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @param opq an opaque pointer to be given to the call back function. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ +int32_t kcdboccupy(KCDB* db, int32_t writable, KCFILEPROC proc, void* opq); + + +/** + * Create a copy of the database file. + * @param db a database object. + * @param dest the path of the destination file. + * @return true on success, or false on failure. + */ +int32_t kcdbcopy(KCDB* db, const char* dest); + + +/** + * Begin transaction. + * @param db a database object. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ +int32_t kcdbbegintran(KCDB* db, int32_t hard); + + +/** + * Try to begin transaction. + * @param db a database object. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ +int32_t kcdbbegintrantry(KCDB* db, int32_t hard); + + +/** + * End transaction. + * @param db a database object. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ +int32_t kcdbendtran(KCDB* db, int32_t commit); + + +/** + * Remove all records. + * @param db a database object. + * @return true on success, or false on failure. + */ +int32_t kcdbclear(KCDB* db); + + +/** + * Dump records into a file. + * @param db a database object. + * @param dest the path of the destination file. + * @return true on success, or false on failure. + */ +int32_t kcdbdumpsnap(KCDB* db, const char* dest); + + +/** + * Load records from a file. + * @param db a database object. + * @param src the path of the source file. + * @return true on success, or false on failure. + */ +int32_t kcdbloadsnap(KCDB* db, const char* src); + + +/** + * Get the number of records. + * @param db a database object. + * @return the number of records, or -1 on failure. + */ +int64_t kcdbcount(KCDB* db); + + +/** + * Get the size of the database file. + * @param db a database object. + * @return the size of the database file in bytes, or -1 on failure. + */ +int64_t kcdbsize(KCDB* db); + + +/** + * Get the path of the database file. + * @param db a database object. + * @return the path of the database file, or an empty string on failure. + * @note The region of the return value should be released with the kcfree function when it is + * no longer in use. + */ +char* kcdbpath(KCDB* db); + + +/** + * Get the miscellaneous status information. + * @param db a database object. + * @return the result string of tab saparated values, or NULL on failure. Each line consists of + * the attribute name and its value separated by a tab character. + * @note The region of the return value should be released with the kcfree function when it is + * no longer in use. + */ +char* kcdbstatus(KCDB* db); + + +/** + * Get keys matching a prefix string. + * @param db a database object. + * @param prefix the prefix string. + * @param strary an array to contain the result. Its size must be sufficient. + * @param max the maximum number to retrieve. + * @return the number of retrieved keys or -1 on failure. + * @note The region of each element of the result should be released with the kcfree function + * when it is no longer in use. + */ +int64_t kcdbmatchprefix(KCDB* db, const char* prefix, char** strary, size_t max); + + +/** + * Get keys matching a regular expression string. + * @param db a database object. + * @param regex the regular expression string. + * @param strary an array to contain the result. Its size must be sufficient. + * @param max the maximum number to retrieve. + * @return the number of retrieved keys or -1 on failure. + * @note The region of each element of the result should be released with the kcfree function + * when it is no longer in use. + */ +int64_t kcdbmatchregex(KCDB* db, const char* regex, char** strary, size_t max); + + +/** + * Get keys similar to a string in terms of the levenshtein distance. + * @param db a database object. + * @param origin the origin string. + * @param range the maximum distance of keys to adopt. + * @param utf flag to treat keys as UTF-8 strings. + * @param strary an array to contain the result. Its size must be sufficient. + * @param max the maximum number to retrieve. + * @return the number of retrieved keys or -1 on failure. + * @note The region of each element of the result should be released with the kcfree function + * when it is no longer in use. + */ +int64_t kcdbmatchsimilar(KCDB* db, const char* origin, uint32_t range, int32_t utf, + char** strary, size_t max); + + +/** + * Merge records from other databases. + * @param db a database object. + * @param srcary an array of the source detabase objects. + * @param srcnum the number of the elements of the source array. + * @param mode the merge mode. KCMSET to overwrite the existing value, KCMADD to keep the + * existing value, KCMREPLACE to modify the existing record only, KCMAPPEND to append the new + * value. + * @return true on success, or false on failure. + */ +int32_t kcdbmerge(KCDB* db, KCDB** srcary, size_t srcnum, uint32_t mode); + + +/** + * Create a polymorphic cursor object. + * @param db a database object. + * @return the return value is the created cursor object. + * @note The object of the return value should be released with the kccurdel function when it is + * no longer in use. + */ +KCCUR* kcdbcursor(KCDB* db); + + +/** + * Destroy a cursor object. + * @param cur the cursor object. + */ +void kccurdel(KCCUR* cur); + + +/** + * Accept a visitor to the current record. + * @param cur a cursor object. + * @param fullproc a call back function to visit a record. + * @param opq an opaque pointer to be given to the call back functions. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ +int32_t kccuraccept(KCCUR* cur, KCVISITFULL fullproc, void* opq, + int32_t writable, int32_t step); + + +/** + * Set the value of the current record. + * @param cur a cursor object. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + */ +int32_t kccursetvalue(KCCUR* cur, const char* vbuf, size_t vsiz, int32_t step); + + +/** + * Remove the current record. + * @param cur a cursor object. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. The cursor is moved to the + * next record implicitly. + */ +int32_t kccurremove(KCCUR* cur); + + +/** + * Get the key of the current record. + * @param cur a cursor object. + * @param sp the pointer to the variable into which the size of the region of the return value + * is assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the key region of the current record, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. The region of the return value should be released with the + * kcfree function when it is no longer in use. + */ +char* kccurgetkey(KCCUR* cur, size_t* sp, int32_t step); + + +/** + * Get the value of the current record. + * @param cur a cursor object. + * @param sp the pointer to the variable into which the size of the region of the return value + * is assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the value region of the current record, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. The region of the return value should be released with the + * kcfree function when it is no longer in use. + */ +char* kccurgetvalue(KCCUR* cur, size_t* sp, int32_t step); + + +/** + * Get a pair of the key and the value of the current record. + * @param cur a cursor object. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @param step true to move the cursor to the next record, or false for no move. + * @return the pointer to the pair of the key region, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero code is + * appended at the end of each region of the key and the value, each region can be treated + * as a C-style string. The region of the return value should be released with the kcfree + * function when it is no longer in use. + */ +char* kccurget(KCCUR* cur, size_t* ksp, const char** vbp, size_t* vsp, int32_t step); + + +/** + * Get a pair of the key and the value of the current record and remove it atomically. + * @param cur a cursor object. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @return the pointer to the pair of the key region, or NULL on failure. + * @note If the cursor is invalidated, NULL is returned. Because an additional zero code is + * appended at the end of each region of the key and the value, each region can be treated + * as a C-style string. The region of the return value should be released with the kcfree + * function when it is no longer in use. The cursor is moved to the next record implicitly. + */ +char* kccurseize(KCCUR* cur, size_t* ksp, const char** vbp, size_t* vsp); + + +/** + * Jump the cursor to the first record for forward scan. + * @param cur a cursor object. + * @return true on success, or false on failure. + */ +int32_t kccurjump(KCCUR* cur); + + +/** + * Jump the cursor to a record for forward scan. + * @param cur a cursor object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ +int32_t kccurjumpkey(KCCUR* cur, const char* kbuf, size_t ksiz); + + +/** + * Jump the cursor to the last record for backward scan. + * @param cur a cursor object. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, may provide a dummy implementation. + */ +int32_t kccurjumpback(KCCUR* cur); + + +/** + * Jump the cursor to a record for backward scan. + * @param cur a cursor object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, will provide a dummy implementation. + */ +int32_t kccurjumpbackkey(KCCUR* cur, const char* kbuf, size_t ksiz); + + +/** + * Step the cursor to the next record. + * @param cur a cursor object. + * @return true on success, or false on failure. + */ +int32_t kccurstep(KCCUR* cur); + + +/** + * Step the cursor to the previous record. + * @param cur a cursor object. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, may provide a dummy implementation. + */ +int32_t kccurstepback(KCCUR* cur); + + +/** + * Get the database object. + * @param cur a cursor object. + * @return the database object. + */ +KCDB* kccurdb(KCCUR* cur); + + +/** + * Get the code of the last happened error. + * @param cur a cursor object. + * @return the code of the last happened error. + */ +int32_t kccurecode(KCCUR* cur); + + +/** + * Get the supplement message of the last happened error. + * @param cur a cursor object. + * @return the supplement message of the last happened error. + */ +const char* kccuremsg(KCCUR* cur); + + +/** + * C wrapper of index database. + */ +typedef struct { + void* db; /**< dummy member */ +} KCIDX; + + +/** + * Create an index database object. + * @return the created database object. + * @note The object of the return value should be released with the kcidxdel function when it is + * no longer in use. + */ +KCIDX* kcidxnew(void); + + +/** + * Destroy a database object. + * @param idx the database object. + */ +void kcidxdel(KCIDX* idx); + + +/** + * Open a database file. + * @param idx a database object. + * @param path the path of a database file. The same as with the polymorphic database. + * @param mode the connection mode. The same as with the polymorphic database. + * @return true on success, or false on failure. + */ +int32_t kcidxopen(KCIDX* idx, const char* path, uint32_t mode); + + +/** + * Close the database file. + * @param idx a database object. + * @return true on success, or false on failure. + */ +int32_t kcidxclose(KCIDX* idx); + + +/** + * Get the code of the last happened error. + * @param idx a database object. + * @return the code of the last happened error. + */ +int32_t kcidxecode(KCIDX* idx); + + +/** + * Get the supplement message of the last happened error. + * @param idx a database object. + * @return the supplement message of the last happened error. + */ +const char* kcidxemsg(KCIDX* idx); + + +/** + * Set the value of a record. + * @param idx a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the value is overwritten. + */ +int32_t kcidxset(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Add a record. + * @param idx a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the record is not modified and false is returned. + */ +int32_t kcidxadd(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Replace the value of a record. + * @param idx a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, no new record is created and false is returned. + * If the corresponding record exists, the value is modified. + */ +int32_t kcidxreplace(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Append the value of a record. + * @param idx a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the given value is appended at the end of the existing value. + */ +int32_t kcidxappend(KCIDX* idx, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Remove a record. + * @param idx a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. + */ +int32_t kcidxremove(KCIDX* idx, const char* kbuf, size_t ksiz); + + +/** + * Retrieve the value of a record. + * @param idx a database object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + * @note If no record corresponds to the key, NULL is returned. Because an additional zero + * code is appended at the end of the region of the return value, the return value can be + * treated as a C-style string. The region of the return value should be released with the + * kcfree function when it is no longer in use. + */ +char* kcidxget(KCIDX* idx, const char* kbuf, size_t ksiz, size_t* sp); + + +/** + * Synchronize updated contents with the file and the device. + * @param idx a database object. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor call back function. If it is NULL, no postprocessing is + * performed. + * @param opq an opaque pointer to be given to the call back function. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ +int32_t kcidxsync(KCIDX* idx, int32_t hard, KCFILEPROC proc, void* opq); + + +/** + * Remove all records. + * @param idx a database object. + * @return true on success, or false on failure. + */ +int32_t kcidxclear(KCIDX* idx); + + +/** + * Get the number of records. + * @param idx a database object. + * @return the number of records, or -1 on failure. + */ +int64_t kcidxcount(KCIDX* idx); + + +/** + * Get the size of the database file. + * @param idx a database object. + * @return the size of the database file in bytes, or -1 on failure. + */ +int64_t kcidxsize(KCIDX* idx); + + +/** + * Get the path of the database file. + * @param idx a database object. + * @return the path of the database file, or an empty string on failure. + * @note The region of the return value should be released with the kcfree function when it is + * no longer in use. + */ +char* kcidxpath(KCIDX* idx); + + +/** + * Get the miscellaneous status information. + * @param idx a database object. + * @return the result string of tab saparated values, or NULL on failure. Each line consists of + * the attribute name and its value separated by a tab character. + * @note The region of the return value should be released with the kcfree function when it is + * no longer in use. + */ +char* kcidxstatus(KCIDX* idx); + + +/** + * Reveal the inner database object. + * @return the inner database object, or NULL on failure. + */ +KCDB* kcidxrevealinnerdb(KCIDX* idx); + + +/** + * C wrapper of memory-saving string hash map. + */ +typedef struct { + void* map; /**< dummy member */ +} KCMAP; + + +/** + * C wrapper of iterator of memory-saving string hash map. + */ +typedef struct { + void* iter; /**< dummy member */ +} KCMAPITER; + + +/** + * C wrapper of sorter of memory-saving string hash map. + */ +typedef struct { + void* iter; /**< dummy member */ +} KCMAPSORT; + + +/** + * Create a string hash map object. + * @param bnum the number of buckets of the hash table. If it is not more than 0, the default + * setting 31 is specified. + * @return the created map object. + * @note The object of the return value should be released with the kcmapdel function when it is + * no longer in use. + */ +KCMAP* kcmapnew(size_t bnum); + + +/** + * Destroy a map object. + * @param map the map object. + */ +void kcmapdel(KCMAP* map); + + +/** + * Set the value of a record. + * @param map the map object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the value is overwritten. + */ +void kcmapset(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Add a record. + * @param map the map object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the record is not modified and false is returned. + */ +int32_t kcmapadd(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Replace the value of a record. + * @param map the map object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, no new record is created and false is returned. + * If the corresponding record exists, the value is modified. + */ +int32_t kcmapreplace(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Append the value of a record. + * @param map the map object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the given value is appended at the end of the existing value. + */ +void kcmapappend(KCMAP* map, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); + + +/** + * Remove a record. + * @param map the map object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. + */ +int32_t kcmapremove(KCMAP* map, const char* kbuf, size_t ksiz); + + +/** + * Retrieve the value of a record. + * @param map the map object. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + */ +const char* kcmapget(KCMAP* map, const char* kbuf, size_t ksiz, size_t* sp); + + +/** + * Remove all records. + * @param map the map object. + */ +void kcmapclear(KCMAP* map); + + +/** + * Get the number of records. + * @param map the map object. + * @return the number of records. + */ +size_t kcmapcount(KCMAP* map); + + +/** + * Create a string hash map iterator object. + * @param map a map object. + * @return the return value is the created iterator object. + * @note The object of the return value should be released with the kcmapiterdel function when + * it is no longer in use. + * @note This object will not be invalidated even when the map object is updated once. + * However, phantom records may be retrieved if they are removed after creation of each iterator. + */ +KCMAPITER* kcmapiterator(KCMAP* map); + + +/** + * Destroy an iterator object. + * @param iter the iterator object. + */ +void kcmapiterdel(KCMAPITER* iter); + + +/** + * Get the key of the current record. + * @param iter the iterator object. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the key region of the current record, or NULL on failure. + */ +const char* kcmapitergetkey(KCMAPITER* iter, size_t* sp); + + +/** + * Get the value of the current record. + * @param iter the iterator object. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the current record, or NULL on failure. + */ +const char* kcmapitergetvalue(KCMAPITER* iter, size_t* sp); + + +/** + * Get a pair of the key and the value of the current record. + * @param iter the iterator object. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @return the pointer to the key region, or NULL on failure. + */ +const char* kcmapiterget(KCMAPITER* iter, size_t* ksp, const char** vbp, size_t* vsp); + + +/** + * Step the cursor to the next record. + * @param iter the iterator object. + */ +void kcmapiterstep(KCMAPITER* iter); + + +/** + * Create a string hash map sorter object. + * @param map a map object. + * @return the return value is the created sorter object. + * @note The object of the return value should be released with the kcmapsortdel function when + * it is no longer in use. + * @note This object will not be invalidated even when the map object is updated once. + * However, phantom records may be retrieved if they are removed after creation of each sorter. + */ +KCMAPSORT* kcmapsorter(KCMAP* map); + + +/** + * Destroy an sorter object. + * @param sort the sorter object. + */ +void kcmapsortdel(KCMAPSORT* sort); + + +/** + * Get the key of the current record. + * @param sort the sorter object. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the key region of the current record, or NULL on failure. + */ +const char* kcmapsortgetkey(KCMAPSORT* sort, size_t* sp); + + +/** + * Get the value of the current record. + * @param sort the sorter object. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the current record, or NULL on failure. + */ +const char* kcmapsortgetvalue(KCMAPSORT* sort, size_t* sp); + + +/** + * Get a pair of the key and the value of the current record. + * @param sort the sorter object. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @return the pointer to the key region, or NULL on failure. + */ +const char* kcmapsortget(KCMAPSORT* sort, size_t* ksp, const char** vbp, size_t* vsp); + + +/** + * Step the cursor to the next record. + * @param sort the sorter object. + */ +void kcmapsortstep(KCMAPSORT* sort); + + +/** + * C wrapper of memory-saving string hash map. + */ +typedef struct { + void* list; /**< dummy member */ +} KCLIST; + + +/** + * Create a string array list object. + * @return the created list object. + * @note The object of the return value should be released with the kclistdel function when it is + * no longer in use. + */ +KCLIST* kclistnew(); + + +/** + * Destroy a list object. + * @param list the list object. + */ +void kclistdel(KCLIST* list); + + +/** + * Insert a record at the bottom of the list. + * @param list the list object. + * @param buf the pointer to the record region. + * @param size the size of the record region. + */ +void kclistpush(KCLIST* list, const char* buf, size_t size); + + +/** + * Remove a record at the bottom of the list. + * @param list the list object. + * @return true if the operation success, or false if there is no record in the list. + */ +int32_t kclistpop(KCLIST* list); + + +/** + * Insert a record at the top of the list. + * @param list the list object. + * @param buf the pointer to the record region. + * @param size the size of the record region. + */ +void kclistunshift(KCLIST* list, const char* buf, size_t size); + + +/** + * Remove a record at the top of the list. + * @param list the list object. + * @return true if the operation success, or false if there is no record in the list. + */ +int32_t kclistshift(KCLIST* list); + + +/** + * Insert a record at the position of the given index of the list. + * @param list the list object. + * @param buf the pointer to the record region. + * @param size the size of the record region. + * @param idx the index of the position. It must be equal to or less than the number of + * records. + */ +void kclistinsert(KCLIST* list, const char* buf, size_t size, size_t idx); + + +/** + * Remove a record at the position of the given index of the list. + * @param list the list object. + * @param idx the index of the position. It must be less than the number of records. + */ +void kclistremove(KCLIST* list, size_t idx); + + +/** + * Retrieve a record at the position of the given index of the list. + * @param list the list object. + * @param idx the index of the position. It must be less than the number of records. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the region of the retrieved record. + */ +const char* kclistget(KCLIST* list, size_t idx, size_t* sp); + + +/** + * Remove all records. + * @param list the list object. + */ +void kclistclear(KCLIST* list); + + +/** + * Get the number of records. + * @param list the list object. + * @return the number of records. + */ +size_t kclistcount(KCLIST* list); + + +#if defined(__cplusplus) +} +#endif + +#endif /* duplication check */ + +/* END OF FILE */ diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kclangctest.c b/plugins/Dbx_kyoto/src/kyotocabinet/kclangctest.c new file mode 100644 index 0000000000..c123346683 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kclangctest.c @@ -0,0 +1,1189 @@ +/************************************************************************************************* + * The test cases of the C language binding + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kclangc.h> + +#define RECBUFSIZ 64 /* buffer size for a record */ +#define RECBUFSIZL 1024 /* buffer size for a long record */ +#if !defined(TRUE) +#define TRUE 1 /* boolean true */ +#endif +#if !defined(FALSE) +#define FALSE 0 /* boolean false */ +#endif + +typedef struct { /* arguments of visitor */ + int64_t rnum; + int32_t rnd; + int64_t cnt; + char rbuf[RECBUFSIZ]; +} VISARG; + + +/* global variables */ +const char* g_progname; /* program name */ + + +/* function prototypes */ +int main(int argc, char** argv); +static void usage(void); +static int64_t myrand(int64_t range); +static void oprintf(const char* format, ...); +static void oputchar(char c); +static void eprintf(const char* format, ...); +static void dberrprint(KCDB* db, int32_t line, const char* func); +static void idxerrprint(KCIDX* idx, int32_t line, const char* func); +static void dbmetaprint(KCDB* db, int32_t verbose); +static void idxmetaprint(KCIDX* idx, int32_t verbose); +const char* visitfull(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp, void* opq); +static int32_t runorder(int argc, char** argv); +static int32_t runindex(int argc, char** argv); +static int32_t runmap(int argc, char** argv); +static int32_t runlist(int argc, char** argv); +static int32_t procorder(const char* path, int64_t rnum, int32_t rnd, int32_t etc, + int32_t tran, int32_t oflags); +static int32_t procindex(const char* path, int64_t rnum, int32_t rnd, int32_t etc, + int32_t oflags); +static int32_t procmap(int64_t rnum, int32_t rnd, int32_t etc, int64_t bnum); +static int32_t proclist(int64_t rnum, int32_t rnd, int32_t etc); + + +/* main routine */ +int main(int argc, char **argv) { + int32_t i, rv; + g_progname = argv[0]; + srand(time(NULL)); + if (argc < 2) usage(); + rv = 0; + if (!strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!strcmp(argv[1], "index")) { + rv = runindex(argc, argv); + } else if (!strcmp(argv[1], "map")) { + rv = runmap(argc, argv); + } else if (!strcmp(argv[1], "list")) { + rv = runlist(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED:"); + for (i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +/* print the usage and exit */ +static void usage() { + eprintf("%s: test cases of the C binding of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-rnd] [-etc] [-tran] [-oat|-oas|-onl|-otl|-onr] path rnum\n", + g_progname); + eprintf(" %s index [-rnd] [-etc] [-oat|-oas|-onl|-otl|-onr] path rnum\n", g_progname); + eprintf(" %s map [-rnd] [-etc] [-bnum num] rnum\n", g_progname); + eprintf(" %s list [-rnd] [-etc] rnum\n", g_progname); + eprintf("\n"); + exit(1); +} + + +/* get a random number */ +static int64_t myrand(int64_t range) { + uint64_t base, mask; + if (range < 2) return 0; + base = range * (rand() / (RAND_MAX + 1.0)); + mask = (uint64_t)rand() << 30; + mask += (uint64_t)rand() >> 2; + return (base ^ mask) % range; +} + + +/* print formatted error string and flush the buffer */ +static void oprintf(const char* format, ...) { + va_list ap; + va_start(ap, format); + vprintf(format, ap); + va_end(ap); + fflush(stdout); +} + + +/* print a character and flush the buffer */ +static void oputchar(char c) { + putchar(c); + fflush(stdout); +} + + +/* print formatted error string and flush the buffer */ +static void eprintf(const char* format, ...) { + va_list ap; + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + fflush(stderr); +} + + +/* print error message of database */ +static void dberrprint(KCDB* db, int32_t line, const char* func) { + char* path; + const char* emsg; + int32_t ecode; + path = kcdbpath(db); + ecode = kcdbecode(db); + emsg = kcdbemsg(db); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, path ? path : "-", ecode, kcecodename(ecode), emsg); + kcfree(path); +} + + +/* print error message of database */ +static void idxerrprint(KCIDX* idx, int32_t line, const char* func) { + dberrprint(kcidxrevealinnerdb(idx), line, func); +} + + +/* print members of database */ +static void dbmetaprint(KCDB* db, int32_t verbose) { + char* status, *rp; + if (verbose) { + status = kcdbstatus(db); + if (status) { + rp = status; + while (*rp != '\0') { + if (*rp == '\t') { + printf(": "); + } else { + putchar(*rp); + } + rp++; + } + kcfree(status); + } + } else { + oprintf("count: %ld\n", (long)kcdbcount(db)); + oprintf("size: %ld\n", (long)kcdbsize(db)); + } +} + + +/* print members of database */ +static void idxmetaprint(KCIDX* idx, int32_t verbose) { + dbmetaprint(kcidxrevealinnerdb(idx), verbose); +} + + +/* visit a full record */ +const char* visitfull(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp, void* opq) { + VISARG* arg; + const char* rv; + arg = opq; + arg->cnt++; + rv = KCVISNOP; + switch (arg->rnd ? myrand(7) : arg->cnt % 7) { + case 0: { + rv = arg->rbuf; + *sp = arg->rnd ? (size_t)myrand(sizeof(arg->rbuf)) : + sizeof(arg->rbuf) / (arg->cnt % 5 + 1); + break; + } + case 1: { + rv = KCVISREMOVE; + break; + } + } + if (arg->rnum > 250 && arg->cnt % (arg->rnum / 250) == 0) { + oputchar('.'); + if (arg->cnt == arg->rnum || arg->cnt % (arg->rnum / 10) == 0) + oprintf(" (%08ld)\n", (long)arg->cnt); + } + return rv; +} + + +/* parse arguments of order command */ +static int32_t runorder(int argc, char** argv) { + int32_t argbrk = FALSE; + const char* path, *rstr; + int32_t rnd, etc, tran, mode, oflags, i; + int64_t rnum; + path = NULL; + rstr = NULL; + rnd = FALSE; + etc = FALSE; + mode = 0; + tran = FALSE; + oflags = 0; + for (i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!strcmp(argv[i], "--")) { + argbrk = TRUE; + } else if (!strcmp(argv[i], "-rnd")) { + rnd = TRUE; + } else if (!strcmp(argv[i], "-etc")) { + etc = TRUE; + } else if (!strcmp(argv[i], "-tran")) { + tran = TRUE; + } else if (!strcmp(argv[i], "-oat")) { + oflags |= KCOAUTOTRAN; + } else if (!strcmp(argv[i], "-oas")) { + oflags |= KCOAUTOSYNC; + } else if (!strcmp(argv[i], "-onl")) { + oflags |= KCONOLOCK; + } else if (!strcmp(argv[i], "-otl")) { + oflags |= KCOTRYLOCK; + } else if (!strcmp(argv[i], "-onr")) { + oflags |= KCONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = TRUE; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + rnum = kcatoix(rstr); + if (rnum < 1) usage(); + return procorder(path, rnum, rnd, etc, tran, oflags); +} + + +/* parse arguments of index command */ +static int32_t runindex(int argc, char** argv) { + int32_t argbrk = FALSE; + const char* path, *rstr; + int32_t rnd, etc, mode, oflags, i; + int64_t rnum; + path = NULL; + rstr = NULL; + rnd = FALSE; + etc = FALSE; + mode = 0; + oflags = 0; + for (i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!strcmp(argv[i], "--")) { + argbrk = TRUE; + } else if (!strcmp(argv[i], "-rnd")) { + rnd = TRUE; + } else if (!strcmp(argv[i], "-etc")) { + etc = TRUE; + } else if (!strcmp(argv[i], "-oat")) { + oflags |= KCOAUTOTRAN; + } else if (!strcmp(argv[i], "-oas")) { + oflags |= KCOAUTOSYNC; + } else if (!strcmp(argv[i], "-onl")) { + oflags |= KCONOLOCK; + } else if (!strcmp(argv[i], "-otl")) { + oflags |= KCOTRYLOCK; + } else if (!strcmp(argv[i], "-onr")) { + oflags |= KCONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = TRUE; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + rnum = kcatoix(rstr); + if (rnum < 1) usage(); + return procindex(path, rnum, rnd, etc, oflags); +} + + +/* parse arguments of map command */ +static int32_t runmap(int argc, char** argv) { + int32_t argbrk = FALSE; + const char* rstr; + int32_t rnd, etc, i; + int64_t rnum, bnum; + rstr = NULL; + rnd = FALSE; + etc = FALSE; + bnum = -1; + for (i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!strcmp(argv[i], "--")) { + argbrk = TRUE; + } else if (!strcmp(argv[i], "-rnd")) { + rnd = TRUE; + } else if (!strcmp(argv[i], "-etc")) { + etc = TRUE; + } else if (!strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kcatoix(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = TRUE; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + rnum = kcatoix(rstr); + if (rnum < 1) usage(); + return procmap(rnum, rnd, etc, bnum); +} + + +/* parse arguments of list command */ +static int32_t runlist(int argc, char** argv) { + int32_t argbrk = FALSE; + const char* rstr; + int32_t rnd, etc, i; + int64_t rnum; + rstr = NULL; + rnd = FALSE; + etc = FALSE; + for (i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!strcmp(argv[i], "--")) { + argbrk = TRUE; + } else if (!strcmp(argv[i], "-rnd")) { + rnd = TRUE; + } else if (!strcmp(argv[i], "-etc")) { + etc = TRUE; + } else { + usage(); + } + } else if (!rstr) { + argbrk = TRUE; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + rnum = kcatoix(rstr); + if (rnum < 1) usage(); + return proclist(rnum, rnd, etc); +} + + +/* perform order command */ +static int32_t procorder(const char* path, int64_t rnum, int32_t rnd, int32_t etc, + int32_t tran, int32_t oflags) { + KCDB* db; + KCCUR* cur, *paracur; + int32_t err; + char kbuf[RECBUFSIZ], *vbuf, wbuf[RECBUFSIZ], *corepath, *copypath, *snappath; + size_t ksiz, vsiz, psiz; + int32_t wsiz; + int64_t i, cnt; + double stime, etime; + VISARG visarg; + oprintf("<In-order Test>\n path=%s rnum=%ld rnd=%d etc=%d tran=%d oflags=%d\n\n", + path, (long)rnum, rnd, etc, tran, oflags); + err = FALSE; + db = kcdbnew(); + oprintf("opening the database:\n"); + stime = kctime(); + if (!kcdbopen(db, path, KCOWRITER | KCOCREATE | KCOTRUNCATE | oflags)) { + dberrprint(db, __LINE__, "kcdbopen"); + err = TRUE; + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + oprintf("setting records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcdbset(db, kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db, __LINE__, "kcdbset"); + err = TRUE; + } + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("adding records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcdbadd(db, kbuf, ksiz, kbuf, ksiz) && kcdbecode(db) != KCEDUPREC) { + dberrprint(db, __LINE__, "kcdbadd"); + err = TRUE; + } + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("appending records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcdbappend(db, kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db, __LINE__, "kcdbadd"); + err = TRUE; + } + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("getting records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + vbuf = kcdbget(db, kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db, __LINE__, "kcdbget"); + err = TRUE; + } + kcfree(vbuf); + } else if (!rnd || kcdbecode(db) != KCENOREC) { + dberrprint(db, __LINE__, "kcdbget"); + err = TRUE; + } + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("getting records with a buffer:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + wsiz = kcdbgetbuf(db, kbuf, ksiz, wbuf, sizeof(wbuf)); + if (wsiz >= 0) { + if (wsiz < (int32_t)ksiz || memcmp(wbuf, kbuf, ksiz)) { + dberrprint(db, __LINE__, "kcdbgetbuf"); + err = TRUE; + } + } else if (!rnd || kcdbecode(db) != KCENOREC) { + dberrprint(db, __LINE__, "kcdbgetbuf"); + err = TRUE; + } + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the inner iterator:\n"); + stime = kctime(); + cnt = kcdbcount(db); + visarg.rnum = rnum; + visarg.rnd = rnd; + visarg.cnt = 0; + memset(visarg.rbuf, '+', sizeof(visarg.rbuf)); + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + if (!kcdbiterate(db, visitfull, &visarg, TRUE)) { + dberrprint(db, __LINE__, "kcdbiterate"); + err = TRUE; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (visarg.cnt != cnt) { + dberrprint(db, __LINE__, "kcdbiterate"); + err = TRUE; + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the outer cursor:\n"); + stime = kctime(); + cnt = kcdbcount(db); + visarg.rnum = rnum; + visarg.rnd = rnd; + visarg.cnt = 0; + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + cur = kcdbcursor(db); + if (!kccurjump(cur) && kccurecode(cur) != KCENOREC) { + dberrprint(db, __LINE__, "kccurjump"); + err = TRUE; + } + paracur = kcdbcursor(db); + while (!err && kccuraccept(cur, &visitfull, &visarg, TRUE, !rnd)) { + if (rnd) { + ksiz = sprintf(kbuf, "%08ld", (long)myrand(rnum)); + switch (myrand(3)) { + case 0: { + if (!kcdbremove(db, kbuf, ksiz) && kcdbecode(db) != KCENOREC) { + dberrprint(db, __LINE__, "kcdbremove"); + err = TRUE; + } + break; + } + case 1: { + if (!kccurjumpkey(paracur, kbuf, ksiz) && kccurecode(paracur) != KCENOREC) { + dberrprint(db, __LINE__, "kccurjump"); + err = TRUE; + } + break; + } + default: { + if (!kccurstep(cur) && kccurecode(cur) != KCENOREC) { + dberrprint(db, __LINE__, "kccurstep"); + err = TRUE; + } + break; + } + } + } + } + oprintf(" (end)\n"); + kccurdel(paracur); + kccurdel(cur); + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (!rnd && visarg.cnt != cnt) { + dberrprint(db, __LINE__, "kccuraccept"); + err = TRUE; + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("synchronizing the database:\n"); + stime = kctime(); + if (!kcdbsync(db, FALSE, NULL, NULL)) { + dberrprint(db, __LINE__, "kcdbsync"); + err = TRUE; + } + if (!kcdboccupy(db, FALSE, NULL, NULL)) { + dberrprint(db, __LINE__, "kcdboccupy"); + err = TRUE; + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + corepath = kcdbpath(db); + psiz = strlen(corepath); + if (strstr(corepath, ".kch") || strstr(corepath, ".kct")) { + copypath = kcmalloc(psiz + 256); + sprintf(copypath, "%s.tmp", corepath); + snappath = kcmalloc(psiz + 256); + sprintf(snappath, "%s.kcss", corepath); + } else { + copypath = kcmalloc(256); + sprintf(copypath, "kclangctest.tmp"); + snappath = kcmalloc(256); + sprintf(snappath, "kclangctest.kcss"); + } + oprintf("copying the database file:\n"); + stime = kctime(); + if (!kcdbcopy(db, copypath)) { + dberrprint(db, __LINE__, "kcdbcopy"); + err = TRUE; + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + remove(copypath); + oprintf("dumping records into snapshot:\n"); + stime = kctime(); + if (!kcdbdumpsnap(db, snappath)) { + dberrprint(db, __LINE__, "kcdbdumpsnap"); + err = TRUE; + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records into snapshot:\n"); + stime = kctime(); + cnt = kcdbcount(db); + if (rnd && myrand(2) == 0 && !kcdbclear(db)) { + dberrprint(db, __LINE__, "kcdbclear"); + err = TRUE; + } + if (!kcdbloadsnap(db, snappath) || kcdbcount(db) != cnt) { + dberrprint(db, __LINE__, "kcdbloadsnap"); + err = TRUE; + } + etime = kctime(); + dbmetaprint(db, FALSE); + oprintf("time: %.3f\n", etime - stime); + remove(snappath); + kcfree(copypath); + kcfree(snappath); + kcfree(corepath); + } + oprintf("removing records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + if (tran && !kcdbbegintran(db, FALSE)) { + dberrprint(db, __LINE__, "kcdbbegintran"); + err = TRUE; + } + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcdbremove(db, kbuf, ksiz) && + ((!rnd && !etc) || kcdbecode(db) != KCENOREC)) { + dberrprint(db, __LINE__, "kcdbremove"); + err = TRUE; + } + if (tran && !kcdbendtran(db, TRUE)) { + dberrprint(db, __LINE__, "kcdbendtran"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + dbmetaprint(db, TRUE); + oprintf("time: %.3f\n", etime - stime); + oprintf("closing the database:\n"); + stime = kctime(); + if (!kcdbclose(db)) { + dberrprint(db, __LINE__, "kcdbclose"); + err = TRUE; + } + etime = kctime(); + oprintf("time: %.3f\n", etime - stime); + kcdbdel(db); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +/* perform index command */ +static int32_t procindex(const char* path, int64_t rnum, int32_t rnd, int32_t etc, + int32_t oflags) { + KCIDX* idx; + int32_t err; + char kbuf[RECBUFSIZ], *vbuf; + size_t ksiz, vsiz; + int64_t i; + double stime, etime; + oprintf("<Index Database Test>\n path=%s rnum=%ld rnd=%d etc=%d oflags=%d\n\n", + path, (long)rnum, rnd, etc, oflags); + err = FALSE; + idx = kcidxnew(); + oprintf("opening the database:\n"); + stime = kctime(); + if (!kcidxopen(idx, path, KCOWRITER | KCOCREATE | KCOTRUNCATE | oflags)) { + idxerrprint(idx, __LINE__, "kcidxopen"); + err = TRUE; + } + etime = kctime(); + idxmetaprint(idx, FALSE); + oprintf("time: %.3f\n", etime - stime); + oprintf("setting records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcidxset(idx, kbuf, ksiz, kbuf, ksiz)) { + idxerrprint(idx, __LINE__, "kcidxset"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + idxmetaprint(idx, FALSE); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("adding records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcidxadd(idx, kbuf, ksiz, kbuf, ksiz) && kcidxecode(idx) != KCEDUPREC) { + idxerrprint(idx, __LINE__, "kcidxadd"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + idxmetaprint(idx, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("appending records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcidxappend(idx, kbuf, ksiz, kbuf, ksiz)) { + idxerrprint(idx, __LINE__, "kcidxadd"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + idxmetaprint(idx, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("getting records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + vbuf = kcidxget(idx, kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || memcmp(vbuf, kbuf, ksiz)) { + idxerrprint(idx, __LINE__, "kcidxget"); + err = TRUE; + } + kcfree(vbuf); + } else if (!rnd || kcidxecode(idx) != KCENOREC) { + idxerrprint(idx, __LINE__, "kcidxget"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + idxmetaprint(idx, FALSE); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("synchronizing the database:\n"); + stime = kctime(); + if (!kcidxsync(idx, FALSE, NULL, NULL)) { + idxerrprint(idx, __LINE__, "kcidxsync"); + err = TRUE; + } + etime = kctime(); + idxmetaprint(idx, FALSE); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("removing records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcidxremove(idx, kbuf, ksiz) && + ((!rnd && !etc) || kcidxecode(idx) != KCENOREC)) { + idxerrprint(idx, __LINE__, "kcidxremove"); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + idxmetaprint(idx, TRUE); + oprintf("time: %.3f\n", etime - stime); + oprintf("closing the database:\n"); + stime = kctime(); + if (!kcidxclose(idx)) { + idxerrprint(idx, __LINE__, "kcidxclose"); + err = TRUE; + } + etime = kctime(); + oprintf("time: %.3f\n", etime - stime); + kcidxdel(idx); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +/* perform map command */ +static int32_t procmap(int64_t rnum, int32_t rnd, int32_t etc, int64_t bnum) { + KCMAP* map; + KCMAPITER* iter; + KCMAPSORT* sort; + int32_t err; + char kbuf[RECBUFSIZ]; + const char* vbuf, *ikbuf; + size_t ksiz, vsiz; + int64_t i, cnt; + double stime, etime; + oprintf("<Memory-saving Hash Map Test>\n rnum=%ld rnd=%d etc=%d bnum=%ld\n\n", + (long)rnum, rnd, etc, (long)bnum); + err = FALSE; + if (bnum < 0) bnum = 0; + map = kcmapnew(bnum); + oprintf("setting records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + kcmapset(map, kbuf, ksiz, kbuf, ksiz); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kcmapcount(map)); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("adding records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + kcmapadd(map, kbuf, ksiz, kbuf, ksiz); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kcmapcount(map)); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("appending records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + kcmapappend(map, kbuf, ksiz, kbuf, ksiz); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kcmapcount(map)); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("getting records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + vbuf = kcmapget(map, kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || memcmp(vbuf, kbuf, ksiz)) { + eprintf("%s: kcmapget failed\n", g_progname); + err = TRUE; + } + } else if (!rnd) { + eprintf("%s: kcmapget failed\n", g_progname); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kcmapcount(map)); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("traversing records:\n"); + stime = kctime(); + cnt = 0; + iter = kcmapiterator(map); + while (!err && (ikbuf = kcmapiterget(iter, &ksiz, &vbuf, &vsiz)) != NULL) { + if (rnd) { + ksiz = sprintf(kbuf, "%08ld", (long)myrand(rnum)); + switch (myrand(3)) { + case 0: { + kcmapremove(map, kbuf, ksiz); + break; + } + case 1: { + kcmapappend(map, kbuf, ksiz, kbuf, ksiz); + break; + } + } + } + if (!kcmapitergetkey(iter, &ksiz)) { + eprintf("%s: kcmapitergetkey failed\n", g_progname); + err = TRUE; + } + if (!kcmapitergetvalue(iter, &vsiz)) { + eprintf("%s: kcmapitergetvalue failed\n", g_progname); + err = TRUE; + } + cnt++; + if (rnum > 250 && cnt % (rnum / 250) == 0) { + oputchar('.'); + if (cnt == rnum || cnt % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)cnt); + } + kcmapiterstep(iter); + } + if (rnd) oprintf(" (end)\n"); + kcmapiterdel(iter); + if (!rnd && cnt != (int64_t)kcmapcount(map)) { + eprintf("%s: kcmapcount failed\n", g_progname); + err = TRUE; + } + etime = kctime(); + oprintf("count: %ld\n", (long)kcmapcount(map)); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("sorting records:\n"); + stime = kctime(); + cnt = 0; + sort = kcmapsorter(map); + while (!err && (ikbuf = kcmapsortget(sort, &ksiz, &vbuf, &vsiz)) != NULL) { + if (!kcmapsortgetkey(sort, &ksiz)) { + eprintf("%s: kcmapsortgetkey failed\n", g_progname); + err = TRUE; + } + if (!kcmapsortgetvalue(sort, &vsiz)) { + eprintf("%s: kcmapsortgetvalue failed\n", g_progname); + err = TRUE; + } + cnt++; + if (rnum > 250 && cnt % (rnum / 250) == 0) { + oputchar('.'); + if (cnt == rnum || cnt % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)cnt); + } + kcmapsortstep(sort); + } + if (rnd) oprintf(" (end)\n"); + kcmapsortdel(sort); + if (!rnd && cnt != (int64_t)kcmapcount(map)) { + eprintf("%s: kcmapcount failed\n", g_progname); + err = TRUE; + } + etime = kctime(); + oprintf("count: %ld\n", (long)kcmapcount(map)); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("removing records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); + if (!kcmapremove(map, kbuf, ksiz) && !rnd) { + eprintf("%s: kcmapremove failed\n", g_progname); + err = TRUE; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kcmapcount(map)); + oprintf("time: %.3f\n", etime - stime); + kcmapdel(map); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +/* perform list command */ +static int32_t proclist(int64_t rnum, int32_t rnd, int32_t etc) { + KCLIST* list; + int32_t err; + char buf[RECBUFSIZ]; + size_t size; + int64_t i, cnt; + double stime, etime; + oprintf("<Memory-saving Array List Test>\n rnum=%ld rnd=%d etc=%d\n\n", + (long)rnum, rnd, etc); + err = FALSE; + list = kclistnew(); + oprintf("setting records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + size = sprintf(buf, "%08ld", (long)i); + if (rnd && myrand(2) == 0) { + kclistunshift(list, buf, size); + } else { + kclistpush(list, buf, size); + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kclistcount(list)); + oprintf("time: %.3f\n", etime - stime); + oprintf("getting records:\n"); + stime = kctime(); + cnt = kclistcount(list); + for (i = 1; !err && i <= rnum; i++) { + kclistget(list, rnd ? myrand(cnt) : i - 1, &size); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kclistcount(list)); + oprintf("time: %.3f\n", etime - stime); + oprintf("removing records:\n"); + stime = kctime(); + for (i = 1; !err && i <= rnum; i++) { + if (rnd && myrand(2) == 0) { + kclistshift(list); + } else { + kclistpop(list); + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("count: %ld\n", (long)kclistcount(list)); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("wicked testing:\n"); + stime = kctime(); + memset(buf, '*', sizeof(buf)); + for (i = 1; !err && i <= rnum; i++) { + size = rnd ? (size_t)myrand(sizeof(buf)) : sizeof(buf); + cnt = kclistcount(list); + switch (rnd ? myrand(10) : i % 10) { + case 0: { + kclistpop(list); + break; + } + case 1: { + kclistunshift(list, buf, size); + break; + } + case 2: { + kclistshift(list); + break; + } + case 3: { + kclistinsert(list, buf, size, rnd && cnt > 0 ? myrand(cnt) : cnt / 2); + break; + } + case 4: { + if (cnt > 0) kclistremove(list, rnd ? myrand(cnt) : cnt / 2); + break; + } + case 5: { + if (cnt > 0) kclistget(list, rnd ? myrand(cnt) : cnt / 2, &size); + break; + } + case 6: { + if (rnd ? myrand(100) == 0 : i % 127 == 0) kclistclear(list); + break; + } + default: { + kclistpush(list, buf, size); + break; + } + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); + } + } + etime = kctime(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %ld\n", (long)kclistcount(list)); + } + kclistdel(list); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +/* END OF FILE */ diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcmap.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcmap.cc new file mode 100644 index 0000000000..65bc8a0e50 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcmap.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Data mapping structures + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcmap.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcmap.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcmap.h new file mode 100644 index 0000000000..7729c6504a --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcmap.h @@ -0,0 +1,1379 @@ +/************************************************************************************************* + * Data mapping structures + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCMAP_H // duplication check +#define _KCMAP_H + +#include <kccommon.h> +#include <kcutil.h> + +namespace kyotocabinet { // common namespace + + +/** + * Doubly-linked hash map. + * @param KEY the key type. + * @param VALUE the value type. + * @param HASH the hash functor. + * @param EQUALTO the equality checking functor. + */ +template <class KEY, class VALUE, + class HASH = std::hash<KEY>, class EQUALTO = std::equal_to<KEY> > +class LinkedHashMap { + public: + class Iterator; + private: + struct Record; + /** The default bucket number of hash table. */ + static const size_t MAPDEFBNUM = 31; + /** The mininum number of buckets to use mmap. */ + static const size_t MAPZMAPBNUM = 32768; + public: + /** + * Iterator of records. + */ + class Iterator { + friend class LinkedHashMap; + public: + /** + * Copy constructor. + * @param src the source object. + */ + Iterator(const Iterator& src) : map_(src.map_), rec_(src.rec_) { + _assert_(true); + } + /** + * Get the key. + */ + const KEY& key() { + _assert_(true); + return rec_->key; + } + /** + * Get the value. + */ + VALUE& value() { + _assert_(true); + return rec_->value; + } + /** + * Assignment operator from the self type. + * @param right the right operand. + * @return the reference to itself. + */ + Iterator& operator =(const Iterator& right) { + _assert_(true); + if (&right == this) return *this; + map_ = right.map_; + rec_ = right.rec_; + return *this; + } + /** + * Equality operator with the self type. + * @param right the right operand. + * @return true if the both are equal, or false if not. + */ + bool operator ==(const Iterator& right) const { + _assert_(true); + return map_ == right.map_ && rec_ == right.rec_; + } + /** + * Non-equality operator with the self type. + * @param right the right operand. + * @return false if the both are equal, or true if not. + */ + bool operator !=(const Iterator& right) const { + _assert_(true); + return map_ != right.map_ || rec_ != right.rec_; + } + /** + * Preposting increment operator. + * @return the iterator itself. + */ + Iterator& operator ++() { + _assert_(true); + rec_ = rec_->next; + return *this; + } + /** + * Postpositive increment operator. + * @return an iterator of the old position. + */ + Iterator operator ++(int) { + _assert_(true); + Iterator old(*this); + rec_ = rec_->next; + return old; + } + /** + * Preposting decrement operator. + * @return the iterator itself. + */ + Iterator& operator --() { + _assert_(true); + if (rec_) { + rec_ = rec_->prev; + } else { + rec_ = map_->last_; + } + return *this; + } + /** + * Postpositive decrement operator. + * @return an iterator of the old position. + */ + Iterator operator --(int) { + _assert_(true); + Iterator old(*this); + if (rec_) { + rec_ = rec_->prev; + } else { + rec_ = map_->last_; + } + return old; + } + private: + /** + * Constructor. + * @param map the container. + * @param rec the pointer to the current record. + */ + explicit Iterator(LinkedHashMap* map, Record* rec) : map_(map), rec_(rec) { + _assert_(map); + } + /** The container. */ + LinkedHashMap* map_; + /** The current record. */ + Record* rec_; + }; + /** + * Moving Modes. + */ + enum MoveMode { + MCURRENT, ///< keep the current position + MFIRST, ///< move to the first + MLAST ///< move to the last + }; + /** + * Default constructor. + */ + explicit LinkedHashMap() : + buckets_(NULL), bnum_(MAPDEFBNUM), first_(NULL), last_(NULL), count_(0) { + _assert_(true); + initialize(); + } + /** + * Constructor. + * @param bnum the number of buckets of the hash table. + */ + explicit LinkedHashMap(size_t bnum) : + buckets_(NULL), bnum_(bnum), first_(NULL), last_(NULL), count_(0) { + _assert_(true); + if (bnum_ < 1) bnum_ = MAPDEFBNUM; + initialize(); + } + /** + * Destructor. + */ + ~LinkedHashMap() { + _assert_(true); + destroy(); + } + /** + * Store a record. + * @param key the key. + * @param value the value. + * @param mode the moving mode. + * @return the pointer to the value of the stored record. + */ + VALUE *set(const KEY& key, const VALUE& value, MoveMode mode) { + _assert_(true); + size_t bidx = hash_(key) % bnum_; + Record* rec = buckets_[bidx]; + Record** entp = buckets_ + bidx; + while (rec) { + if (equalto_(rec->key, key)) { + rec->value = value; + switch (mode) { + default: { + break; + } + case MFIRST: { + if (first_ != rec) { + if (last_ == rec) last_ = rec->prev; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + rec->prev = NULL; + rec->next = first_; + first_->prev = rec; + first_ = rec; + } + break; + } + case MLAST: { + if (last_ != rec) { + if (first_ == rec) first_ = rec->next; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + rec->prev = last_; + rec->next = NULL; + last_->next = rec; + last_ = rec; + } + break; + } + } + return &rec->value; + } else { + entp = &rec->child; + rec = rec->child; + } + } + rec = new Record(key, value); + switch (mode) { + default: { + rec->prev = last_; + if (!first_) first_ = rec; + if (last_) last_->next = rec; + last_ = rec; + break; + } + case MFIRST: { + rec->next = first_; + if (!last_) last_ = rec; + if (first_) first_->prev = rec; + first_ = rec; + break; + } + } + *entp = rec; + count_++; + return &rec->value; + } + /** + * Remove a record. + * @param key the key. + * @return true on success, or false on failure. + */ + bool remove(const KEY& key) { + _assert_(true); + size_t bidx = hash_(key) % bnum_; + Record* rec = buckets_[bidx]; + Record** entp = buckets_ + bidx; + while (rec) { + if (equalto_(rec->key, key)) { + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + if (rec == first_) first_ = rec->next; + if (rec == last_) last_ = rec->prev; + *entp = rec->child; + count_--; + delete rec; + return true; + } else { + entp = &rec->child; + rec = rec->child; + } + } + return false; + } + /** + * Migrate a record to another map. + * @param key the key. + * @param dist the destination map. + * @param mode the moving mode. + * @return the pointer to the value of the migrated record, or NULL on failure. + */ + VALUE* migrate(const KEY& key, LinkedHashMap* dist, MoveMode mode) { + _assert_(dist); + size_t hash = hash_(key); + size_t bidx = hash % bnum_; + Record* rec = buckets_[bidx]; + Record** entp = buckets_ + bidx; + while (rec) { + if (equalto_(rec->key, key)) { + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + if (rec == first_) first_ = rec->next; + if (rec == last_) last_ = rec->prev; + *entp = rec->child; + count_--; + rec->child = NULL; + rec->prev = NULL; + rec->next = NULL; + bidx = hash % dist->bnum_; + Record* drec = dist->buckets_[bidx]; + entp = dist->buckets_ + bidx; + while (drec) { + if (dist->equalto_(drec->key, key)) { + if (drec->child) rec->child = drec->child; + if (drec->prev) { + rec->prev = drec->prev; + rec->prev->next = rec; + } + if (drec->next) { + rec->next = drec->next; + rec->next->prev = rec; + } + if (dist->first_ == drec) dist->first_ = rec; + if (dist->last_ == drec) dist->last_ = rec; + *entp = rec; + delete drec; + switch (mode) { + default: { + break; + } + case MFIRST: { + if (dist->first_ != rec) { + if (dist->last_ == rec) dist->last_ = rec->prev; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + rec->prev = NULL; + rec->next = dist->first_; + dist->first_->prev = rec; + dist->first_ = rec; + } + break; + } + case MLAST: { + if (dist->last_ != rec) { + if (dist->first_ == rec) dist->first_ = rec->next; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + rec->prev = dist->last_; + rec->next = NULL; + dist->last_->next = rec; + dist->last_ = rec; + } + break; + } + } + return &rec->value; + } else { + entp = &drec->child; + drec = drec->child; + } + } + switch (mode) { + default: { + rec->prev = dist->last_; + if (!dist->first_) dist->first_ = rec; + if (dist->last_) dist->last_->next = rec; + dist->last_ = rec; + break; + } + case MFIRST: { + rec->next = dist->first_; + if (!dist->last_) dist->last_ = rec; + if (dist->first_) dist->first_->prev = rec; + dist->first_ = rec; + break; + } + } + *entp = rec; + dist->count_++; + return &rec->value; + } else { + entp = &rec->child; + rec = rec->child; + } + } + return NULL; + } + /** + * Retrieve a record. + * @param key the key. + * @param mode the moving mode. + * @return the pointer to the value of the corresponding record, or NULL on failure. + */ + VALUE* get(const KEY& key, MoveMode mode) { + _assert_(true); + size_t bidx = hash_(key) % bnum_; + Record* rec = buckets_[bidx]; + while (rec) { + if (equalto_(rec->key, key)) { + switch (mode) { + default: { + break; + } + case MFIRST: { + if (first_ != rec) { + if (last_ == rec) last_ = rec->prev; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + rec->prev = NULL; + rec->next = first_; + first_->prev = rec; + first_ = rec; + } + break; + } + case MLAST: { + if (last_ != rec) { + if (first_ == rec) first_ = rec->next; + if (rec->prev) rec->prev->next = rec->next; + if (rec->next) rec->next->prev = rec->prev; + rec->prev = last_; + rec->next = NULL; + last_->next = rec; + last_ = rec; + } + break; + } + } + return &rec->value; + } else { + rec = rec->child; + } + } + return NULL; + } + /** + * Remove all records. + */ + void clear() { + _assert_(true); + if (count_ < 1) return; + Record* rec = last_; + while (rec) { + Record* prev = rec->prev; + delete rec; + rec = prev; + } + for (size_t i = 0; i < bnum_; i++) { + buckets_[i] = NULL; + } + first_ = NULL; + last_ = NULL; + count_ = 0; + } + /** + * Get the number of records. + */ + size_t count() { + _assert_(true); + return count_; + } + /** + * Get an iterator at the first record. + */ + Iterator begin() { + _assert_(true); + return Iterator(this, first_); + } + /** + * Get an iterator of the end sentry. + */ + Iterator end() { + _assert_(true); + return Iterator(this, NULL); + } + /** + * Get an iterator at a record. + * @param key the key. + * @return the pointer to the value of the corresponding record, or NULL on failure. + */ + Iterator find(const KEY& key) { + _assert_(true); + size_t bidx = hash_(key) % bnum_; + Record* rec = buckets_[bidx]; + while (rec) { + if (equalto_(rec->key, key)) { + return Iterator(this, rec); + } else { + rec = rec->child; + } + } + return Iterator(this, NULL); + } + /** + * Get the reference of the key of the first record. + * @return the reference of the key of the first record. + */ + const KEY& first_key() { + _assert_(true); + return first_->key; + } + /** + * Get the reference of the value of the first record. + * @return the reference of the value of the first record. + */ + VALUE& first_value() { + _assert_(true); + return first_->value; + } + /** + * Get the reference of the key of the last record. + * @return the reference of the key of the last record. + */ + const KEY& last_key() { + _assert_(true); + return last_->key; + } + /** + * Get the reference of the value of the last record. + * @return the reference of the value of the last record. + */ + VALUE& last_value() { + _assert_(true); + return last_->value; + } + private: + /** + * Record data. + */ + struct Record { + KEY key; ///< key + VALUE value; ///< value + Record* child; ///< child record + Record* prev; ///< previous record + Record* next; ///< next record + /** constructor */ + explicit Record(const KEY& k, const VALUE& v) : + key(k), value(v), child(NULL), prev(NULL), next(NULL) { + _assert_(true); + } + }; + /** + * Initialize fields. + */ + void initialize() { + _assert_(true); + if (bnum_ >= MAPZMAPBNUM) { + buckets_ = (Record**)mapalloc(sizeof(*buckets_) * bnum_); + } else { + buckets_ = new Record*[bnum_]; + for (size_t i = 0; i < bnum_; i++) { + buckets_[i] = NULL; + } + } + } + /** + * Clean up fields. + */ + void destroy() { + _assert_(true); + Record* rec = last_; + while (rec) { + Record* prev = rec->prev; + delete rec; + rec = prev; + } + if (bnum_ >= MAPZMAPBNUM) { + mapfree(buckets_); + } else { + delete[] buckets_; + } + } + /** Dummy constructor to forbid the use. */ + LinkedHashMap(const LinkedHashMap&); + /** Dummy Operator to forbid the use. */ + LinkedHashMap& operator =(const LinkedHashMap&); + /** The functor of the hash function. */ + HASH hash_; + /** The functor of the equalto function. */ + EQUALTO equalto_; + /** The bucket array. */ + Record** buckets_; + /** The number of buckets. */ + size_t bnum_; + /** The first record. */ + Record* first_; + /** The last record. */ + Record* last_; + /** The number of records. */ + size_t count_; +}; + + +/** + * Memory-saving string hash map. + */ +class TinyHashMap { + public: + class Iterator; + private: + struct Record; + struct RecordComparator; + /** The default bucket number of hash table. */ + static const size_t MAPDEFBNUM = 31; + /** The mininum number of buckets to use mmap. */ + static const size_t MAPZMAPBNUM = 32768; + public: + /** + * Iterator of records. + */ + class Iterator { + friend class TinyHashMap; + public: + /** + * Constructor. + * @param map the container. + * @note This object will not be invalidated even when the map object is updated once. + * However, phantom records may be retrieved if they are removed after creation of each + * iterator. + */ + explicit Iterator(TinyHashMap* map) : map_(map), bidx_(-1), ridx_(0), recs_() { + _assert_(map); + step(); + } + /** + * Destructor. + */ + ~Iterator() { + _assert_(true); + free_records(); + } + /** + * Get the key of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the key region of the current record, or NULL on failure. + */ + const char* get_key(size_t* sp) { + _assert_(sp); + if (ridx_ >= recs_.size()) return NULL; + Record rec(recs_[ridx_]); + *sp = rec.ksiz_; + return rec.kbuf_; + } + /** + * Get the value of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the current record, or NULL on failure. + */ + const char* get_value(size_t* sp) { + _assert_(sp); + if (ridx_ >= recs_.size()) return NULL; + Record rec(recs_[ridx_]); + *sp = rec.vsiz_; + return rec.vbuf_; + } + /** + * Get a pair of the key and the value of the current record. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @return the pointer to the key region, or NULL on failure. + */ + const char* get(size_t* ksp, const char** vbp, size_t* vsp) { + _assert_(ksp && vbp && vsp); + if (ridx_ >= recs_.size()) return NULL; + Record rec(recs_[ridx_]); + *ksp = rec.ksiz_; + *vbp = rec.vbuf_; + *vsp = rec.vsiz_; + return rec.kbuf_; + } + /** + * Step the cursor to the next record. + */ + void step() { + _assert_(true); + if (++ridx_ >= recs_.size()) { + ridx_ = 0; + free_records(); + while (true) { + bidx_++; + if (bidx_ >= (int64_t)map_->bnum_) return; + read_records(); + if (recs_.size() > 0) break; + } + } + } + private: + /** + * Read records of the current bucket. + */ + void read_records() { + char* rbuf = map_->buckets_[bidx_]; + while (rbuf) { + Record rec(rbuf); + size_t rsiz = sizeof(rec.child_) + sizevarnum(rec.ksiz_) + rec.ksiz_ + + sizevarnum(rec.vsiz_) + rec.vsiz_ + sizevarnum(rec.psiz_); + char* nbuf = new char[rsiz]; + std::memcpy(nbuf, rbuf, rsiz); + recs_.push_back(nbuf); + rbuf = rec.child_; + } + } + /** + * Release recources of the current records. + */ + void free_records() { + std::vector<char*>::iterator it = recs_.begin(); + std::vector<char*>::iterator itend = recs_.end(); + while (it != itend) { + char* rbuf = *it; + delete[] rbuf; + ++it; + } + recs_.clear(); + } + /** Dummy constructor to forbid the use. */ + Iterator(const Iterator&); + /** Dummy Operator to forbid the use. */ + Iterator& operator =(const Iterator&); + /** The container. */ + TinyHashMap* map_; + /** The current bucket index. */ + int64_t bidx_; + /** The current record index. */ + size_t ridx_; + /** The current records. */ + std::vector<char*> recs_; + }; + /** + * Sorter of records. + */ + class Sorter { + public: + /** + * Constructor. + * @param map the container. + * @note This object will be invalidated when the map object is updated once. + */ + explicit Sorter(TinyHashMap* map) : map_(map), ridx_(0), recs_() { + _assert_(map); + char** buckets = map_->buckets_; + size_t bnum = map_->bnum_; + for (size_t i = 0; i < bnum; i++) { + char* rbuf = buckets[i]; + while (rbuf) { + Record rec(rbuf); + recs_.push_back(rbuf); + rbuf = *(char**)rbuf; + } + } + std::sort(recs_.begin(), recs_.end(), RecordComparator()); + } + /** + * Destructor. + */ + ~Sorter() { + _assert_(true); + } + /** + * Get the key of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the key region of the current record, or NULL on failure. + */ + const char* get_key(size_t* sp) { + _assert_(sp); + if (ridx_ >= recs_.size()) return NULL; + Record rec(recs_[ridx_]); + *sp = rec.ksiz_; + return rec.kbuf_; + } + /** + * Get the value of the current record. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the current record, or NULL on failure. + */ + const char* get_value(size_t* sp) { + _assert_(sp); + if (ridx_ >= recs_.size()) return NULL; + Record rec(recs_[ridx_]); + *sp = rec.vsiz_; + return rec.vbuf_; + } + /** + * Get a pair of the key and the value of the current record. + * @param ksp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @param vbp the pointer to the variable into which the pointer to the value region is + * assigned. + * @param vsp the pointer to the variable into which the size of the value region is + * assigned. + * @return the pointer to the key region, or NULL on failure. + */ + const char* get(size_t* ksp, const char** vbp, size_t* vsp) { + _assert_(ksp && vbp && vsp); + if (ridx_ >= recs_.size()) return NULL; + Record rec(recs_[ridx_]); + *ksp = rec.ksiz_; + *vbp = rec.vbuf_; + *vsp = rec.vsiz_; + return rec.kbuf_; + } + /** + * Step the cursor to the next record. + */ + void step() { + _assert_(true); + ridx_++; + } + /** The container. */ + TinyHashMap* map_; + /** The current record index. */ + size_t ridx_; + /** The current records. */ + std::vector<char*> recs_; + }; + /** + * Default constructor. + */ + explicit TinyHashMap() : buckets_(NULL), bnum_(MAPDEFBNUM), count_(0) { + _assert_(true); + initialize(); + } + /** + * Constructor. + * @param bnum the number of buckets of the hash table. + */ + explicit TinyHashMap(size_t bnum) : buckets_(NULL), bnum_(bnum), count_(0) { + _assert_(true); + if (bnum_ < 1) bnum_ = MAPDEFBNUM; + initialize(); + } + /** + * Destructor. + */ + ~TinyHashMap() { + _assert_(true); + destroy(); + } + /** + * Set the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the value is overwritten. + */ + void set(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + char* rbuf = buckets_[bidx]; + char** entp = buckets_ + bidx; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) { + int32_t oh = (int32_t)sizevarnum(vsiz) - (int32_t)sizevarnum(rec.vsiz_); + int64_t psiz = (int64_t)(rec.vsiz_ + rec.psiz_) - (int64_t)(vsiz + oh); + if (psiz >= 0) { + rec.overwrite(rbuf, vbuf, vsiz, psiz); + } else { + Record nrec(rec.child_, kbuf, ksiz, vbuf, vsiz, 0); + delete[] rbuf; + *entp = nrec.serialize(); + } + return; + } + entp = (char**)rbuf; + rbuf = rec.child_; + } + Record nrec(NULL, kbuf, ksiz, vbuf, vsiz, 0); + *entp = nrec.serialize(); + count_++; + } + /** + * Add a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the record is not modified and false is returned. + */ + bool add(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + char* rbuf = buckets_[bidx]; + char** entp = buckets_ + bidx; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) return false; + entp = (char**)rbuf; + rbuf = rec.child_; + } + Record nrec(NULL, kbuf, ksiz, vbuf, vsiz, 0); + *entp = nrec.serialize(); + count_++; + return true; + } + /** + * Replace the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, no new record is created and false is returned. + * If the corresponding record exists, the value is modified. + */ + bool replace(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + char* rbuf = buckets_[bidx]; + char** entp = buckets_ + bidx; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) { + int32_t oh = (int32_t)sizevarnum(vsiz) - (int32_t)sizevarnum(rec.vsiz_); + int64_t psiz = (int64_t)(rec.vsiz_ + rec.psiz_) - (int64_t)(vsiz + oh); + if (psiz >= 0) { + rec.overwrite(rbuf, vbuf, vsiz, psiz); + } else { + Record nrec(rec.child_, kbuf, ksiz, vbuf, vsiz, 0); + delete[] rbuf; + *entp = nrec.serialize(); + } + return true; + } + entp = (char**)rbuf; + rbuf = rec.child_; + } + return false; + } + /** + * Append the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param vbuf the pointer to the value region. + * @param vsiz the size of the value region. + * @note If no record corresponds to the key, a new record is created. If the corresponding + * record exists, the given value is appended at the end of the existing value. + */ + void append(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + char* rbuf = buckets_[bidx]; + char** entp = buckets_ + bidx; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) { + size_t nsiz = rec.vsiz_ + vsiz; + int32_t oh = (int32_t)sizevarnum(nsiz) - (int32_t)sizevarnum(rec.vsiz_); + int64_t psiz = (int64_t)(rec.vsiz_ + rec.psiz_) - (int64_t)(nsiz + oh); + if (psiz >= 0) { + rec.append(rbuf, oh, vbuf, vsiz, psiz); + } else { + psiz = nsiz + nsiz / 2; + Record nrec(rec.child_, kbuf, ksiz, "", 0, psiz); + char* nbuf = nrec.serialize(); + oh = (int32_t)sizevarnum(nsiz) - 1; + psiz = (int64_t)psiz - (int64_t)(nsiz + oh); + rec.concatenate(nbuf, rec.vbuf_, rec.vsiz_, vbuf, vsiz, psiz); + delete[] rbuf; + *entp = nbuf; + } + return; + } + entp = (char**)rbuf; + rbuf = rec.child_; + } + Record nrec(NULL, kbuf, ksiz, vbuf, vsiz, 0); + *entp = nrec.serialize(); + count_++; + } + /** + * Remove a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note If no record corresponds to the key, false is returned. + */ + bool remove(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + char* rbuf = buckets_[bidx]; + char** entp = buckets_ + bidx; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) { + *entp = rec.child_; + delete[] rbuf; + count_--; + return true; + } + entp = (char**)rbuf; + rbuf = rec.child_; + } + return false; + } + /** + * Retrieve the value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the value region of the corresponding record, or NULL on failure. + */ + const char* get(const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && sp); + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + char* rbuf = buckets_[bidx]; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) { + *sp = rec.vsiz_; + return rec.vbuf_; + } + rbuf = rec.child_; + } + return NULL; + } + /** + * Remove all records. + */ + void clear() { + _assert_(true); + if (count_ < 1) return; + for (size_t i = 0; i < bnum_; i++) { + char* rbuf = buckets_[i]; + while (rbuf) { + Record rec(rbuf); + char* child = rec.child_; + delete[] rbuf; + rbuf = child; + } + buckets_[i] = NULL; + } + count_ = 0; + } + /** + * Get the number of records. + * @return the number of records. + */ + size_t count() { + _assert_(true); + return count_; + } + /** + * Get the hash value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the hash value. + */ + static size_t hash_record(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + return hashmurmur(kbuf, ksiz); + } + private: + /** + * Record data. + */ + struct Record { + /** constructor */ + Record(char* child, const char* kbuf, uint64_t ksiz, + const char* vbuf, uint64_t vsiz, uint64_t psiz) : + child_(child), kbuf_(kbuf), ksiz_(ksiz), vbuf_(vbuf), vsiz_(vsiz), psiz_(psiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && psiz <= MEMMAXSIZ); + } + /** constructor */ + Record(const char* rbuf) : + child_(NULL), kbuf_(NULL), ksiz_(0), vbuf_(NULL), vsiz_(0), psiz_(0) { + _assert_(rbuf); + deserialize(rbuf); + } + /** overwrite the buffer */ + void overwrite(char* rbuf, const char* vbuf, size_t vsiz, size_t psiz) { + _assert_(rbuf && vbuf && vsiz <= MEMMAXSIZ && psiz <= MEMMAXSIZ); + char* wp = rbuf + sizeof(child_) + sizevarnum(ksiz_) + ksiz_; + wp += writevarnum(wp, vsiz); + std::memcpy(wp, vbuf, vsiz); + wp += vsiz; + writevarnum(wp, psiz); + } + /** append a value */ + void append(char* rbuf, int32_t oh, const char* vbuf, size_t vsiz, size_t psiz) { + _assert_(rbuf && vbuf && vsiz <= MEMMAXSIZ && psiz <= MEMMAXSIZ); + char* wp = rbuf + sizeof(child_) + sizevarnum(ksiz_) + ksiz_; + if (oh > 0) { + char* pv = wp + sizevarnum(vsiz_); + std::memmove(pv + oh, pv, vsiz_); + wp += writevarnum(wp, vsiz_ + vsiz); + wp = pv + oh + vsiz_; + } else { + wp += writevarnum(wp, vsiz_ + vsiz); + wp += vsiz_; + } + std::memcpy(wp, vbuf, vsiz); + wp += vsiz; + writevarnum(wp, psiz); + } + /** concatenate two values */ + void concatenate(char* rbuf, const char* ovbuf, size_t ovsiz, + const char* nvbuf, size_t nvsiz, size_t psiz) { + _assert_(rbuf && ovbuf && ovsiz <= MEMMAXSIZ && nvbuf && nvsiz <= MEMMAXSIZ); + char* wp = rbuf + sizeof(child_) + sizevarnum(ksiz_) + ksiz_; + wp += writevarnum(wp, ovsiz + nvsiz); + std::memcpy(wp, ovbuf, ovsiz); + wp += ovsiz; + std::memcpy(wp, nvbuf, nvsiz); + wp += nvsiz; + writevarnum(wp, psiz); + } + /** serialize data into a buffer */ + char* serialize() { + _assert_(true); + uint64_t rsiz = sizeof(child_) + sizevarnum(ksiz_) + ksiz_ + sizevarnum(vsiz_) + vsiz_ + + sizevarnum(psiz_) + psiz_; + char* rbuf = new char[rsiz]; + char* wp = rbuf; + *(char**)wp = child_; + wp += sizeof(child_); + wp += writevarnum(wp, ksiz_); + std::memcpy(wp, kbuf_, ksiz_); + wp += ksiz_; + wp += writevarnum(wp, vsiz_); + std::memcpy(wp, vbuf_, vsiz_); + wp += vsiz_; + writevarnum(wp, psiz_); + return rbuf; + } + /** deserialize a buffer into object */ + void deserialize(const char* rbuf) { + _assert_(rbuf); + const char* rp = rbuf; + child_ = *(char**)rp; + rp += sizeof(child_); + rp += readvarnum(rp, sizeof(ksiz_), &ksiz_); + kbuf_ = rp; + rp += ksiz_; + rp += readvarnum(rp, sizeof(vsiz_), &vsiz_); + vbuf_ = rp; + rp += vsiz_; + readvarnum(rp, sizeof(psiz_), &psiz_); + } + char* child_; ///< region of the child + const char* kbuf_; ///< region of the key + uint64_t ksiz_; ///< size of the key + const char* vbuf_; ///< region of the value + uint64_t vsiz_; ///< size of the key + uint64_t psiz_; ///< size of the padding + }; + /** + * Comparator for records. + */ + struct RecordComparator { + /** comparing operator */ + bool operator ()(char* const& abuf, char* const& bbuf) { + const char* akbuf = abuf + sizeof(char**); + uint64_t aksiz; + akbuf += readvarnum(akbuf, sizeof(aksiz), &aksiz); + const char* bkbuf = bbuf + sizeof(char**); + uint64_t bksiz; + bkbuf += readvarnum(bkbuf, sizeof(bksiz), &bksiz); + uint64_t msiz = aksiz < bksiz ? aksiz : bksiz; + for (uint64_t i = 0; i < msiz; i++) { + if (((uint8_t*)akbuf)[i] != ((uint8_t*)bkbuf)[i]) + return ((uint8_t*)akbuf)[i] < ((uint8_t*)bkbuf)[i]; + } + return (int32_t)aksiz < (int32_t)bksiz; + } + }; + /** + * Initialize fields. + */ + void initialize() { + _assert_(true); + if (bnum_ >= MAPZMAPBNUM) { + buckets_ = (char**)mapalloc(sizeof(*buckets_) * bnum_); + } else { + buckets_ = new char*[bnum_]; + for (size_t i = 0; i < bnum_; i++) { + buckets_[i] = NULL; + } + } + } + /** + * Clean up fields. + */ + void destroy() { + _assert_(true); + for (size_t i = 0; i < bnum_; i++) { + char* rbuf = buckets_[i]; + while (rbuf) { + Record rec(rbuf); + char* child = rec.child_; + delete[] rbuf; + rbuf = child; + } + } + if (bnum_ >= MAPZMAPBNUM) { + mapfree(buckets_); + } else { + delete[] buckets_; + } + } + /** Dummy constructor to forbid the use. */ + TinyHashMap(const TinyHashMap&); + /** Dummy Operator to forbid the use. */ + TinyHashMap& operator =(const TinyHashMap&); + /** The bucket array. */ + char** buckets_; + /** The number of buckets. */ + size_t bnum_; + /** The number of records. */ + size_t count_; +}; + + +/** + * Memory-saving string array list. + */ +class TinyArrayList { + public: + /** + * Default constructor. + */ + explicit TinyArrayList() : recs_() { + _assert_(true); + } + /** + * Destructor. + */ + ~TinyArrayList() { + _assert_(true); + std::deque<char*>::iterator it = recs_.begin(); + std::deque<char*>::iterator itend = recs_.end(); + while (it != itend) { + delete[] *it; + ++it; + } + } + /** + * Insert a record at the bottom of the list. + * @param buf the pointer to the record region. + * @param size the size of the record region. + */ + void push(const char* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + size_t rsiz = sizevarnum(size) + size; + char* rbuf = new char[rsiz]; + char* wp = rbuf + writevarnum(rbuf, size); + std::memcpy(wp, buf, size); + recs_.push_back(rbuf); + } + /** + * Remove a record at the bottom of the list. + * @return true if the operation success, or false if there is no record in the list. + */ + bool pop() { + _assert_(true); + if (recs_.empty()) return false; + delete[] recs_.back(); + recs_.pop_back(); + return true; + } + /** + * Insert a record at the top of the list. + * @param buf the pointer to the record region. + * @param size the size of the record region. + */ + void unshift(const char* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + size_t rsiz = sizevarnum(size) + size; + char* rbuf = new char[rsiz]; + char* wp = rbuf + writevarnum(rbuf, size); + std::memcpy(wp, buf, size); + recs_.push_front(rbuf); + } + /** + * Remove a record at the top of the list. + * @return true if the operation success, or false if there is no record in the list. + */ + bool shift() { + _assert_(true); + if (recs_.empty()) return false; + delete[] recs_.front(); + recs_.pop_front(); + return true; + } + /** + * Insert a record at the position of the given index of the list. + * @param buf the pointer to the record region. + * @param size the size of the record region. + * @param idx the index of the position. It must be equal to or less than the number of + * records. + */ + void insert(const char* buf, size_t size, size_t idx) { + size_t rsiz = sizevarnum(size) + size; + char* rbuf = new char[rsiz]; + char* wp = rbuf + writevarnum(rbuf, size); + std::memcpy(wp, buf, size); + recs_.insert(recs_.begin() + idx, rbuf); + } + /** + * Remove a record at the position of the given index of the list. + * @param idx the index of the position. It must be less than the number of records. + */ + void remove(size_t idx) { + _assert_(true); + std::deque<char*>::iterator it = recs_.begin() + idx; + delete[] *it; + recs_.erase(it); + } + /** + * Retrieve a record at the position of the given index of the list. + * @param idx the index of the position. It must be less than the number of records. + * @param sp the pointer to the variable into which the size of the region of the return + * value is assigned. + * @return the pointer to the region of the retrieved record. + */ + const char* get(size_t idx, size_t* sp) { + _assert_(sp); + const char* rbuf = recs_[idx]; + uint64_t rsiz; + const char* rp = rbuf + readvarnum(rbuf, sizeof(uint64_t), &rsiz); + *sp = rsiz; + return rp; + } + /** + * Remove all records. + */ + void clear() { + _assert_(true); + std::deque<char*>::iterator it = recs_.begin(); + std::deque<char*>::iterator itend = recs_.end(); + while (it != itend) { + delete[] *it; + ++it; + } + recs_.clear(); + } + /** + * Get the number of records. + * @return the number of records. + */ + size_t count() { + _assert_(true); + return recs_.size(); + } + private: + /** Dummy constructor to forbid the use. */ + TinyArrayList(const TinyArrayList&); + /** Dummy Operator to forbid the use. */ + TinyArrayList& operator =(const TinyArrayList&); + /** The record list. */ + std::deque<char*> recs_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcplantdb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcplantdb.cc new file mode 100644 index 0000000000..f6c0f2ce18 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcplantdb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Plant database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcplantdb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcplantdb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcplantdb.h new file mode 100644 index 0000000000..83750db5f7 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcplantdb.h @@ -0,0 +1,3943 @@ +/************************************************************************************************* + * Plant database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCPLANTDB_H // duplication check +#define _KCPLANTDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> + +#define KCPDBMETAKEY "@" ///< key of the record for meta data +#define KCPDBTMPPATHEXT "tmpkct" ///< extension of the temporary file +#define KCPDRECBUFSIZ 128 ///< size of the record buffer + +namespace kyotocabinet { // common namespace + + +/** + * Plant database. + * @param BASEDB a class compatible with the file hash database class. + * @param DBTYPE the database type number of the class. + * @note This class template is a template for concrete classes to operate tree databases. + * Template instance classes can be inherited but overwriting methods is forbidden. The class + * TreeDB is the instance of the file tree database. The class ForestDB is the instance of the + * directory tree database. Before every database operation, it is necessary to call the + * BasicDB::open method in order to open a database file and connect the database object to it. + * To avoid data missing or corruption, it is important to close every database file by the + * BasicDB::close method when the database is no longer in use. It is forbidden for multible + * database objects in a process to open the same database at the same time. It is forbidden to + * share a database object with child processes. + */ +template <class BASEDB, uint8_t DBTYPE> +class PlantDB : public BasicDB { + public: + class Cursor; + private: + struct Record; + struct RecordComparator; + struct LeafNode; + struct Link; + struct LinkComparator; + struct InnerNode; + struct LeafSlot; + struct InnerSlot; + class ScopedVisitor; + /** An alias of array of records. */ + typedef std::vector<Record*> RecordArray; + /** An alias of array of records. */ + typedef std::vector<Link*> LinkArray; + /** An alias of leaf node cache. */ + typedef LinkedHashMap<int64_t, LeafNode*> LeafCache; + /** An alias of inner node cache. */ + typedef LinkedHashMap<int64_t, InnerNode*> InnerCache; + /** An alias of list of cursors. */ + typedef std::list<Cursor*> CursorList; + /** The number of cache slots. */ + static const int32_t SLOTNUM = 16; + /** The default alignment power. */ + static const uint8_t DEFAPOW = 8; + /** The default free block pool power. */ + static const uint8_t DEFFPOW = 10; + /** The default bucket number. */ + static const int64_t DEFBNUM = 64LL << 10; + /** The default page size. */ + static const int32_t DEFPSIZ = 8192; + /** The default capacity size of the page cache. */ + static const int64_t DEFPCCAP = 64LL << 20; + /** The size of the header. */ + static const int64_t HEADSIZ = 80; + /** The offset of the numbers. */ + static const int64_t MOFFNUMS = 8; + /** The prefix of leaf nodes. */ + static const char LNPREFIX = 'L'; + /** The prefix of inner nodes. */ + static const char INPREFIX = 'I'; + /** The average number of ways of each node. */ + static const size_t AVGWAY = 16; + /** The ratio of the warm cache. */ + static const size_t WARMRATIO = 4; + /** The ratio of flushing inner nodes. */ + static const size_t INFLRATIO = 32; + /** The default number of items in each leaf node. */ + static const size_t DEFLINUM = 64; + /** The default number of items in each inner node. */ + static const size_t DEFIINUM = 128; + /** The base ID number for inner nodes. */ + static const int64_t INIDBASE = 1LL << 48; + /** The minimum number of links in each inner node. */ + static const size_t INLINKMIN = 8; + /** The maximum level of B+ tree. */ + static const int32_t LEVELMAX = 16; + /** The number of cached nodes for auto transaction. */ + static const int32_t ATRANCNUM = 256; + /** The threshold of busy loop and sleep for locking. */ + static const uint32_t LOCKBUSYLOOP = 8192; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class PlantDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(PlantDB* db) : + db_(db), stack_(), kbuf_(NULL), ksiz_(0), lid_(0), back_(false) { + _assert_(db); + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.push_back(this); + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + if (!db_) return; + ScopedRWLock lock(&db_->mlock_, true); + if (kbuf_) clear_position(); + db_->curs_.remove(this); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + bool wrlock = writable && (db_->tran_ || db_->autotran_); + if (wrlock) { + db_->mlock_.lock_writer(); + } else { + db_->mlock_.lock_reader(); + } + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + db_->mlock_.unlock(); + return false; + } + if (writable && !(db_->writer_)) { + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + db_->mlock_.unlock(); + return false; + } + if (!kbuf_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + db_->mlock_.unlock(); + return false; + } + bool err = false; + bool hit = false; + + + if (lid_ > 0 && !accept_spec(visitor, writable, step, &hit)) err = true; + + + if (!err && !hit) { + if (!wrlock) { + db_->mlock_.unlock(); + db_->mlock_.lock_writer(); + } + if (kbuf_) { + bool retry = true; + while (!err && retry) { + if (!accept_atom(visitor, step, &retry)) err = true; + } + } else { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + err = true; + } + } + db_->mlock_.unlock(); + return !err; + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, false); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + back_ = false; + if (kbuf_) clear_position(); + bool err = false; + if (!set_position(db_->first_)) err = true; + return !err; + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, false); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + back_ = false; + if (kbuf_) clear_position(); + set_position(kbuf, ksiz, 0); + bool err = false; + if (!adjust_position()) { + if (kbuf_) clear_position(); + err = true; + } + return !err; + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + return jump(key.c_str(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, may provide a dummy implementation. + */ + bool jump_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, false); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + back_ = true; + if (kbuf_) clear_position(); + bool err = false; + if (!set_position_back(db_->last_)) err = true; + return !err; + } + /** + * Jump the cursor to a record for backward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, false); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + back_ = true; + if (kbuf_) clear_position(); + set_position(kbuf, ksiz, 0); + bool err = false; + if (adjust_position()) { + if (db_->reccomp_.comp->compare(kbuf, ksiz, kbuf_, ksiz_) < 0) { + bool hit = false; + if (lid_ > 0 && !back_position_spec(&hit)) err = true; + if (!err && !hit) { + db_->mlock_.unlock(); + db_->mlock_.lock_writer(); + if (kbuf_) { + if (!back_position_atom()) err = true; + } else { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + err = true; + } + } + } + } else { + if (kbuf_) clear_position(); + if (!set_position_back(db_->last_)) err = true; + } + return !err; + } + /** + * Jump the cursor to a record for backward scan. + * @note Equal to the original Cursor::jump_back method except that the parameter is + * std::string. + */ + bool jump_back(const std::string& key) { + _assert_(true); + return jump_back(key.c_str(), key.size()); + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + back_ = false; + DB::Visitor visitor; + if (!accept(&visitor, false, true)) return false; + if (!kbuf_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Step the cursor to the previous record. + * @return true on success, or false on failure. + */ + bool step_back() { + _assert_(true); + db_->mlock_.lock_reader(); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + db_->mlock_.unlock(); + return false; + } + if (!kbuf_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + db_->mlock_.unlock(); + return false; + } + back_ = true; + bool err = false; + bool hit = false; + if (lid_ > 0 && !back_position_spec(&hit)) err = true; + if (!err && !hit) { + db_->mlock_.unlock(); + db_->mlock_.lock_writer(); + if (kbuf_) { + if (!back_position_atom()) err = true; + } else { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + err = true; + } + } + db_->mlock_.unlock(); + return !err; + } + /** + * Get the database object. + * @return the database object. + */ + PlantDB* db() { + _assert_(true); + return db_; + } + private: + /** + * Clear the position. + */ + void clear_position() { + _assert_(true); + if (kbuf_ != stack_) delete[] kbuf_; + kbuf_ = NULL; + lid_ = 0; + } + /** + * Set the current position. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param id the ID of the current node. + */ + void set_position(const char* kbuf, size_t ksiz, int64_t id) { + _assert_(kbuf); + kbuf_ = ksiz > sizeof(stack_) ? new char[ksiz] : stack_; + ksiz_ = ksiz; + std::memcpy(kbuf_, kbuf, ksiz); + lid_ = id; + } + /** + * Set the current position with a record. + * @param rec the current record. + * @param id the ID of the current node. + */ + void set_position(Record* rec, int64_t id) { + _assert_(rec); + char* dbuf = (char*)rec + sizeof(*rec); + set_position(dbuf, rec->ksiz, id); + } + /** + * Set the current position at the next node. + * @param id the ID of the next node. + * @return true on success, or false on failure. + */ + bool set_position(int64_t id) { + _assert_(true); + while (id > 0) { + LeafNode* node = db_->load_leaf_node(id, false); + if (!node) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "missing leaf node"); + db_->db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)id); + return false; + } + ScopedRWLock lock(&node->lock, false); + RecordArray& recs = node->recs; + if (!recs.empty()) { + set_position(recs.front(), id); + return true; + } else { + id = node->next; + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + /** + * Set the current position at the previous node. + * @param id the ID of the previous node. + * @return true on success, or false on failure. + */ + bool set_position_back(int64_t id) { + _assert_(true); + while (id > 0) { + LeafNode* node = db_->load_leaf_node(id, false); + if (!node) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "missing leaf node"); + db_->db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)id); + return false; + } + ScopedRWLock lock(&node->lock, false); + RecordArray& recs = node->recs; + if (!recs.empty()) { + set_position(recs.back(), id); + return true; + } else { + id = node->prev; + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + /** + * Accept a visitor to the current record speculatively. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @param hitp the pointer to the variable for the hit flag. + * @return true on success, or false on failure. + */ + bool accept_spec(Visitor* visitor, bool writable, bool step, bool* hitp) { + _assert_(visitor && hitp); + bool err = false; + bool hit = false; + char rstack[KCPDRECBUFSIZ]; + size_t rsiz = sizeof(Record) + ksiz_; + char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + Record* rec = (Record*)rbuf; + rec->ksiz = ksiz_; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); + LeafNode* node = db_->load_leaf_node(lid_, false); + if (node) { + char lstack[KCPDRECBUFSIZ]; + char* lbuf = NULL; + size_t lsiz = 0; + Link* link = NULL; + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + if (writable) { + node->lock.lock_writer(); + } else { + node->lock.lock_reader(); + } + RecordArray& recs = node->recs; + if (!recs.empty()) { + Record* frec = recs.front(); + Record* lrec = recs.back(); + if (!db_->reccomp_(rec, frec) && !db_->reccomp_(lrec, rec)) { + typename RecordArray::iterator ritend = recs.end(); + typename RecordArray::iterator rit = std::lower_bound(recs.begin(), ritend, + rec, db_->reccomp_); + if (rit != ritend) { + hit = true; + if (db_->reccomp_(rec, *rit)) { + clear_position(); + set_position(*rit, node->id); + if (rbuf != rstack) delete[] rbuf; + rsiz = sizeof(Record) + ksiz_; + rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + rec = (Record*)rbuf; + rec->ksiz = ksiz_; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); + } + rec = *rit; + char* kbuf = (char*)rec + sizeof(*rec); + size_t ksiz = rec->ksiz; + size_t vsiz; + const char* vbuf = visitor->visit_full(kbuf, ksiz, kbuf + ksiz, + rec->vsiz, &vsiz); + if (vbuf == Visitor::REMOVE) { + rsiz = sizeof(*rec) + rec->ksiz + rec->vsiz; + db_->count_ -= 1; + db_->cusage_ -= rsiz; + node->size -= rsiz; + node->dirty = true; + if (recs.size() <= 1) { + lsiz = sizeof(Link) + ksiz; + lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz; + std::memcpy(lbuf + sizeof(*link), kbuf, ksiz); + } + xfree(rec); + if (back_) { + if (rit == recs.begin()) { + step = true; + } else { + typename RecordArray::iterator ritprev = rit - 1; + set_position(*ritprev, node->id); + step = false; + } + } else { + typename RecordArray::iterator ritnext = rit + 1; + if (ritnext == ritend) { + step = true; + } else { + clear_position(); + set_position(*ritnext, node->id); + step = false; + } + } + recs.erase(rit); + } else if (vbuf != Visitor::NOP) { + int64_t diff = (int64_t)vsiz - (int64_t)rec->vsiz; + db_->cusage_ += diff; + node->size += diff; + node->dirty = true; + if (vsiz > rec->vsiz) { + *rit = (Record*)xrealloc(rec, sizeof(*rec) + rec->ksiz + vsiz); + rec = *rit; + kbuf = (char*)rec + sizeof(*rec); + } + std::memcpy(kbuf + rec->ksiz, vbuf, vsiz); + rec->vsiz = vsiz; + if (node->size > db_->psiz_ && recs.size() > 1) { + lsiz = sizeof(Link) + ksiz; + lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz; + std::memcpy(lbuf + sizeof(*link), kbuf, ksiz); + } + } + if (step) { + if (back_) { + if (rit != recs.begin()) { + --rit; + set_position(*rit, node->id); + step = false; + } + } else { + ++rit; + if (rit != ritend) { + clear_position(); + set_position(*rit, node->id); + step = false; + } + } + } + } + } + } + bool atran = db_->autotran_ && !db_->tran_ && node->dirty; + bool async = db_->autosync_ && !db_->autotran_ && !db_->tran_ && node->dirty; + node->lock.unlock(); + if (hit && step) { + clear_position(); + if (back_) { + set_position_back(node->prev); + } else { + set_position(node->next); + } + } + if (hit) { + bool flush = db_->cusage_ > db_->pccap_; + if (link || flush || async) { + int64_t id = node->id; + if (atran && !link && !db_->fix_auto_transaction_leaf(node)) err = true; + db_->mlock_.unlock(); + db_->mlock_.lock_writer(); + if (link) { + node = db_->search_tree(link, true, hist, &hnum); + if (node) { + if (!db_->reorganize_tree(node, hist, hnum)) err = true; + if (atran && !db_->tran_ && !db_->fix_auto_transaction_tree()) err = true; + } else { + db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + err = true; + } + } else if (flush) { + int32_t idx = id % SLOTNUM; + LeafSlot* lslot = db_->lslots_ + idx; + if (!db_->flush_leaf_cache_part(lslot)) err = true; + InnerSlot* islot = db_->islots_ + idx; + if (islot->warm->count() > lslot->warm->count() + lslot->hot->count() + 1 && + !db_->flush_inner_cache_part(islot)) err = true; + } + if (async && !db_->fix_auto_synchronization()) err = true; + } else { + if (!db_->fix_auto_transaction_leaf(node)) err = true; + } + } + if (lbuf != lstack) delete[] lbuf; + } + if (rbuf != rstack) delete[] rbuf; + *hitp = hit; + return !err; + } + /** + * Accept a visitor to the current record atomically. + * @param visitor a visitor object. + * @param step true to move the cursor to the next record, or false for no move. + * @param retryp the pointer to the variable for the retry flag. + * @return true on success, or false on failure. + */ + bool accept_atom(Visitor* visitor, bool step, bool *retryp) { + _assert_(visitor && retryp); + bool err = false; + bool reorg = false; + *retryp = false; + char lstack[KCPDRECBUFSIZ]; + size_t lsiz = sizeof(Link) + ksiz_; + char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + Link* link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz_; + std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + LeafNode* node = db_->search_tree(link, true, hist, &hnum); + if (!node) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + if (lbuf != lstack) delete[] lbuf; + return false; + } + if (node->recs.empty()) { + if (lbuf != lstack) delete[] lbuf; + clear_position(); + if (!set_position(node->next)) return false; + node = db_->load_leaf_node(lid_, false); + if (!node) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + return false; + } + lsiz = sizeof(Link) + ksiz_; + char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + Link* link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz_; + std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); + node = db_->search_tree(link, true, hist, &hnum); + if (node->id != lid_) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "invalid tree"); + if (lbuf != lstack) delete[] lbuf; + return false; + } + } + char rstack[KCPDRECBUFSIZ]; + size_t rsiz = sizeof(Record) + ksiz_; + char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + Record* rec = (Record*)rbuf; + rec->ksiz = ksiz_; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); + RecordArray& recs = node->recs; + typename RecordArray::iterator ritend = recs.end(); + typename RecordArray::iterator rit = std::lower_bound(recs.begin(), ritend, + rec, db_->reccomp_); + if (rit != ritend) { + if (db_->reccomp_(rec, *rit)) { + clear_position(); + set_position(*rit, node->id); + if (rbuf != rstack) delete[] rbuf; + rsiz = sizeof(Record) + ksiz_; + rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + rec = (Record*)rbuf; + rec->ksiz = ksiz_; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); + } + rec = *rit; + char* kbuf = (char*)rec + sizeof(*rec); + size_t ksiz = rec->ksiz; + size_t vsiz; + const char* vbuf = visitor->visit_full(kbuf, ksiz, kbuf + ksiz, + rec->vsiz, &vsiz); + if (vbuf == Visitor::REMOVE) { + rsiz = sizeof(*rec) + rec->ksiz + rec->vsiz; + db_->count_ -= 1; + db_->cusage_ -= rsiz; + node->size -= rsiz; + node->dirty = true; + xfree(rec); + step = false; + clear_position(); + if (back_) { + if (rit == recs.begin()) { + set_position_back(node->prev); + } else { + typename RecordArray::iterator ritprev = rit - 1; + set_position(*ritprev, node->id); + } + } else { + typename RecordArray::iterator ritnext = rit + 1; + if (ritnext == ritend) { + set_position(node->next); + } else { + set_position(*ritnext, node->id); + } + } + recs.erase(rit); + if (recs.empty()) reorg = true; + } else if (vbuf != Visitor::NOP) { + int64_t diff = (int64_t)vsiz - (int64_t)rec->vsiz; + db_->cusage_ += diff; + node->size += diff; + node->dirty = true; + if (vsiz > rec->vsiz) { + *rit = (Record*)xrealloc(rec, sizeof(*rec) + rec->ksiz + vsiz); + rec = *rit; + kbuf = (char*)rec + sizeof(*rec); + } + std::memcpy(kbuf + rec->ksiz, vbuf, vsiz); + rec->vsiz = vsiz; + if (node->size > db_->psiz_ && recs.size() > 1) reorg = true; + } + if (step) { + clear_position(); + if (back_) { + if (rit == recs.begin()) { + set_position_back(node->prev); + } else { + --rit; + set_position(*rit, node->id); + } + } else { + ++rit; + if (rit == ritend) { + set_position(node->next); + } else { + set_position(*rit, node->id); + } + } + } + bool atran = db_->autotran_ && !db_->tran_ && node->dirty; + bool async = db_->autosync_ && !db_->autotran_ && !db_->tran_ && node->dirty; + if (atran && !reorg && !db_->fix_auto_transaction_leaf(node)) err = true; + if (reorg) { + if (!db_->reorganize_tree(node, hist, hnum)) err = true; + if (atran && !db_->fix_auto_transaction_tree()) err = true; + } else if (db_->cusage_ > db_->pccap_) { + int32_t idx = node->id % SLOTNUM; + LeafSlot* lslot = db_->lslots_ + idx; + if (!db_->flush_leaf_cache_part(lslot)) err = true; + InnerSlot* islot = db_->islots_ + idx; + if (islot->warm->count() > lslot->warm->count() + lslot->hot->count() + 1 && + !db_->flush_inner_cache_part(islot)) err = true; + } + if (async && !db_->fix_auto_synchronization()) err = true; + } else { + int64_t lid = lid_; + clear_position(); + if (back_) { + if (set_position_back(node->prev)) { + if (lid_ == lid) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "invalid leaf node"); + err = true; + } else { + *retryp = true; + } + } else { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + err = true; + } + } else { + if (set_position(node->next)) { + if (lid_ == lid) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "invalid leaf node"); + err = true; + } else { + *retryp = true; + } + } else { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + err = true; + } + } + } + if (rbuf != rstack) delete[] rbuf; + if (lbuf != lstack) delete[] lbuf; + return !err; + } + /** + * Adjust the position to an existing record. + * @return true on success, or false on failure. + */ + bool adjust_position() { + _assert_(true); + char lstack[KCPDRECBUFSIZ]; + size_t lsiz = sizeof(Link) + ksiz_; + char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + Link* link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz_; + std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + LeafNode* node = db_->search_tree(link, true, hist, &hnum); + if (!node) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + if (lbuf != lstack) delete[] lbuf; + return false; + } + char rstack[KCPDRECBUFSIZ]; + size_t rsiz = sizeof(Record) + ksiz_; + char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + Record* rec = (Record*)rbuf; + rec->ksiz = ksiz_; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); + bool err = false; + node->lock.lock_reader(); + const RecordArray& recs = node->recs; + typename RecordArray::const_iterator ritend = node->recs.end(); + typename RecordArray::const_iterator rit = std::lower_bound(recs.begin(), ritend, + rec, db_->reccomp_); + clear_position(); + if (rit == ritend) { + node->lock.unlock(); + if (!set_position(node->next)) err = true; + } else { + set_position(*rit, node->id); + node->lock.unlock(); + } + if (rbuf != rstack) delete[] rbuf; + if (lbuf != lstack) delete[] lbuf; + return !err; + } + /** + * Back the position to the previous record speculatively. + * @param hitp the pointer to the variable for the hit flag. + * @return true on success, or false on failure. + */ + bool back_position_spec(bool* hitp) { + _assert_(hitp); + bool err = false; + bool hit = false; + char rstack[KCPDRECBUFSIZ]; + size_t rsiz = sizeof(Record) + ksiz_; + char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + Record* rec = (Record*)rbuf; + rec->ksiz = ksiz_; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); + LeafNode* node = db_->load_leaf_node(lid_, false); + if (node) { + node->lock.lock_reader(); + RecordArray& recs = node->recs; + if (recs.empty()) { + node->lock.unlock(); + } else { + Record* frec = recs.front(); + Record* lrec = recs.back(); + if (db_->reccomp_(rec, frec)) { + hit = true; + clear_position(); + node->lock.unlock(); + if (!set_position_back(node->prev)) err = true; + } else if (db_->reccomp_(lrec, rec)) { + node->lock.unlock(); + } else { + hit = true; + typename RecordArray::iterator ritbeg = recs.begin(); + typename RecordArray::iterator ritend = recs.end(); + typename RecordArray::iterator rit = std::lower_bound(recs.begin(), ritend, + rec, db_->reccomp_); + clear_position(); + if (rit == ritbeg) { + node->lock.unlock(); + if (!set_position_back(node->prev)) err = true; + } else { + --rit; + set_position(*rit, node->id); + node->lock.unlock(); + } + } + } + } + if (rbuf != rstack) delete[] rbuf; + *hitp = hit; + return !err; + } + /** + * Back the position to the previous record atomically. + * @return true on success, or false on failure. + */ + bool back_position_atom() { + _assert_(true); + char lstack[KCPDRECBUFSIZ]; + size_t lsiz = sizeof(Link) + ksiz_; + char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + Link* link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz_; + std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + LeafNode* node = db_->search_tree(link, true, hist, &hnum); + if (!node) { + db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + if (lbuf != lstack) delete[] lbuf; + return false; + } + char rstack[KCPDRECBUFSIZ]; + size_t rsiz = sizeof(Record) + ksiz_; + char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + Record* rec = (Record*)rbuf; + rec->ksiz = ksiz_; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); + bool err = false; + node->lock.lock_reader(); + const RecordArray& recs = node->recs; + typename RecordArray::const_iterator ritbeg = node->recs.begin(); + typename RecordArray::const_iterator ritend = node->recs.end(); + typename RecordArray::const_iterator rit = std::lower_bound(recs.begin(), ritend, + rec, db_->reccomp_); + clear_position(); + if (rit == ritbeg) { + node->lock.unlock(); + if (!set_position_back(node->prev)) err = true; + } else if (rit == ritend) { + ritend--; + set_position(*ritend, node->id); + node->lock.unlock(); + } else { + --rit; + set_position(*rit, node->id); + node->lock.unlock(); + } + if (rbuf != rstack) delete[] rbuf; + if (lbuf != lstack) delete[] lbuf; + return !err; + } + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + PlantDB* db_; + /** The stack buffer for the key. */ + char stack_[KCPDRECBUFSIZ]; + /** The pointer to the key region. */ + char* kbuf_; + /** The size of the key region. */ + size_t ksiz_; + /** The last visited leaf. */ + int64_t lid_; + /** The backward flag. */ + bool back_; + }; + /** + * Tuning options. + */ + enum Option { + TSMALL = BASEDB::TSMALL, ///< use 32-bit addressing + TLINEAR = BASEDB::TLINEAR, ///< use linear collision chaining + TCOMPRESS = BASEDB::TCOMPRESS ///< compress each record + }; + /** + * Status flags. + */ + enum Flag { + FOPEN = BASEDB::FOPEN, ///< whether opened + FFATAL = BASEDB::FFATAL ///< whether with fatal error + }; + /** + * Default constructor. + */ + explicit PlantDB() : + mlock_(), mtrigger_(NULL), omode_(0), writer_(false), autotran_(false), autosync_(false), + db_(), curs_(), apow_(DEFAPOW), fpow_(DEFFPOW), opts_(0), bnum_(DEFBNUM), + psiz_(DEFPSIZ), pccap_(DEFPCCAP), + root_(0), first_(0), last_(0), lcnt_(0), icnt_(0), count_(0), cusage_(0), + lslots_(), islots_(), reccomp_(), linkcomp_(), + tran_(false), trclock_(0), trlcnt_(0), trcount_(0) { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~PlantDB() { + _assert_(true); + if (omode_ != 0) close(); + if (!curs_.empty()) { + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->db_ = NULL; + ++cit; + } + } + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + bool wrlock = writable && (tran_ || autotran_); + if (wrlock) { + mlock_.lock_writer(); + } else { + mlock_.lock_reader(); + } + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + char lstack[KCPDRECBUFSIZ]; + size_t lsiz = sizeof(Link) + ksiz; + char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + Link* link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz; + std::memcpy(lbuf + sizeof(*link), kbuf, ksiz); + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + LeafNode* node = search_tree(link, true, hist, &hnum); + if (!node) { + set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + if (lbuf != lstack) delete[] lbuf; + mlock_.unlock(); + return false; + } + char rstack[KCPDRECBUFSIZ]; + size_t rsiz = sizeof(Record) + ksiz; + char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + Record* rec = (Record*)rbuf; + rec->ksiz = ksiz; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf, ksiz); + if (writable) { + node->lock.lock_writer(); + } else { + node->lock.lock_reader(); + } + bool reorg = accept_impl(node, rec, visitor); + bool atran = autotran_ && !tran_ && node->dirty; + bool async = autosync_ && !autotran_ && !tran_ && node->dirty; + node->lock.unlock(); + bool flush = false; + bool err = false; + int64_t id = node->id; + if (atran && !reorg && !fix_auto_transaction_leaf(node)) err = true; + if (cusage_ > pccap_) { + int32_t idx = id % SLOTNUM; + LeafSlot* lslot = lslots_ + idx; + if (!clean_leaf_cache_part(lslot)) err = true; + flush = true; + } + if (reorg) { + if (!wrlock) { + mlock_.unlock(); + mlock_.lock_writer(); + } + node = search_tree(link, false, hist, &hnum); + if (node) { + if (!reorganize_tree(node, hist, hnum)) err = true; + if (atran && !tran_ && !fix_auto_transaction_tree()) err = true; + } + mlock_.unlock(); + } else if (flush) { + if (!wrlock) { + mlock_.unlock(); + mlock_.lock_writer(); + } + int32_t idx = id % SLOTNUM; + LeafSlot* lslot = lslots_ + idx; + if (!flush_leaf_cache_part(lslot)) err = true; + InnerSlot* islot = islots_ + idx; + if (islot->warm->count() > lslot->warm->count() + lslot->hot->count() + 1 && + !flush_inner_cache_part(islot)) err = true; + mlock_.unlock(); + } else { + mlock_.unlock(); + } + if (rbuf != rstack) delete[] rbuf; + if (lbuf != lstack) delete[] lbuf; + if (async) { + mlock_.lock_writer(); + if (!fix_auto_synchronization()) err = true; + mlock_.unlock(); + } + return !err; + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + if (keys.empty()) return true; + bool err = false; + std::vector<std::string>::const_iterator kit = keys.begin(); + std::vector<std::string>::const_iterator kitend = keys.end(); + while (!err && kit != kitend) { + const char* kbuf = kit->data(); + size_t ksiz = kit->size(); + char lstack[KCPDRECBUFSIZ]; + size_t lsiz = sizeof(Link) + ksiz; + char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + Link* link = (Link*)lbuf; + link->child = 0; + link->ksiz = ksiz; + std::memcpy(lbuf + sizeof(*link), kbuf, ksiz); + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + LeafNode* node = search_tree(link, true, hist, &hnum); + if (!node) { + set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + if (lbuf != lstack) delete[] lbuf; + err = true; + break; + } + char rstack[KCPDRECBUFSIZ]; + size_t rsiz = sizeof(Record) + ksiz; + char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; + Record* rec = (Record*)rbuf; + rec->ksiz = ksiz; + rec->vsiz = 0; + std::memcpy(rbuf + sizeof(*rec), kbuf, ksiz); + bool reorg = accept_impl(node, rec, visitor); + bool atran = autotran_ && !tran_ && node->dirty; + bool async = autosync_ && !autotran_ && !tran_ && node->dirty; + if (atran && !reorg && !fix_auto_transaction_leaf(node)) err = true; + if (reorg) { + if (!reorganize_tree(node, hist, hnum)) err = true; + if (atran && !fix_auto_transaction_tree()) err = true; + } else if (cusage_ > pccap_) { + int32_t idx = node->id % SLOTNUM; + LeafSlot* lslot = lslots_ + idx; + if (!flush_leaf_cache_part(lslot)) err = true; + InnerSlot* islot = islots_ + idx; + if (islot->warm->count() > lslot->warm->count() + lslot->hot->count() + 1 && + !flush_inner_cache_part(islot)) err = true; + } + if (rbuf != rstack) delete[] rbuf; + if (lbuf != lstack) delete[] lbuf; + if (async && !fix_auto_synchronization()) err = true; + ++kit; + } + return !err; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + int64_t allcnt = count_; + if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + bool err = false; + bool atran = false; + if (autotran_ && writable && !tran_) { + if (begin_transaction_impl(autosync_)) { + atran = true; + } else { + err = true; + } + } + int64_t id = first_; + int64_t flcnt = 0; + int64_t curcnt = 0; + while (!err && id > 0) { + LeafNode* node = load_leaf_node(id, false); + if (!node) { + set_error(_KCCODELINE_, Error::BROKEN, "missing leaf node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)id); + return false; + } + id = node->next; + const RecordArray& recs = node->recs; + RecordArray keys; + keys.reserve(recs.size()); + typename RecordArray::const_iterator rit = recs.begin(); + typename RecordArray::const_iterator ritend = recs.end(); + while (rit != ritend) { + Record* rec = *rit; + size_t rsiz = sizeof(*rec) + rec->ksiz; + char* dbuf = (char*)rec + sizeof(*rec); + Record* key = (Record*)xmalloc(rsiz); + key->ksiz = rec->ksiz; + key->vsiz = 0; + char* kbuf = (char*)key + sizeof(*key); + std::memcpy(kbuf, dbuf, rec->ksiz); + keys.push_back(key); + ++rit; + } + typename RecordArray::const_iterator kit = keys.begin(); + typename RecordArray::const_iterator kitend = keys.end(); + bool reorg = false; + while (kit != kitend) { + Record* rec = *kit; + if (accept_impl(node, rec, visitor)) reorg = true; + curcnt++; + if (checker && !checker->check("iterate", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + break; + } + ++kit; + } + if (reorg) { + Record* rec = keys.front(); + char* dbuf = (char*)rec + sizeof(*rec); + char lstack[KCPDRECBUFSIZ]; + size_t lsiz = sizeof(Link) + rec->ksiz; + char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; + Link* link = (Link*)lbuf; + link->child = 0; + link->ksiz = rec->ksiz; + std::memcpy(lbuf + sizeof(*link), dbuf, rec->ksiz); + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + node = search_tree(link, false, hist, &hnum); + if (node) { + if (!reorganize_tree(node, hist, hnum)) err = true; + } else { + set_error(_KCCODELINE_, Error::BROKEN, "search failed"); + err = true; + } + if (lbuf != lstack) delete[] lbuf; + } + if (cusage_ > pccap_) { + for (int32_t i = 0; i < SLOTNUM; i++) { + LeafSlot* lslot = lslots_ + i; + if (!flush_leaf_cache_part(lslot)) err = true; + } + InnerSlot* islot = islots_ + (flcnt++) % SLOTNUM; + if (islot->warm->count() > 2 && !flush_inner_cache_part(islot)) err = true; + } + kit = keys.begin(); + while (kit != kitend) { + xfree(*kit); + ++kit; + } + } + if (checker && !checker->check("iterate", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + if (atran && !commit_transaction()) err = true; + if (autosync_ && !autotran_ && writable && !fix_auto_synchronization()) err = true; + trigger_meta(MetaTrigger::ITERATE, "iterate"); + return !err; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (thnum < 1) thnum = 0; + if (thnum > (size_t)INT8MAX) thnum = INT8MAX; + bool err = false; + if (writer_) { + if (checker && !checker->check("scan_parallel", "cleaning the leaf node cache", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!clean_leaf_cache()) err = true; + } + ScopedVisitor svis(visitor); + int64_t allcnt = count_; + if (checker && !checker->check("scan_parallel", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + class ProgressCheckerImpl : public ProgressChecker { + public: + explicit ProgressCheckerImpl() : ok_(1) {} + void stop() { + ok_.set(0); + } + private: + bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) { + return ok_ > 0; + } + AtomicInt64 ok_; + }; + ProgressCheckerImpl ichecker; + class VisitorImpl : public Visitor { + public: + explicit VisitorImpl(PlantDB* db, Visitor* visitor, + ProgressChecker* checker, int64_t allcnt, + ProgressCheckerImpl* ichecker) : + db_(db), visitor_(visitor), checker_(checker), allcnt_(allcnt), + ichecker_(ichecker), error_() {} + const Error& error() { + return error_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (ksiz < 2 || ksiz >= NUMBUFSIZ || kbuf[0] != LNPREFIX) return NOP; + uint64_t prev; + size_t step = readvarnum(vbuf, vsiz, &prev); + if (step < 1) return NOP; + vbuf += step; + vsiz -= step; + uint64_t next; + step = readvarnum(vbuf, vsiz, &next); + if (step < 1) return NOP; + vbuf += step; + vsiz -= step; + while (vsiz > 1) { + uint64_t rksiz; + step = readvarnum(vbuf, vsiz, &rksiz); + if (step < 1) break; + vbuf += step; + vsiz -= step; + uint64_t rvsiz; + step = readvarnum(vbuf, vsiz, &rvsiz); + if (step < 1) break; + vbuf += step; + vsiz -= step; + if (vsiz < rksiz + rvsiz) break; + size_t xvsiz; + visitor_->visit_full(vbuf, rksiz, vbuf + rksiz, rvsiz, &xvsiz); + vbuf += rksiz; + vsiz -= rksiz; + vbuf += rvsiz; + vsiz -= rvsiz; + if (checker_ && !checker_->check("scan_parallel", "processing", -1, allcnt_)) { + db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + error_ = db_->error(); + ichecker_->stop(); + break; + } + } + return NOP; + } + PlantDB* db_; + Visitor* visitor_; + ProgressChecker* checker_; + int64_t allcnt_; + ProgressCheckerImpl* ichecker_; + Error error_; + }; + VisitorImpl ivisitor(this, visitor, checker, allcnt, &ichecker); + if (!db_.scan_parallel(&ivisitor, thnum, &ichecker)) err = true; + if (ivisitor.error() != Error::SUCCESS) { + const Error& e = ivisitor.error(); + db_.set_error(_KCCODELINE_, e.code(), e.message()); + err = true; + } + if (checker && !checker->check("scan_parallel", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + trigger_meta(MetaTrigger::ITERATE, "scan_parallel"); + return !err; + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + return db_.error(); + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + db_.set_error(file, line, func, code, message); + } + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. BasicDB::OWRITER as a writer, BasicDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: BasicDB::OCREATE, + * which means it creates a new database if the file does not exist, BasicDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, BasicDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, BasicDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: BasicDB::ONOLOCK, which means it opens the database file without file locking, + * BasicDB::OTRYLOCK, which means locking is performed without blocking, BasicDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the BasicDB::close method when it is no + * longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", path.c_str()); + if (DBTYPE == TYPEGRASS) { + mode &= ~OREADER; + mode |= OWRITER | OCREATE; + } + writer_ = false; + autotran_ = false; + autosync_ = false; + if (mode & OWRITER) { + writer_ = true; + if (mode & OAUTOTRAN) autotran_ = true; + if (mode & OAUTOSYNC) autosync_ = true; + } + if (!db_.tune_type(DBTYPE)) return false; + if (!db_.tune_alignment(apow_)) return false; + if (!db_.tune_fbp(fpow_)) return false; + if (!db_.tune_options(opts_)) return false; + if (!db_.tune_buckets(bnum_)) return false; + if (!db_.open(path, mode)) return false; + if (db_.type() != DBTYPE) { + set_error(_KCCODELINE_, Error::INVALID, "invalid database type"); + db_.close(); + return false; + } + if (db_.reorganized()) { + if (!reorganize_file(mode)) return false; + } else if (db_.recovered()) { + if (!writer_) { + if (!db_.close()) return false; + uint32_t tmode = (mode & ~OREADER) | OWRITER; + if (!db_.open(path, tmode)) return false; + } + if (!recalc_count()) return false; + if (!writer_) { + if (!db_.close()) return false; + if (!db_.open(path, mode)) return false; + } + if (count_ == INT64MAX && !reorganize_file(mode)) return false; + } + if (writer_ && db_.count() < 1) { + root_ = 0; + first_ = 0; + last_ = 0; + count_ = 0; + create_leaf_cache(); + create_inner_cache(); + lcnt_ = 0; + create_leaf_node(0, 0); + root_ = 1; + first_ = 1; + last_ = 1; + lcnt_ = 1; + icnt_ = 0; + count_ = 0; + if (!reccomp_.comp) reccomp_.comp = LEXICALCOMP; + if (!dump_meta() || !flush_leaf_cache(true) || !load_meta()) { + delete_inner_cache(); + delete_leaf_cache(); + db_.close(); + return false; + } + } else { + if (!load_meta()) { + db_.close(); + return false; + } + create_leaf_cache(); + create_inner_cache(); + } + if (psiz_ < 1 || root_ < 1 || first_ < 1 || last_ < 1 || + lcnt_ < 1 || icnt_ < 0 || count_ < 0 || bnum_ < 1) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data"); + db_.report(_KCCODELINE_, Logger::WARN, "psiz=%lld root=%lld first=%lld last=%lld" + " lcnt=%lld icnt=%lld count=%lld bnum=%lld", + (long long)psiz_, (long long)root_, (long long)first_, (long long)last_, + (long long)lcnt_, (long long)icnt_, (long long)count_, (long long)bnum_); + delete_inner_cache(); + delete_leaf_cache(); + db_.close(); + return false; + } + omode_ = mode; + cusage_ = 0; + tran_ = false; + trclock_ = 0; + trigger_meta(MetaTrigger::OPEN, "open"); + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + const std::string& path = db_.path(); + report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", path.c_str()); + bool err = false; + disable_cursors(); + int64_t lsiz = calc_leaf_cache_size(); + int64_t isiz = calc_inner_cache_size(); + if (cusage_ != lsiz + isiz) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid cache usage"); + db_.report(_KCCODELINE_, Logger::WARN, "cusage=%lld lsiz=%lld isiz=%lld", + (long long)cusage_, (long long)lsiz, (long long)isiz); + err = true; + } + if (!flush_leaf_cache(true)) err = true; + if (!flush_inner_cache(true)) err = true; + lsiz = calc_leaf_cache_size(); + isiz = calc_inner_cache_size(); + int64_t lcnt = calc_leaf_cache_count(); + int64_t icnt = calc_inner_cache_count(); + if (cusage_ != 0 || lsiz != 0 || isiz != 0 || lcnt != 0 || icnt != 0) { + set_error(_KCCODELINE_, Error::BROKEN, "remaining cache"); + db_.report(_KCCODELINE_, Logger::WARN, "cusage=%lld lsiz=%lld isiz=%lld" + " lcnt=%lld icnt=%lld", (long long)cusage_, (long long)lsiz, (long long)isiz, + (long long)lcnt, (long long)icnt); + err = true; + } + delete_inner_cache(); + delete_leaf_cache(); + if (writer_ && !dump_meta()) err = true; + if (!db_.close()) err = true; + omode_ = 0; + trigger_meta(MetaTrigger::CLOSE, "close"); + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + mlock_.lock_reader(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + bool err = false; + if (writer_) { + if (checker && !checker->check("synchronize", "cleaning the leaf node cache", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + mlock_.unlock(); + return false; + } + if (!clean_leaf_cache()) err = true; + if (checker && !checker->check("synchronize", "cleaning the inner node cache", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + mlock_.unlock(); + return false; + } + if (!clean_inner_cache()) err = true; + mlock_.unlock(); + mlock_.lock_writer(); + if (checker && !checker->check("synchronize", "flushing the leaf node cache", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + mlock_.unlock(); + return false; + } + if (!flush_leaf_cache(true)) err = true; + if (checker && !checker->check("synchronize", "flushing the inner node cache", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + mlock_.unlock(); + return false; + } + if (!flush_inner_cache(true)) err = true; + if (checker && !checker->check("synchronize", "dumping the meta data", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + mlock_.unlock(); + return false; + } + if (!dump_meta()) err = true; + } + class Wrapper : public FileProcessor { + public: + Wrapper(FileProcessor* proc, int64_t count) : proc_(proc), count_(count) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + if (proc_) return proc_->process(path, count_, size); + return true; + } + FileProcessor* proc_; + int64_t count_; + } wrapper(proc, count_); + if (!db_.synchronize(hard, &wrapper, checker)) err = true; + trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize"); + mlock_.unlock(); + return !err; + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, writable); + bool err = false; + if (proc && !proc->process(db_.path(), count_, db_.size())) { + set_error(_KCCODELINE_, Error::LOGIC, "processing failed"); + err = true; + } + trigger_meta(MetaTrigger::OCCUPY, "occupy"); + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + uint32_t wcnt = 0; + while (true) { + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!tran_) break; + mlock_.unlock(); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + if (!begin_transaction_impl(hard)) { + mlock_.unlock(); + return false; + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); + mlock_.unlock(); + return true; + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (tran_) { + set_error(_KCCODELINE_, Error::LOGIC, "competition avoided"); + mlock_.unlock(); + return false; + } + if (!begin_transaction_impl(hard)) { + mlock_.unlock(); + return false; + } + tran_ = true; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try"); + mlock_.unlock(); + return true; + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!tran_) { + set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); + return false; + } + bool err = false; + if (commit) { + if (!commit_transaction()) err = true; + } else { + if (!abort_transaction()) err = true; + } + tran_ = false; + trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction"); + return !err; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + disable_cursors(); + flush_leaf_cache(false); + flush_inner_cache(false); + bool err = false; + if (!db_.clear()) err = true; + lcnt_ = 0; + create_leaf_node(0, 0); + root_ = 1; + first_ = 1; + last_ = 1; + lcnt_ = 1; + icnt_ = 0; + count_ = 0; + if (!dump_meta()) err = true; + if (!flush_leaf_cache(true)) err = true; + cusage_ = 0; + trigger_meta(MetaTrigger::CLEAR, "clear"); + return !err; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return count_; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return db_.size(); + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return ""; + } + return db_.path(); + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!db_.status(strmap)) return false; + (*strmap)["type"] = strprintf("%u", (unsigned)DBTYPE); + (*strmap)["psiz"] = strprintf("%d", psiz_); + (*strmap)["pccap"] = strprintf("%lld", (long long)pccap_); + const char* compname = "external"; + if (reccomp_.comp == LEXICALCOMP) { + compname = "lexical"; + } else if (reccomp_.comp == DECIMALCOMP) { + compname = "decimal"; + } else if (reccomp_.comp == LEXICALDESCCOMP) { + compname = "lexicaldesc"; + } else if (reccomp_.comp == DECIMALDESCCOMP) { + compname = "decimaldesc"; + } + (*strmap)["rcomp"] = compname; + (*strmap)["root"] = strprintf("%lld", (long long)root_); + (*strmap)["first"] = strprintf("%lld", (long long)first_); + (*strmap)["last"] = strprintf("%lld", (long long)last_); + (*strmap)["lcnt"] = strprintf("%lld", (long long)lcnt_); + (*strmap)["icnt"] = strprintf("%lld", (long long)icnt_); + (*strmap)["count"] = strprintf("%lld", (long long)count_); + (*strmap)["bnum"] = strprintf("%lld", (long long)bnum_); + (*strmap)["pnum"] = strprintf("%lld", (long long)db_.count()); + (*strmap)["cusage"] = strprintf("%lld", (long long)cusage_); + if (strmap->count("cusage_lcnt") > 0) + (*strmap)["cusage_lcnt"] = strprintf("%lld", (long long)calc_leaf_cache_count()); + if (strmap->count("cusage_lsiz") > 0) + (*strmap)["cusage_lsiz"] = strprintf("%lld", (long long)calc_leaf_cache_size()); + if (strmap->count("cusage_icnt") > 0) + (*strmap)["cusage_icnt"] = strprintf("%lld", (long long)calc_inner_cache_count()); + if (strmap->count("cusage_isiz") > 0) + (*strmap)["cusage_isiz"] = strprintf("%lld", (long long)calc_inner_cache_size()); + if (strmap->count("tree_level") > 0) { + Link link; + link.ksiz = 0; + int64_t hist[LEVELMAX]; + int32_t hnum = 0; + search_tree(&link, false, hist, &hnum); + (*strmap)["tree_level"] = strprintf("%d", hnum + 1); + } + return true; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + ScopedRWLock lock(&mlock_, false); + db_.log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + return db_.tune_logger(logger, kinds); + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + /** + * Set the power of the alignment of record size. + * @param apow the power of the alignment of record size. + * @return true on success, or false on failure. + */ + bool tune_alignment(int8_t apow) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + apow_ = apow >= 0 ? apow : DEFAPOW; + return true; + } + /** + * Set the power of the capacity of the free block pool. + * @param fpow the power of the capacity of the free block pool. + * @return true on success, or false on failure. + */ + bool tune_fbp(int8_t fpow) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + fpow_ = fpow >= 0 ? fpow : DEFFPOW; + return true; + } + /** + * Set the optional features. + * @param opts the optional features by bitwise-or: BasicDB::TSMALL to use 32-bit addressing, + * BasicDB::TLINEAR to use linear collision chaining, BasicDB::TCOMPRESS to compress each + * record. + * @return true on success, or false on failure. + */ + bool tune_options(int8_t opts) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + opts_ = opts; + return true; + } + /** + * Set the number of buckets of the hash table. + * @param bnum the number of buckets of the hash table. + * @return true on success, or false on failure. + */ + bool tune_buckets(int64_t bnum) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + bnum_ = bnum > 0 ? bnum : DEFBNUM; + return true; + } + /** + * Set the size of each page. + * @param psiz the size of each page. + * @return true on success, or false on failure. + */ + bool tune_page(int32_t psiz) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + psiz_ = psiz > 0 ? psiz : DEFPSIZ; + return true; + } + /** + * Set the size of the internal memory-mapped region. + * @param msiz the size of the internal memory-mapped region. + * @return true on success, or false on failure. + */ + bool tune_map(int64_t msiz) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + return db_.tune_map(msiz); + } + /** + * Set the unit step number of auto defragmentation. + * @param dfunit the unit step number of auto defragmentation. + * @return true on success, or false on failure. + */ + bool tune_defrag(int64_t dfunit) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + return db_.tune_defrag(dfunit); + } + /** + * Set the capacity size of the page cache. + * @param pccap the capacity size of the page cache. + * @return true on success, or false on failure. + */ + bool tune_page_cache(int64_t pccap) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + pccap_ = pccap > 0 ? pccap : DEFPCCAP; + return true; + } + /** + * Set the data compressor. + * @param comp the data compressor object. + * @return true on success, or false on failure. + */ + bool tune_compressor(Compressor* comp) { + _assert_(comp); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + return db_.tune_compressor(comp); + } + /** + * Set the record comparator. + * @param rcomp the record comparator object. + * @return true on success, or false on failure. + * @note Several built-in comparators are provided. LEXICALCOMP for the default lexical + * comparator. DECIMALCOMP for the decimal comparator. LEXICALDESCCOMP for the lexical + * descending comparator. DECIMALDESCCOMP for the lexical descending comparator. + */ + bool tune_comparator(Comparator* rcomp) { + _assert_(rcomp); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + reccomp_.comp = rcomp; + return true; + } + /** + * Get the opaque data. + * @return the pointer to the opaque data region, whose size is 16 bytes. + */ + char* opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return db_.opaque(); + } + /** + * Synchronize the opaque data. + * @return true on success, or false on failure. + */ + bool synchronize_opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_.synchronize_opaque(); + } + /** + * Perform defragmentation of the file. + * @param step the number of steps. If it is not more than 0, the whole region is defraged. + * @return true on success, or false on failure. + */ + bool defrag(int64_t step = 0) { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bool err = false; + if (step < 1 && writer_) { + if (!clean_leaf_cache()) err = true; + if (!clean_inner_cache()) err = true; + } + if (!db_.defrag(step)) err = true; + return !err; + } + /** + * Get the status flags. + * @return the status flags, or 0 on failure. + */ + uint8_t flags() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return db_.flags(); + } + /** + * Get the record comparator. + * @return the record comparator object. + */ + Comparator* rcomp() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return 0; + } + return reccomp_.comp; + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, ...) { + _assert_(file && line > 0 && func && format); + va_list ap; + va_start(ap, format); + db_.report_valist(file, line, func, kind, format, ap); + va_end(ap); + } + /** + * Report a message for debugging with variable number of arguments. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ap used according to the format string. + */ + void report_valist(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, va_list ap) { + _assert_(file && line > 0 && func && format); + db_.report_valist(file, line, func, kind, format, ap); + } + /** + * Report the content of a binary buffer for debugging. + * @param file the file name of the epicenter. + * @param line the line number of the epicenter. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param name the name of the information. + * @param buf the binary buffer. + * @param size the size of the binary buffer + */ + void report_binary(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* name, const char* buf, size_t size) { + _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ); + db_.report_binary(file, line, func, kind, name, buf, size); + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN for beginning + * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaTrigger::ABORTTRAN + * for aborting transaction, and MetaTrigger::MISC for miscellaneous operations. + * @param message the supplement message. + */ + void trigger_meta(MetaTrigger::Kind kind, const char* message) { + _assert_(message); + if (mtrigger_) mtrigger_->trigger(kind, message); + } + private: + /** + * Record data. + */ + struct Record { + uint32_t ksiz; ///< size of the key + uint32_t vsiz; ///< size of the value + }; + /** + * Comparator for records. + */ + struct RecordComparator { + Comparator* comp; ///< comparator + /** constructor */ + explicit RecordComparator() : comp(NULL) {} + /** comparing operator */ + bool operator ()(const Record* const& a, const Record* const& b) const { + _assert_(true); + char* akbuf = (char*)a + sizeof(*a); + char* bkbuf = (char*)b + sizeof(*b); + return comp->compare(akbuf, a->ksiz, bkbuf, b->ksiz) < 0; + } + }; + /** + * Leaf node of B+ tree. + */ + struct LeafNode { + RWLock lock; ///< lock + int64_t id; ///< page ID number + RecordArray recs; ///< sorted array of records + int64_t size; ///< total size of records + int64_t prev; ///< previous leaf node + int64_t next; ///< next leaf node + bool hot; ///< whether in the hot cache + bool dirty; ///< whether to be written back + bool dead; ///< whether to be removed + }; + /** + * Link to a node. + */ + struct Link { + int64_t child; ///< child node + int32_t ksiz; ///< size of the key + }; + /** + * Comparator for links. + */ + struct LinkComparator { + Comparator* comp; ///< comparator + /** constructor */ + explicit LinkComparator() : comp(NULL) { + _assert_(true); + } + /** comparing operator */ + bool operator ()(const Link* const& a, const Link* const& b) const { + _assert_(true); + char* akbuf = (char*)a + sizeof(*a); + char* bkbuf = (char*)b + sizeof(*b); + return comp->compare(akbuf, a->ksiz, bkbuf, b->ksiz) < 0; + } + }; + /** + * Inner node of B+ tree. + */ + struct InnerNode { + RWLock lock; ///< lock + int64_t id; ///< page ID numger + int64_t heir; ///< child before the first link + LinkArray links; ///< sorted array of links + int64_t size; ///< total size of links + bool dirty; ///< whether to be written back + bool dead; ///< whether to be removed + }; + /** + * Slot cache of leaf nodes. + */ + struct LeafSlot { + Mutex lock; ///< lock + LeafCache* hot; ///< hot cache + LeafCache* warm; ///< warm cache + }; + /** + * Slot cache of inner nodes. + */ + struct InnerSlot { + Mutex lock; ///< lock + InnerCache* warm; ///< warm cache + }; + /** + * Scoped visitor. + */ + class ScopedVisitor { + public: + /** constructor */ + explicit ScopedVisitor(Visitor* visitor) : visitor_(visitor) { + _assert_(visitor); + visitor_->visit_before(); + } + /** destructor */ + ~ScopedVisitor() { + _assert_(true); + visitor_->visit_after(); + } + private: + Visitor* visitor_; ///< visitor + }; + /** + * Open the leaf cache. + */ + void create_leaf_cache() { + _assert_(true); + int64_t bnum = bnum_ / SLOTNUM + 1; + if (bnum < INT8MAX) bnum = INT8MAX; + bnum = nearbyprime(bnum); + for (int32_t i = 0; i < SLOTNUM; i++) { + lslots_[i].hot = new LeafCache(bnum); + lslots_[i].warm = new LeafCache(bnum); + } + } + /** + * Close the leaf cache. + */ + void delete_leaf_cache() { + _assert_(true); + for (int32_t i = SLOTNUM - 1; i >= 0; i--) { + LeafSlot* slot = lslots_ + i; + delete slot->warm; + delete slot->hot; + } + } + /** + * Remove all leaf nodes from the leaf cache. + * @param save whether to save dirty nodes. + * @return true on success, or false on failure. + */ + bool flush_leaf_cache(bool save) { + _assert_(true); + bool err = false; + for (int32_t i = SLOTNUM - 1; i >= 0; i--) { + LeafSlot* slot = lslots_ + i; + typename LeafCache::Iterator it = slot->warm->begin(); + typename LeafCache::Iterator itend = slot->warm->end(); + while (it != itend) { + LeafNode* node = it.value(); + ++it; + if (!flush_leaf_node(node, save)) err = true; + } + it = slot->hot->begin(); + itend = slot->hot->end(); + while (it != itend) { + LeafNode* node = it.value(); + ++it; + if (!flush_leaf_node(node, save)) err = true; + } + } + return !err; + } + /** + * Flush a part of the leaf cache. + * @param slot a slot of leaf nodes. + * @return true on success, or false on failure. + */ + bool flush_leaf_cache_part(LeafSlot* slot) { + _assert_(slot); + bool err = false; + if (slot->warm->count() > 0) { + LeafNode* node = slot->warm->first_value(); + if (!flush_leaf_node(node, true)) err = true; + } else if (slot->hot->count() > 0) { + LeafNode* node = slot->hot->first_value(); + if (!flush_leaf_node(node, true)) err = true; + } + return !err; + } + /** + * Clean all of the leaf cache. + * @return true on success, or false on failure. + */ + bool clean_leaf_cache() { + _assert_(true); + bool err = false; + for (int32_t i = 0; i < SLOTNUM; i++) { + LeafSlot* slot = lslots_ + i; + ScopedMutex lock(&slot->lock); + typename LeafCache::Iterator it = slot->warm->begin(); + typename LeafCache::Iterator itend = slot->warm->end(); + while (it != itend) { + LeafNode* node = it.value(); + if (!save_leaf_node(node)) err = true; + ++it; + } + it = slot->hot->begin(); + itend = slot->hot->end(); + while (it != itend) { + LeafNode* node = it.value(); + if (!save_leaf_node(node)) err = true; + ++it; + } + } + return !err; + } + /** + * Clean a part of the leaf cache. + * @param slot a slot of leaf nodes. + * @return true on success, or false on failure. + */ + bool clean_leaf_cache_part(LeafSlot* slot) { + _assert_(slot); + bool err = false; + ScopedMutex lock(&slot->lock); + if (slot->warm->count() > 0) { + LeafNode* node = slot->warm->first_value(); + if (!save_leaf_node(node)) err = true; + } else if (slot->hot->count() > 0) { + LeafNode* node = slot->hot->first_value(); + if (!save_leaf_node(node)) err = true; + } + return !err; + } + /** + * Create a new leaf node. + * @param prev the ID of the previous node. + * @param next the ID of the next node. + * @return the created leaf node. + */ + LeafNode* create_leaf_node(int64_t prev, int64_t next) { + _assert_(true); + LeafNode* node = new LeafNode; + node->id = ++lcnt_; + node->size = sizeof(int32_t) * 2; + node->recs.reserve(DEFLINUM); + node->prev = prev; + node->next = next; + node->hot = false; + node->dirty = true; + node->dead = false; + int32_t sidx = node->id % SLOTNUM; + LeafSlot* slot = lslots_ + sidx; + slot->warm->set(node->id, node, LeafCache::MLAST); + cusage_ += node->size; + return node; + } + /** + * Remove a leaf node from the cache. + * @param node the leaf node. + * @param save whether to save dirty node. + * @return true on success, or false on failure. + */ + bool flush_leaf_node(LeafNode* node, bool save) { + _assert_(node); + bool err = false; + if (save && !save_leaf_node(node)) err = true; + typename RecordArray::const_iterator rit = node->recs.begin(); + typename RecordArray::const_iterator ritend = node->recs.end(); + while (rit != ritend) { + Record* rec = *rit; + xfree(rec); + ++rit; + } + int32_t sidx = node->id % SLOTNUM; + LeafSlot* slot = lslots_ + sidx; + if (node->hot) { + slot->hot->remove(node->id); + } else { + slot->warm->remove(node->id); + } + cusage_ -= node->size; + delete node; + return !err; + } + /** + * Save a leaf node. + * @param node the leaf node. + * @return true on success, or false on failure. + */ + bool save_leaf_node(LeafNode* node) { + _assert_(node); + ScopedRWLock lock(&node->lock, false); + if (!node->dirty) return true; + bool err = false; + char hbuf[NUMBUFSIZ]; + size_t hsiz = write_key(hbuf, LNPREFIX, node->id); + if (node->dead) { + if (!db_.remove(hbuf, hsiz) && db_.error().code() != Error::NOREC) err = true; + } else { + char* rbuf = new char[node->size]; + char* wp = rbuf; + wp += writevarnum(wp, node->prev); + wp += writevarnum(wp, node->next); + typename RecordArray::const_iterator rit = node->recs.begin(); + typename RecordArray::const_iterator ritend = node->recs.end(); + while (rit != ritend) { + Record* rec = *rit; + wp += writevarnum(wp, rec->ksiz); + wp += writevarnum(wp, rec->vsiz); + char* dbuf = (char*)rec + sizeof(*rec); + std::memcpy(wp, dbuf, rec->ksiz); + wp += rec->ksiz; + std::memcpy(wp, dbuf + rec->ksiz, rec->vsiz); + wp += rec->vsiz; + ++rit; + } + if (!db_.set(hbuf, hsiz, rbuf, wp - rbuf)) err = true; + delete[] rbuf; + } + node->dirty = false; + return !err; + } + /** + * Load a leaf node. + * @param id the ID number of the leaf node. + * @param prom whether to promote the warm cache. + * @return the loaded leaf node. + */ + LeafNode* load_leaf_node(int64_t id, bool prom) { + _assert_(id > 0); + int32_t sidx = id % SLOTNUM; + LeafSlot* slot = lslots_ + sidx; + ScopedMutex lock(&slot->lock); + LeafNode** np = slot->hot->get(id, LeafCache::MLAST); + if (np) return *np; + if (prom) { + if (slot->hot->count() * WARMRATIO > slot->warm->count() + WARMRATIO) { + slot->hot->first_value()->hot = false; + slot->hot->migrate(slot->hot->first_key(), slot->warm, LeafCache::MLAST); + } + np = slot->warm->migrate(id, slot->hot, LeafCache::MLAST); + if (np) { + (*np)->hot = true; + return *np; + } + } else { + LeafNode** np = slot->warm->get(id, LeafCache::MLAST); + if (np) return *np; + } + char hbuf[NUMBUFSIZ]; + size_t hsiz = write_key(hbuf, LNPREFIX, id); + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl() : node_(NULL) {} + LeafNode* pop() { + return node_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + uint64_t prev; + size_t step = readvarnum(vbuf, vsiz, &prev); + if (step < 1) return NOP; + vbuf += step; + vsiz -= step; + uint64_t next; + step = readvarnum(vbuf, vsiz, &next); + if (step < 1) return NOP; + vbuf += step; + vsiz -= step; + LeafNode* node = new LeafNode; + node->size = sizeof(int32_t) * 2; + node->prev = prev; + node->next = next; + while (vsiz > 1) { + uint64_t rksiz; + step = readvarnum(vbuf, vsiz, &rksiz); + if (step < 1) break; + vbuf += step; + vsiz -= step; + uint64_t rvsiz; + step = readvarnum(vbuf, vsiz, &rvsiz); + if (step < 1) break; + vbuf += step; + vsiz -= step; + if (vsiz < rksiz + rvsiz) break; + size_t rsiz = sizeof(Record) + rksiz + rvsiz; + Record* rec = (Record*)xmalloc(rsiz); + rec->ksiz = rksiz; + rec->vsiz = rvsiz; + char* dbuf = (char*)rec + sizeof(*rec); + std::memcpy(dbuf, vbuf, rksiz); + vbuf += rksiz; + vsiz -= rksiz; + std::memcpy(dbuf + rksiz, vbuf, rvsiz); + vbuf += rvsiz; + vsiz -= rvsiz; + node->recs.push_back(rec); + node->size += rsiz; + } + if (vsiz != 0) { + typename RecordArray::const_iterator rit = node->recs.begin(); + typename RecordArray::const_iterator ritend = node->recs.end(); + while (rit != ritend) { + Record* rec = *rit; + xfree(rec); + ++rit; + } + delete node; + return NOP; + } + node_ = node; + return NOP; + } + LeafNode* node_; + } visitor; + if (!db_.accept(hbuf, hsiz, &visitor, false)) return NULL; + LeafNode* node = visitor.pop(); + if (!node) return NULL; + node->id = id; + node->hot = false; + node->dirty = false; + node->dead = false; + slot->warm->set(id, node, LeafCache::MLAST); + cusage_ += node->size; + return node; + } + /** + * Check whether a record is in the range of a leaf node. + * @param node the leaf node. + * @param rec the record containing the key only. + * @return true for in range, or false for out of range. + */ + bool check_leaf_node_range(LeafNode* node, Record* rec) { + _assert_(node && rec); + RecordArray& recs = node->recs; + if (recs.empty()) return false; + Record* frec = recs.front(); + Record* lrec = recs.back(); + return !reccomp_(rec, frec) && !reccomp_(lrec, rec); + } + /** + * Accept a visitor at a leaf node. + * @param node the leaf node. + * @param rec the record containing the key only. + * @param visitor a visitor object. + * @return true to reorganize the tree, or false if not. + */ + bool accept_impl(LeafNode* node, Record* rec, Visitor* visitor) { + _assert_(node && rec && visitor); + bool reorg = false; + RecordArray& recs = node->recs; + typename RecordArray::iterator ritend = recs.end(); + typename RecordArray::iterator rit = std::lower_bound(recs.begin(), ritend, rec, reccomp_); + if (rit != ritend && !reccomp_(rec, *rit)) { + Record* rec = *rit; + char* kbuf = (char*)rec + sizeof(*rec); + size_t ksiz = rec->ksiz; + size_t vsiz; + const char* vbuf = visitor->visit_full(kbuf, ksiz, kbuf + ksiz, rec->vsiz, &vsiz); + if (vbuf == Visitor::REMOVE) { + size_t rsiz = sizeof(*rec) + rec->ksiz + rec->vsiz; + count_ -= 1; + cusage_ -= rsiz; + node->size -= rsiz; + node->dirty = true; + xfree(rec); + recs.erase(rit); + if (recs.empty()) reorg = true; + } else if (vbuf != Visitor::NOP) { + int64_t diff = (int64_t)vsiz - (int64_t)rec->vsiz; + cusage_ += diff; + node->size += diff; + node->dirty = true; + if (vsiz > rec->vsiz) { + *rit = (Record*)xrealloc(rec, sizeof(*rec) + rec->ksiz + vsiz); + rec = *rit; + kbuf = (char*)rec + sizeof(*rec); + } + std::memcpy(kbuf + rec->ksiz, vbuf, vsiz); + rec->vsiz = vsiz; + if (node->size > psiz_ && recs.size() > 1) reorg = true; + } + } else { + const char* kbuf = (char*)rec + sizeof(*rec); + size_t ksiz = rec->ksiz; + size_t vsiz; + const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + size_t rsiz = sizeof(*rec) + ksiz + vsiz; + count_ += 1; + cusage_ += rsiz; + node->size += rsiz; + node->dirty = true; + rec = (Record*)xmalloc(rsiz); + rec->ksiz = ksiz; + rec->vsiz = vsiz; + char* dbuf = (char*)rec + sizeof(*rec); + std::memcpy(dbuf, kbuf, ksiz); + std::memcpy(dbuf + ksiz, vbuf, vsiz); + recs.insert(rit, rec); + if (node->size > psiz_ && recs.size() > 1) reorg = true; + } + } + return reorg; + } + /** + * Devide a leaf node into two. + * @param node the leaf node. + * @return the created node, or NULL on failure. + */ + LeafNode* divide_leaf_node(LeafNode* node) { + _assert_(node); + LeafNode* newnode = create_leaf_node(node->id, node->next); + if (newnode->next > 0) { + LeafNode* nextnode = load_leaf_node(newnode->next, false); + if (!nextnode) { + set_error(_KCCODELINE_, Error::BROKEN, "missing leaf node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)newnode->next); + return NULL; + } + nextnode->prev = newnode->id; + nextnode->dirty = true; + } + node->next = newnode->id; + node->dirty = true; + RecordArray& recs = node->recs; + typename RecordArray::iterator mid = recs.begin() + recs.size() / 2; + typename RecordArray::iterator rit = mid; + typename RecordArray::iterator ritend = recs.end(); + RecordArray& newrecs = newnode->recs; + while (rit != ritend) { + Record* rec = *rit; + newrecs.push_back(rec); + size_t rsiz = sizeof(*rec) + rec->ksiz + rec->vsiz; + node->size -= rsiz; + newnode->size += rsiz; + ++rit; + } + escape_cursors(node->id, node->next, *mid); + recs.erase(mid, ritend); + return newnode; + } + /** + * Open the inner cache. + */ + void create_inner_cache() { + _assert_(true); + int64_t bnum = (bnum_ / AVGWAY) / SLOTNUM + 1; + if (bnum < INT8MAX) bnum = INT8MAX; + bnum = nearbyprime(bnum); + for (int32_t i = 0; i < SLOTNUM; i++) { + islots_[i].warm = new InnerCache(bnum); + } + } + /** + * Close the inner cache. + */ + void delete_inner_cache() { + _assert_(true); + for (int32_t i = SLOTNUM - 1; i >= 0; i--) { + InnerSlot* slot = islots_ + i; + delete slot->warm; + } + } + /** + * Remove all inner nodes from the inner cache. + * @param save whether to save dirty nodes. + * @return true on success, or false on failure. + */ + bool flush_inner_cache(bool save) { + _assert_(true); + bool err = false; + for (int32_t i = SLOTNUM - 1; i >= 0; i--) { + InnerSlot* slot = islots_ + i; + typename InnerCache::Iterator it = slot->warm->begin(); + typename InnerCache::Iterator itend = slot->warm->end(); + while (it != itend) { + InnerNode* node = it.value(); + ++it; + if (!flush_inner_node(node, save)) err = true; + } + } + return !err; + } + /** + * Flush a part of the inner cache. + * @param slot a slot of inner nodes. + * @return true on success, or false on failure. + */ + bool flush_inner_cache_part(InnerSlot* slot) { + _assert_(slot); + bool err = false; + if (slot->warm->count() > 0) { + InnerNode* node = slot->warm->first_value(); + if (!flush_inner_node(node, true)) err = true; + } + return !err; + } + /** + * Clean all of the inner cache. + * @return true on success, or false on failure. + */ + bool clean_inner_cache() { + _assert_(true); + bool err = false; + for (int32_t i = 0; i < SLOTNUM; i++) { + InnerSlot* slot = islots_ + i; + ScopedMutex lock(&slot->lock); + typename InnerCache::Iterator it = slot->warm->begin(); + typename InnerCache::Iterator itend = slot->warm->end(); + while (it != itend) { + InnerNode* node = it.value(); + if (!save_inner_node(node)) err = true; + ++it; + } + } + return !err; + } + /** + * Create a new inner node. + * @param heir the ID of the child before the first link. + * @return the created inner node. + */ + InnerNode* create_inner_node(int64_t heir) { + _assert_(true); + InnerNode* node = new InnerNode; + node->id = ++icnt_ + INIDBASE; + node->heir = heir; + node->links.reserve(DEFIINUM); + node->size = sizeof(int64_t); + node->dirty = true; + node->dead = false; + int32_t sidx = node->id % SLOTNUM; + InnerSlot* slot = islots_ + sidx; + slot->warm->set(node->id, node, InnerCache::MLAST); + cusage_ += node->size; + return node; + } + /** + * Remove an inner node from the cache. + * @param node the inner node. + * @param save whether to save dirty node. + * @return true on success, or false on failure. + */ + bool flush_inner_node(InnerNode* node, bool save) { + _assert_(node); + bool err = false; + if (save && !save_inner_node(node)) err = true; + typename LinkArray::const_iterator lit = node->links.begin(); + typename LinkArray::const_iterator litend = node->links.end(); + while (lit != litend) { + Link* link = *lit; + xfree(link); + ++lit; + } + int32_t sidx = node->id % SLOTNUM; + InnerSlot* slot = islots_ + sidx; + slot->warm->remove(node->id); + cusage_ -= node->size; + delete node; + return !err; + } + /** + * Save a inner node. + * @param node the inner node. + * @return true on success, or false on failure. + */ + bool save_inner_node(InnerNode* node) { + _assert_(true); + if (!node->dirty) return true; + bool err = false; + char hbuf[NUMBUFSIZ]; + size_t hsiz = write_key(hbuf, INPREFIX, node->id - INIDBASE); + if (node->dead) { + if (!db_.remove(hbuf, hsiz) && db_.error().code() != Error::NOREC) err = true; + } else { + char* rbuf = new char[node->size]; + char* wp = rbuf; + wp += writevarnum(wp, node->heir); + typename LinkArray::const_iterator lit = node->links.begin(); + typename LinkArray::const_iterator litend = node->links.end(); + while (lit != litend) { + Link* link = *lit; + wp += writevarnum(wp, link->child); + wp += writevarnum(wp, link->ksiz); + char* dbuf = (char*)link + sizeof(*link); + std::memcpy(wp, dbuf, link->ksiz); + wp += link->ksiz; + ++lit; + } + if (!db_.set(hbuf, hsiz, rbuf, wp - rbuf)) err = true; + delete[] rbuf; + } + node->dirty = false; + return !err; + } + /** + * Load an inner node. + * @param id the ID number of the inner node. + * @return the loaded inner node. + */ + InnerNode* load_inner_node(int64_t id) { + _assert_(id > 0); + int32_t sidx = id % SLOTNUM; + InnerSlot* slot = islots_ + sidx; + ScopedMutex lock(&slot->lock); + InnerNode** np = slot->warm->get(id, InnerCache::MLAST); + if (np) return *np; + char hbuf[NUMBUFSIZ]; + size_t hsiz = write_key(hbuf, INPREFIX, id - INIDBASE); + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl() : node_(NULL) {} + InnerNode* pop() { + return node_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + uint64_t heir; + size_t step = readvarnum(vbuf, vsiz, &heir); + if (step < 1) return NOP; + vbuf += step; + vsiz -= step; + InnerNode* node = new InnerNode; + node->size = sizeof(int64_t); + node->heir = heir; + while (vsiz > 1) { + uint64_t child; + step = readvarnum(vbuf, vsiz, &child); + if (step < 1) break; + vbuf += step; + vsiz -= step; + uint64_t rksiz; + step = readvarnum(vbuf, vsiz, &rksiz); + if (step < 1) break; + vbuf += step; + vsiz -= step; + if (vsiz < rksiz) break; + Link* link = (Link*)xmalloc(sizeof(*link) + rksiz); + link->child = child; + link->ksiz = rksiz; + char* dbuf = (char*)link + sizeof(*link); + std::memcpy(dbuf, vbuf, rksiz); + vbuf += rksiz; + vsiz -= rksiz; + node->links.push_back(link); + node->size += sizeof(*link) + rksiz; + } + if (vsiz != 0) { + typename LinkArray::const_iterator lit = node->links.begin(); + typename LinkArray::const_iterator litend = node->links.end(); + while (lit != litend) { + Link* link = *lit; + xfree(link); + ++lit; + } + delete node; + return NOP; + } + node_ = node; + return NOP; + } + InnerNode* node_; + } visitor; + if (!db_.accept(hbuf, hsiz, &visitor, false)) return NULL; + InnerNode* node = visitor.pop(); + if (!node) return NULL; + node->id = id; + node->dirty = false; + node->dead = false; + slot->warm->set(id, node, InnerCache::MLAST); + cusage_ += node->size; + return node; + } + /** + * Search the B+ tree. + * @param link the link containing the key only. + * @param prom whether to promote the warm cache. + * @param hist the array of visiting history. + * @param hnp the pointer to the variable into which the number of the history is assigned. + * @return the corresponding leaf node, or NULL on failure. + */ + LeafNode* search_tree(Link* link, bool prom, int64_t* hist, int32_t* hnp) { + _assert_(link && hist && hnp); + int64_t id = root_; + int32_t hnum = 0; + while (id > INIDBASE) { + InnerNode* node = load_inner_node(id); + if (!node) { + set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)id); + return NULL; + } + hist[hnum++] = id; + const LinkArray& links = node->links; + typename LinkArray::const_iterator litbeg = links.begin(); + typename LinkArray::const_iterator litend = links.end(); + typename LinkArray::const_iterator lit = std::upper_bound(litbeg, litend, link, linkcomp_); + if (lit == litbeg) { + id = node->heir; + } else { + --lit; + Link* link = *lit; + id = link->child; + } + } + *hnp = hnum; + return load_leaf_node(id, prom); + } + /** + * Reorganize the B+ tree. + * @param node a leaf node. + * @param hist the array of visiting history. + * @param hnum the number of the history. + * @return true on success, or false on failure. + */ + bool reorganize_tree(LeafNode* node, int64_t* hist, int32_t hnum) { + _assert_(node && hist && hnum >= 0); + if (node->size > psiz_ && node->recs.size() > 1) { + LeafNode* newnode = divide_leaf_node(node); + if (!newnode) return false; + if (node->id == last_) last_ = newnode->id; + int64_t heir = node->id; + int64_t child = newnode->id; + Record* rec = *newnode->recs.begin(); + char* dbuf = (char*)rec + sizeof(*rec); + int32_t ksiz = rec->ksiz; + char* kbuf = new char[ksiz]; + std::memcpy(kbuf, dbuf, ksiz); + while (true) { + if (hnum < 1) { + InnerNode* inode = create_inner_node(heir); + add_link_inner_node(inode, child, kbuf, ksiz); + root_ = inode->id; + delete[] kbuf; + break; + } + int64_t parent = hist[--hnum]; + InnerNode* inode = load_inner_node(parent); + if (!inode) { + set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)parent); + delete[] kbuf; + return false; + } + add_link_inner_node(inode, child, kbuf, ksiz); + delete[] kbuf; + LinkArray& links = inode->links; + if (inode->size <= psiz_ || links.size() <= INLINKMIN) break; + typename LinkArray::iterator litbeg = links.begin(); + typename LinkArray::iterator mid = litbeg + links.size() / 2; + Link* link = *mid; + InnerNode* newinode = create_inner_node(link->child); + heir = inode->id; + child = newinode->id; + char* dbuf = (char*)link + sizeof(*link); + ksiz = link->ksiz; + kbuf = new char[ksiz]; + std::memcpy(kbuf, dbuf, ksiz); + typename LinkArray::iterator lit = mid + 1; + typename LinkArray::iterator litend = links.end(); + while (lit != litend) { + link = *lit; + char* dbuf = (char*)link + sizeof(*link); + add_link_inner_node(newinode, link->child, dbuf, link->ksiz); + ++lit; + } + int32_t num = newinode->links.size(); + for (int32_t i = 0; i <= num; i++) { + Link* link = links.back(); + size_t rsiz = sizeof(*link) + link->ksiz; + cusage_ -= rsiz; + inode->size -= rsiz; + xfree(link); + links.pop_back(); + } + inode->dirty = true; + } + } else if (node->recs.empty() && hnum > 0) { + if (!escape_cursors(node->id, node->next)) return false; + InnerNode* inode = load_inner_node(hist[--hnum]); + if (!inode) { + set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)hist[hnum]); + return false; + } + if (sub_link_tree(inode, node->id, hist, hnum)) { + if (node->prev > 0) { + LeafNode* tnode = load_leaf_node(node->prev, false); + if (!tnode) { + set_error(_KCCODELINE_, Error::BROKEN, "missing node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)node->prev); + return false; + } + tnode->next = node->next; + tnode->dirty = true; + if (last_ == node->id) last_ = node->prev; + } + if (node->next > 0) { + LeafNode* tnode = load_leaf_node(node->next, false); + if (!tnode) { + set_error(_KCCODELINE_, Error::BROKEN, "missing node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)node->next); + return false; + } + tnode->prev = node->prev; + tnode->dirty = true; + if (first_ == node->id) first_ = node->next; + } + node->dead = true; + } + } + return true; + } + /** + * Add a link to a inner node. + * @param node the inner node. + * @param child the ID number of the child. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + */ + void add_link_inner_node(InnerNode* node, int64_t child, const char* kbuf, size_t ksiz) { + _assert_(node && kbuf); + size_t rsiz = sizeof(Link) + ksiz; + Link* link = (Link*)xmalloc(rsiz); + link->child = child; + link->ksiz = ksiz; + char* dbuf = (char*)link + sizeof(*link); + std::memcpy(dbuf, kbuf, ksiz); + LinkArray& links = node->links; + typename LinkArray::iterator litend = links.end(); + typename LinkArray::iterator lit = std::upper_bound(links.begin(), litend, link, linkcomp_); + links.insert(lit, link); + node->size += rsiz; + node->dirty = true; + cusage_ += rsiz; + } + /** + * Subtract a link from the B+ tree. + * @param node the inner node. + * @param child the ID number of the child. + * @param hist the array of visiting history. + * @param hnum the number of the history. + * @return true on success, or false on failure. + */ + bool sub_link_tree(InnerNode* node, int64_t child, int64_t* hist, int32_t hnum) { + _assert_(node && hist && hnum >= 0); + node->dirty = true; + LinkArray& links = node->links; + typename LinkArray::iterator lit = links.begin(); + typename LinkArray::iterator litend = links.end(); + if (node->heir == child) { + if (!links.empty()) { + Link* link = *lit; + node->heir = link->child; + xfree(link); + links.erase(lit); + return true; + } else if (hnum > 0) { + InnerNode* pnode = load_inner_node(hist[--hnum]); + if (!pnode) { + set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)hist[hnum]); + return false; + } + node->dead = true; + return sub_link_tree(pnode, node->id, hist, hnum); + } + node->dead = true; + root_ = child; + while (child > INIDBASE) { + node = load_inner_node(child); + if (!node) { + set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); + db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)child); + return false; + } + if (node->dead) { + child = node->heir; + root_ = child; + } else { + child = 0; + } + } + return false; + } + while (lit != litend) { + Link* link = *lit; + if (link->child == child) { + xfree(link); + links.erase(lit); + return true; + } + ++lit; + } + set_error(_KCCODELINE_, Error::BROKEN, "invalid tree"); + return false; + } + /** + * Dump the meta data into the file. + * @return true on success, or false on failure. + */ + bool dump_meta() { + _assert_(true); + char head[HEADSIZ]; + std::memset(head, 0, sizeof(head)); + char* wp = head; + if (reccomp_.comp == LEXICALCOMP) { + *(uint8_t*)(wp++) = 0x10; + } else if (reccomp_.comp == DECIMALCOMP) { + *(uint8_t*)(wp++) = 0x11; + } else if (reccomp_.comp == LEXICALDESCCOMP) { + *(uint8_t*)(wp++) = 0x18; + } else if (reccomp_.comp == DECIMALDESCCOMP) { + *(uint8_t*)(wp++) = 0x19; + } else { + *(uint8_t*)(wp++) = 0xff; + } + wp = head + MOFFNUMS; + uint64_t num = hton64(psiz_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(root_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(first_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(last_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(lcnt_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(icnt_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(count_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + num = hton64(bnum_); + std::memcpy(wp, &num, sizeof(num)); + wp += sizeof(num); + std::memcpy(wp, "\x0a\x42\x6f\x6f\x66\x79\x21\x0a", sizeof(num)); + wp += sizeof(num); + if (!db_.set(KCPDBMETAKEY, sizeof(KCPDBMETAKEY) - 1, head, sizeof(head))) return false; + trlcnt_ = lcnt_; + trcount_ = count_; + return true; + } + /** + * Load the meta data from the file. + * @return true on success, or false on failure. + */ + bool load_meta() { + _assert_(true); + char head[HEADSIZ]; + int32_t hsiz = db_.get(KCPDBMETAKEY, sizeof(KCPDBMETAKEY) - 1, head, sizeof(head)); + if (hsiz < 0) return false; + if (hsiz != sizeof(head)) { + set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data record"); + db_.report(_KCCODELINE_, Logger::WARN, "hsiz=%d", hsiz); + return false; + } + const char* rp = head; + if (*(uint8_t*)rp == 0x10) { + reccomp_.comp = LEXICALCOMP; + linkcomp_.comp = LEXICALCOMP; + } else if (*(uint8_t*)rp == 0x11) { + reccomp_.comp = DECIMALCOMP; + linkcomp_.comp = DECIMALCOMP; + } else if (*(uint8_t*)rp == 0x18) { + reccomp_.comp = LEXICALDESCCOMP; + linkcomp_.comp = LEXICALDESCCOMP; + } else if (*(uint8_t*)rp == 0x19) { + reccomp_.comp = DECIMALDESCCOMP; + linkcomp_.comp = DECIMALDESCCOMP; + } else if (*(uint8_t*)rp == 0xff) { + if (!reccomp_.comp) { + set_error(_KCCODELINE_, Error::INVALID, "the custom comparator is not given"); + return false; + } + linkcomp_.comp = reccomp_.comp; + } else { + set_error(_KCCODELINE_, Error::BROKEN, "comparator is invalid"); + return false; + } + rp = head + MOFFNUMS; + uint64_t num; + std::memcpy(&num, rp, sizeof(num)); + psiz_ = ntoh64(num); + rp += sizeof(num); + std::memcpy(&num, rp, sizeof(num)); + root_ = ntoh64(num); + rp += sizeof(num); + std::memcpy(&num, rp, sizeof(num)); + first_ = ntoh64(num); + rp += sizeof(num); + std::memcpy(&num, rp, sizeof(num)); + last_ = ntoh64(num); + rp += sizeof(num); + std::memcpy(&num, rp, sizeof(num)); + lcnt_ = ntoh64(num); + rp += sizeof(num); + std::memcpy(&num, rp, sizeof(num)); + icnt_ = ntoh64(num); + rp += sizeof(num); + std::memcpy(&num, rp, sizeof(num)); + count_ = ntoh64(num); + rp += sizeof(num); + std::memcpy(&num, rp, sizeof(num)); + bnum_ = ntoh64(num); + rp += sizeof(num); + trlcnt_ = lcnt_; + trcount_ = count_; + return true; + } + /** + * Caluculate the total number of nodes in the leaf cache. + * @return the total number of nodes in the leaf cache. + */ + int64_t calc_leaf_cache_count() { + _assert_(true); + int64_t sum = 0; + for (int32_t i = 0; i < SLOTNUM; i++) { + LeafSlot* slot = lslots_ + i; + sum += slot->warm->count(); + sum += slot->hot->count(); + } + return sum; + } + /** + * Caluculate the amount of memory usage of the leaf cache. + * @return the amount of memory usage of the leaf cache. + */ + int64_t calc_leaf_cache_size() { + _assert_(true); + int64_t sum = 0; + for (int32_t i = 0; i < SLOTNUM; i++) { + LeafSlot* slot = lslots_ + i; + typename LeafCache::Iterator it = slot->warm->begin(); + typename LeafCache::Iterator itend = slot->warm->end(); + while (it != itend) { + LeafNode* node = it.value(); + sum += node->size; + ++it; + } + it = slot->hot->begin(); + itend = slot->hot->end(); + while (it != itend) { + LeafNode* node = it.value(); + sum += node->size; + ++it; + } + } + return sum; + } + /** + * Caluculate the total number of nodes in the inner cache. + * @return the total number of nodes in the inner cache. + */ + int64_t calc_inner_cache_count() { + _assert_(true); + int64_t sum = 0; + for (int32_t i = 0; i < SLOTNUM; i++) { + InnerSlot* slot = islots_ + i; + sum += slot->warm->count(); + } + return sum; + } + /** + * Caluculate the amount of memory usage of the inner cache. + * @return the amount of memory usage of the inner cache. + */ + int64_t calc_inner_cache_size() { + _assert_(true); + int64_t sum = 0; + for (int32_t i = 0; i < SLOTNUM; i++) { + InnerSlot* slot = islots_ + i; + typename InnerCache::Iterator it = slot->warm->begin(); + typename InnerCache::Iterator itend = slot->warm->end(); + while (it != itend) { + InnerNode* node = it.value(); + sum += node->size; + ++it; + } + } + return sum; + } + /** + * Disable all cursors. + */ + void disable_cursors() { + _assert_(true); + if (curs_.empty()) return; + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->kbuf_) cur->clear_position(); + ++cit; + } + } + /** + * Escape cursors on a divided leaf node. + * @param src the ID of the source node. + * @param dest the ID of the destination node. + * @param rec the pivot record. + * @return true on success, or false on failure. + */ + void escape_cursors(int64_t src, int64_t dest, Record* rec) { + _assert_(src > 0 && dest >= 0 && rec); + if (curs_.empty()) return; + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->lid_ == src) { + char* dbuf = (char*)rec + sizeof(*rec); + if (reccomp_.comp->compare(cur->kbuf_, cur->ksiz_, dbuf, rec->ksiz) >= 0) + cur->lid_ = dest; + } + ++cit; + } + } + /** + * Escape cursors on a removed leaf node. + * @param src the ID of the source node. + * @param dest the ID of the destination node. + * @return true on success, or false on failure. + */ + bool escape_cursors(int64_t src, int64_t dest) { + _assert_(src > 0 && dest >= 0); + if (curs_.empty()) return true; + bool err = false; + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->lid_ == src) { + cur->clear_position(); + if (!cur->set_position(dest) && db_.error().code() != Error::NOREC) err = true; + } + ++cit; + } + return !err; + } + /** + * Recalculate the count data. + * @return true on success, or false on failure. + */ + bool recalc_count() { + _assert_(true); + if (!load_meta()) return false; + bool err = false; + std::set<int64_t> ids; + std::set<int64_t> prevs; + std::set<int64_t> nexts; + class VisitorImpl : public DB::Visitor { + public: + explicit VisitorImpl(std::set<int64_t>* ids, + std::set<int64_t>* prevs, std::set<int64_t>* nexts) : + ids_(ids), prevs_(prevs), nexts_(nexts), count_(0) {} + int64_t count() { + return count_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (ksiz < 2 || ksiz >= NUMBUFSIZ || kbuf[0] != LNPREFIX) return NOP; + kbuf++; + ksiz--; + char tkbuf[NUMBUFSIZ]; + std::memcpy(tkbuf, kbuf, ksiz); + tkbuf[ksiz] = '\0'; + int64_t id = atoih(tkbuf); + uint64_t prev; + size_t step = readvarnum(vbuf, vsiz, &prev); + if (step < 1) return NOP; + vbuf += step; + vsiz -= step; + uint64_t next; + step = readvarnum(vbuf, vsiz, &next); + if (step < 1) return NOP; + vbuf += step; + vsiz -= step; + ids_->insert(id); + if (prev > 0) prevs_->insert(prev); + if (next > 0) nexts_->insert(next); + while (vsiz > 1) { + uint64_t rksiz; + step = readvarnum(vbuf, vsiz, &rksiz); + if (step < 1) break; + vbuf += step; + vsiz -= step; + uint64_t rvsiz; + step = readvarnum(vbuf, vsiz, &rvsiz); + if (step < 1) break; + vbuf += step; + vsiz -= step; + if (vsiz < rksiz + rvsiz) break; + vbuf += rksiz; + vsiz -= rksiz; + vbuf += rvsiz; + vsiz -= rvsiz; + count_++; + } + return NOP; + } + std::set<int64_t>* ids_; + std::set<int64_t>* prevs_; + std::set<int64_t>* nexts_; + int64_t count_; + } visitor(&ids, &prevs, &nexts); + if (!db_.iterate(&visitor, false)) err = true; + int64_t count = visitor.count(); + db_.report(_KCCODELINE_, Logger::WARN, "recalculated the record count from %lld to %lld", + (long long)count_, (long long)count); + std::set<int64_t>::iterator iitend = ids.end(); + std::set<int64_t>::iterator nit = nexts.begin(); + std::set<int64_t>::iterator nitend = nexts.end(); + while (nit != nitend) { + if (ids.find(*nit) == ids.end()) { + db_.report(_KCCODELINE_, Logger::WARN, "detected missing leaf: %lld", (long long)*nit); + count = INT64MAX; + } + ++nit; + } + std::set<int64_t>::iterator pit = prevs.begin(); + std::set<int64_t>::iterator pitend = prevs.end(); + while (pit != pitend) { + if (ids.find(*pit) == iitend) { + db_.report(_KCCODELINE_, Logger::WARN, "detected missing leaf: %lld", (long long)*pit); + count = INT64MAX; + } + ++pit; + } + count_ = count; + if (!dump_meta()) err = true; + return !err; + } + /** + * Reorganize the database file. + * @param mode the connection mode of the internal database. + * @return true on success, or false on failure. + */ + bool reorganize_file(uint32_t mode) { + _assert_(true); + if (!load_meta()) { + if (reccomp_.comp) { + linkcomp_.comp = reccomp_.comp; + } else { + reccomp_.comp = LEXICALCOMP; + linkcomp_.comp = LEXICALCOMP; + } + } + const std::string& path = db_.path(); + const std::string& npath = path + File::EXTCHR + KCPDBTMPPATHEXT; + PlantDB tdb; + tdb.tune_comparator(reccomp_.comp); + if (!tdb.open(npath, OWRITER | OCREATE | OTRUNCATE)) { + set_error(_KCCODELINE_, tdb.error().code(), "opening the destination failed"); + return false; + } + db_.report(_KCCODELINE_, Logger::WARN, "reorganizing the database"); + bool err = false; + create_leaf_cache(); + create_inner_cache(); + DB::Cursor* cur = db_.cursor(); + cur->jump(); + char* kbuf; + size_t ksiz; + while (!err && (kbuf = cur->get_key(&ksiz)) != NULL) { + if (*kbuf == LNPREFIX) { + int64_t id = std::strtol(kbuf + 1, NULL, 16); + if (id > 0 && id < INIDBASE) { + LeafNode* node = load_leaf_node(id, false); + if (node) { + const RecordArray& recs = node->recs; + typename RecordArray::const_iterator rit = recs.begin(); + typename RecordArray::const_iterator ritend = recs.end(); + while (rit != ritend) { + Record* rec = *rit; + char* dbuf = (char*)rec + sizeof(*rec); + if (!tdb.set(dbuf, rec->ksiz, dbuf + rec->ksiz, rec->vsiz)) { + set_error(_KCCODELINE_, tdb.error().code(), + "opening the destination failed"); + err = true; + } + ++rit; + } + flush_leaf_node(node, false); + } + } + } + delete[] kbuf; + cur->step(); + } + delete cur; + delete_inner_cache(); + delete_leaf_cache(); + if (!tdb.close()) { + set_error(_KCCODELINE_, tdb.error().code(), "opening the destination failed"); + err = true; + } + if (DBTYPE == TYPETREE) { + if (File::rename(npath, path)) { + if (!db_.close()) err = true; + if (!db_.open(path, mode)) err = true; + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destination failed"); + err = true; + } + File::remove(npath); + } else if (DBTYPE == TYPEFOREST) { + const std::string& tpath = npath + File::EXTCHR + KCPDBTMPPATHEXT; + File::remove_recursively(tpath); + if (File::rename(path, tpath)) { + if (File::rename(npath, path)) { + if (!db_.close()) err = true; + if (!db_.open(path, mode)) err = true; + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destination failed"); + File::rename(tpath, path); + err = true; + } + } else { + set_error(_KCCODELINE_, Error::SYSTEM, "renaming the source failed"); + err = true; + } + File::remove_recursively(tpath); + File::remove_recursively(npath); + } else { + BASEDB udb; + if (!err && udb.open(npath, OREADER)) { + if (writer_) { + if (!db_.clear()) err = true; + } else { + if (!db_.close()) err = true; + uint32_t tmode = (mode & ~OREADER) | OWRITER | OCREATE | OTRUNCATE; + if (!db_.open(path, tmode)) err = true; + } + cur = udb.cursor(); + cur->jump(); + const char* vbuf; + size_t vsiz; + while (!err && (kbuf = cur->get(&ksiz, &vbuf, &vsiz)) != NULL) { + if (!db_.set(kbuf, ksiz, vbuf, vsiz)) err = true; + delete[] kbuf; + cur->step(); + } + delete cur; + if (writer_) { + if (!db_.synchronize(false, NULL)) err = true; + } else { + if (!db_.close()) err = true; + if (!db_.open(path, mode)) err = true; + } + if (!udb.close()) { + set_error(_KCCODELINE_, udb.error().code(), "closing the destination failed"); + err = true; + } + } else { + set_error(_KCCODELINE_, udb.error().code(), "opening the destination failed"); + err = true; + } + File::remove_recursively(npath); + } + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_impl(bool hard) { + _assert_(true); + if (!clean_leaf_cache()) return false; + if (!clean_inner_cache()) return false; + int32_t idx = trclock_++ % SLOTNUM; + LeafSlot* lslot = lslots_ + idx; + if (lslot->warm->count() + lslot->hot->count() > 1) flush_leaf_cache_part(lslot); + InnerSlot* islot = islots_ + idx; + if (islot->warm->count() > 1) flush_inner_cache_part(islot); + if ((trlcnt_ != lcnt_ || count_ != trcount_) && !dump_meta()) return false; + if (!db_.begin_transaction(hard)) return false; + return true; + } + /** + * Commit transaction. + * @return true on success, or false on failure. + */ + bool commit_transaction() { + _assert_(true); + bool err = false; + if (!clean_leaf_cache()) return false; + if (!clean_inner_cache()) return false; + if ((trlcnt_ != lcnt_ || count_ != trcount_) && !dump_meta()) err = true; + if (!db_.end_transaction(true)) return false; + return !err; + } + /** + * Abort transaction. + * @return true on success, or false on failure. + */ + bool abort_transaction() { + _assert_(true); + bool err = false; + flush_leaf_cache(false); + flush_inner_cache(false); + if (!db_.end_transaction(false)) err = true; + if (!load_meta()) err = true; + disable_cursors(); + return !err; + } + /** + * Fix auto transaction for the B+ tree. + * @return true on success, or false on failure. + */ + bool fix_auto_transaction_tree() { + _assert_(true); + if (!db_.begin_transaction(autosync_)) return false; + bool err = false; + if (!clean_leaf_cache()) err = true; + if (!clean_inner_cache()) err = true; + size_t cnum = ATRANCNUM / SLOTNUM; + int32_t idx = trclock_++ % SLOTNUM; + LeafSlot* lslot = lslots_ + idx; + if (lslot->warm->count() + lslot->hot->count() > cnum) flush_leaf_cache_part(lslot); + InnerSlot* islot = islots_ + idx; + if (islot->warm->count() > cnum) flush_inner_cache_part(islot); + if (!dump_meta()) err = true; + if (!db_.end_transaction(true)) err = true; + return !err; + } + /** + * Fix auto transaction for a leaf. + * @return true on success, or false on failure. + */ + bool fix_auto_transaction_leaf(LeafNode* node) { + _assert_(node); + bool err = false; + if (!save_leaf_node(node)) err = true; + return !err; + } + /** + * Fix auto synchronization. + * @return true on success, or false on failure. + */ + bool fix_auto_synchronization() { + _assert_(true); + bool err = false; + if (!flush_leaf_cache(true)) err = true; + if (!flush_inner_cache(true)) err = true; + if (!dump_meta()) err = true; + if (!db_.synchronize(true, NULL)) err = true; + return !err; + } + /** + * Write the key pattern into a buffer. + * @param kbuf the destination buffer. + * @param pc the prefix character. + * @param id the ID number of the page. + * @return the size of the key pattern. + */ + size_t write_key(char* kbuf, int32_t pc, int64_t num) { + _assert_(kbuf && num >= 0); + char* wp = kbuf; + *(wp++) = pc; + bool hit = false; + for (size_t i = 0; i < sizeof(num); i++) { + uint8_t c = num >> ((sizeof(num) - 1 - i) * 8); + uint8_t h = c >> 4; + if (h < 10) { + if (hit || h != 0) { + *(wp++) = '0' + h; + hit = true; + } + } else { + *(wp++) = 'A' - 10 + h; + hit = true; + } + uint8_t l = c & 0xf; + if (l < 10) { + if (hit || l != 0) { + *(wp++) = '0' + l; + hit = true; + } + } else { + *(wp++) = 'A' - 10 + l; + hit = true; + } + } + return wp - kbuf; + } + /** Dummy constructor to forbid the use. */ + PlantDB(const PlantDB&); + /** Dummy Operator to forbid the use. */ + PlantDB& operator =(const PlantDB&); + /** The method lock. */ + RWLock mlock_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The open mode. */ + uint32_t omode_; + /** The flag for writer. */ + bool writer_; + /** The flag for auto transaction. */ + bool autotran_; + /** The flag for auto synchronization. */ + bool autosync_; + /** The internal database. */ + BASEDB db_; + /** The cursor objects. */ + CursorList curs_; + /** The alignment power. */ + uint8_t apow_; + /** The free block pool power. */ + uint8_t fpow_; + /** The options. */ + uint8_t opts_; + /** The bucket number. */ + int64_t bnum_; + /** The page size. */ + int32_t psiz_; + /** The capacity of page cache. */ + int64_t pccap_; + /** The root node. */ + int64_t root_; + /** The first node. */ + int64_t first_; + /** The last node. */ + int64_t last_; + /** The count of leaf nodes. */ + int64_t lcnt_; + /** The count of inner nodes. */ + int64_t icnt_; + /** The record number. */ + AtomicInt64 count_; + /** The cache memory usage. */ + AtomicInt64 cusage_; + /** The Slots of leaf nodes. */ + LeafSlot lslots_[SLOTNUM]; + /** The Slots of inner nodes. */ + InnerSlot islots_[SLOTNUM]; + /** The record comparator. */ + RecordComparator reccomp_; + /** The link comparator. */ + LinkComparator linkcomp_; + /** The flag whether in transaction. */ + bool tran_; + /** The logical clock for transaction. */ + int64_t trclock_; + /** The leaf count history for transaction. */ + int64_t trlcnt_; + /** The record count history for transaction. */ + int64_t trcount_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcpolydb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolydb.cc new file mode 100644 index 0000000000..86a3e69111 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolydb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Polymorphic database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcpolydb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcpolydb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolydb.h new file mode 100644 index 0000000000..24f5c1423b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolydb.h @@ -0,0 +1,1656 @@ +/************************************************************************************************* + * Polymorphic database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCPOLYDB_H // duplication check +#define _KCPOLYDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> +#include <kcplantdb.h> +#include <kcprotodb.h> +#include <kcstashdb.h> +#include <kccachedb.h> +#include <kchashdb.h> +#include <kcdirdb.h> +#include <kctextdb.h> + +namespace kyotocabinet { // common namespace + + +/** + * Polymorphic database. + * @note This class is a concrete class to operate an arbitrary database whose type is determined + * in runtime. This class can be inherited but overwriting methods is forbidden. Before every + * database operation, it is necessary to call the PolyDB::open method in order to open a + * database file and connect the database object to it. To avoid data missing or corruption, it + * is important to close every database file by the PolyDB::close method when the database is no + * longer in use. It is forbidden for multible database objects in a process to open the same + * database at the same time. It is forbidden to share a database object with child processes. + */ +class PolyDB : public BasicDB { + public: + class Cursor; + private: + class StreamLogger; + class StreamMetaTrigger; + struct SimilarKey; + struct MergeLine; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class PolyDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(PolyDB* db) : db_(db), cur_(NULL) { + _assert_(db); + if (db_->type_ == TYPEVOID) { + ProtoHashDB tmpdb; + cur_ = tmpdb.cursor(); + } else { + cur_ = db->db_->cursor(); + } + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + delete cur_; + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return cur_->accept(visitor, writable, step); + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return cur_->jump(); + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return cur_->jump(kbuf, ksiz); + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return jump(key.c_str(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, may provide a dummy implementation. + */ + bool jump_back() { + _assert_(true); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return cur_->jump_back(); + } + /** + * Jump the cursor to a record for backward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, will provide a dummy implementation. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return cur_->jump_back(kbuf, ksiz); + } + /** + * Jump the cursor to a record for backward scan. + * @note Equal to the original Cursor::jump_back method except that the parameter is + * std::string. + */ + bool jump_back(const std::string& key) { + _assert_(true); + return jump_back(key.c_str(), key.size()); + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return cur_->step(); + } + /** + * Step the cursor to the previous record. + * @return true on success, or false on failure. + * @note This method is dedicated to tree databases. Some database types, especially hash + * databases, may provide a dummy implementation. + */ + bool step_back() { + _assert_(true); + if (db_->type_ == TYPEVOID) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return cur_->step_back(); + } + /** + * Get the database object. + * @return the database object. + */ + PolyDB* db() { + _assert_(true); + return db_; + } + private: + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + PolyDB* db_; + /** The inner cursor. */ + BasicDB::Cursor* cur_; + }; + /** + * Merge modes. + */ + enum MergeMode { + MSET, ///< overwrite the existing value + MADD, ///< keep the existing value + MREPLACE, ///< modify the existing record only + MAPPEND ///< append the new value + }; + /** + * Default constructor. + */ + explicit PolyDB() : + type_(TYPEVOID), db_(NULL), error_(), + stdlogstrm_(NULL), stdlogger_(NULL), logger_(NULL), logkinds_(0), + stdmtrgstrm_(NULL), stdmtrigger_(NULL), mtrigger_(NULL), zcomp_(NULL) { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~PolyDB() { + _assert_(true); + if (type_ != TYPEVOID) close(); + delete zcomp_; + delete stdmtrigger_; + delete stdmtrgstrm_; + delete stdlogger_; + delete stdlogstrm_; + } + /** + * Set the internal database object. + * @param db the internal database object. Its possession is transferred inside and the + * object is deleted automatically. + * @return true on success, or false on failure. + */ + bool set_internal_db(BasicDB* db) { + _assert_(db); + if (type_ != TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + type_ = TYPEMISC; + db_ = db; + return true; + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->accept(kbuf, ksiz, visitor, writable); + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->accept_bulk(keys, visitor, writable); + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->iterate(visitor, writable, checker); + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->scan_parallel(visitor, thnum, checker); + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + if (type_ == TYPEVOID) return error_; + return db_->error(); + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + if (type_ == TYPEVOID) { + error_.set(code, message); + return; + } + db_->set_error(file, line, func, code, message); + } + /** + * Set the error information without source code information. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(Error::Code code, const char* message) { + _assert_(message); + if (type_ == TYPEVOID) { + error_.set(code, message); + return; + } + db_->set_error(_KCCODELINE_, code, message); + } + /** + * Open a database file. + * @param path the path of a database file. If it is "-", the database will be a prototype + * hash database. If it is "+", the database will be a prototype tree database. If it is ":", + * the database will be a stash database. If it is "*", the database will be a cache hash + * database. If it is "%", the database will be a cache tree database. If its suffix is + * ".kch", the database will be a file hash database. If its suffix is ".kct", the database + * will be a file tree database. If its suffix is ".kcd", the database will be a directory + * hash database. If its suffix is ".kcf", the database will be a directory tree database. + * If its suffix is ".kcx", the database will be a plain text database. Otherwise, this + * function fails. Tuning parameters can trail the name, separated by "#". Each parameter is + * composed of the name and the value, separated by "=". If the "type" parameter is specified, + * the database type is determined by the value in "-", "+", ":", "*", "%", "kch", "kct", + * "kcd", kcf", and "kcx". All database types support the logging parameters of "log", + * "logkinds", and "logpx". The prototype hash database and the prototype tree database do + * not support any other tuning parameter. The stash database supports "bnum". The cache + * hash database supports "opts", "bnum", "zcomp", "capcnt", "capsiz", and "zkey". The cache + * tree database supports all parameters of the cache hash database except for capacity + * limitation, and supports "psiz", "rcomp", "pccap" in addition. The file hash database + * supports "apow", "fpow", "opts", "bnum", "msiz", "dfunit", "zcomp", and "zkey". The file + * tree database supports all parameters of the file hash database and "psiz", "rcomp", + * "pccap" in addition. The directory hash database supports "opts", "zcomp", and "zkey". + * The directory tree database supports all parameters of the directory hash database and + * "psiz", "rcomp", "pccap" in addition. The plain text database does not support any other + * tuning parameter. + * @param mode the connection mode. PolyDB::OWRITER as a writer, PolyDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: PolyDB::OCREATE, + * which means it creates a new database if the file does not exist, PolyDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, PolyDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, PolyDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: PolyDB::ONOLOCK, which means it opens the database file without file locking, + * PolyDB::OTRYLOCK, which means locking is performed without blocking, PolyDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note The tuning parameter "log" is for the original "tune_logger" and the value specifies + * the path of the log file, or "-" for the standard output, or "+" for the standard error. + * "logkinds" specifies kinds of logged messages and the value can be "debug", "info", "warn", + * or "error". "logpx" specifies the prefix of each log message. "opts" is for "tune_options" + * and the value can contain "s" for the small option, "l" for the linear option, and "c" for + * the compress option. "bnum" corresponds to "tune_bucket". "zcomp" is for "tune_compressor" + * and the value can be "zlib" for the ZLIB raw compressor, "def" for the ZLIB deflate + * compressor, "gz" for the ZLIB gzip compressor, "lzo" for the LZO compressor, "lzma" for the + * LZMA compressor, or "arc" for the Arcfour cipher. "zkey" specifies the cipher key of the + * compressor. "capcnt" is for "cap_count". "capsiz" is for "cap_size". "psiz" is for + * "tune_page". "rcomp" is for "tune_comparator" and the value can be "lex" for the lexical + * comparator, "dec" for the decimal comparator, "lexdesc" for the lexical descending + * comparator, or "decdesc" for the decimal descending comparator. "pccap" is for + * "tune_page_cache". "apow" is for "tune_alignment". "fpow" is for "tune_fbp". "msiz" is + * for "tune_map". "dfunit" is for "tune_defrag". Every opened database must be closed by + * the PolyDB::close method when it is no longer in use. It is not allowed for two or more + * database objects in the same process to keep their connections to the same database file at + * the same time. + */ + bool open(const std::string& path = ":", uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + if (type_ == TYPEMISC) { + if (logger_) db_->tune_logger(logger_, logkinds_); + if (mtrigger_) db_->tune_meta_trigger(mtrigger_); + return db_->open(path, mode); + } + if (type_ != TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + std::vector<std::string> elems; + strsplit(path, '#', &elems); + std::string fpath; + Type type = TYPEVOID; + std::string logname = ""; + std::string logpx = ""; + uint32_t logkinds = Logger::WARN | Logger::ERROR; + std::string mtrgname = ""; + std::string mtrgpx = ""; + int64_t bnum = -1; + int64_t capcnt = -1; + int64_t capsiz = -1; + int32_t apow = -1; + int32_t fpow = -1; + bool tsmall = false; + bool tlinear = false; + bool tcompress = false; + int64_t msiz = -1; + int64_t dfunit = -1; + std::string zcompname = ""; + int64_t psiz = -1; + Comparator* rcomp = NULL; + int64_t pccap = 0; + std::string zkey = ""; + std::vector<std::string>::iterator it = elems.begin(); + std::vector<std::string>::iterator itend = elems.end(); + if (it != itend) { + fpath = *it; + ++it; + } + const char* fstr = fpath.c_str(); + const char* pv = std::strrchr(fstr, File::PATHCHR); + if (pv) fstr = pv + 1; + if (!std::strcmp(fstr, "-")) { + type = TYPEPHASH; + } else if (!std::strcmp(fstr, "+")) { + type = TYPEPTREE; + } else if (!std::strcmp(fstr, ":")) { + type = TYPESTASH; + } else if (!std::strcmp(fstr, "*")) { + type = TYPECACHE; + } else if (!std::strcmp(fstr, "%")) { + type = TYPEGRASS; + } else { + pv = std::strrchr(fstr, File::EXTCHR); + if (pv) { + pv++; + if (!std::strcmp(pv, "kcph") || !std::strcmp(pv, "phdb")) { + type = TYPEPHASH; + } else if (!std::strcmp(pv, "kcpt") || !std::strcmp(pv, "ptdb")) { + type = TYPEPTREE; + } else if (!std::strcmp(pv, "kcs") || !std::strcmp(pv, "sdb")) { + type = TYPESTASH; + } else if (!std::strcmp(pv, "kcc") || !std::strcmp(pv, "cdb")) { + type = TYPECACHE; + } else if (!std::strcmp(pv, "kcg") || !std::strcmp(pv, "gdb")) { + type = TYPEGRASS; + } else if (!std::strcmp(pv, "kch") || !std::strcmp(pv, "hdb")) { + type = TYPEHASH; + } else if (!std::strcmp(pv, "kct") || !std::strcmp(pv, "tdb")) { + type = TYPETREE; + } else if (!std::strcmp(pv, "kcd") || !std::strcmp(pv, "ddb")) { + type = TYPEDIR; + } else if (!std::strcmp(pv, "kcf") || !std::strcmp(pv, "fdb")) { + type = TYPEFOREST; + } else if (!std::strcmp(pv, "kcx") || !std::strcmp(pv, "xdb") || + !std::strcmp(pv, "txt") || !std::strcmp(pv, "text") || + !std::strcmp(pv, "tsv") || !std::strcmp(pv, "csv")) { + type = TYPETEXT; + } + } + } + while (it != itend) { + std::vector<std::string> fields; + if (strsplit(*it, '=', &fields) > 1) { + const char* key = fields[0].c_str(); + const char* value = fields[1].c_str(); + if (!std::strcmp(key, "type")) { + if (!std::strcmp(value, "-") || !std::strcmp(value, "kcph") || + !std::strcmp(value, "phdb") || !std::strcmp(value, "phash")) { + type = TYPEPHASH; + } else if (!std::strcmp(value, "+") || !std::strcmp(value, "kcpt") || + !std::strcmp(value, "ptdb") || !std::strcmp(value, "ptree")) { + type = TYPEPTREE; + } else if (!std::strcmp(value, ":") || !std::strcmp(value, "kcs") || + !std::strcmp(value, "sdb") || !std::strcmp(value, "stash")) { + type = TYPESTASH; + } else if (!std::strcmp(value, "*") || !std::strcmp(value, "kcc") || + !std::strcmp(value, "cdb") || !std::strcmp(value, "cache")) { + type = TYPECACHE; + } else if (!std::strcmp(value, "%") || !std::strcmp(value, "kcg") || + !std::strcmp(value, "gdb") || !std::strcmp(value, "grass")) { + type = TYPEGRASS; + } else if (!std::strcmp(value, "kch") || !std::strcmp(value, "hdb") || + !std::strcmp(value, "hash")) { + type = TYPEHASH; + } else if (!std::strcmp(value, "kct") || !std::strcmp(value, "tdb") || + !std::strcmp(value, "tree")) { + type = TYPETREE; + } else if (!std::strcmp(value, "kcd") || !std::strcmp(value, "ddb") || + !std::strcmp(value, "dir") || !std::strcmp(value, "directory")) { + type = TYPEDIR; + } else if (!std::strcmp(value, "kcf") || !std::strcmp(value, "fdb") || + !std::strcmp(value, "for") || !std::strcmp(value, "forest")) { + type = TYPEFOREST; + } else if (!std::strcmp(value, "kcx") || !std::strcmp(value, "xdb") || + !std::strcmp(value, "txt") || !std::strcmp(value, "text")) { + type = TYPETEXT; + } + } else if (!std::strcmp(key, "log") || !std::strcmp(key, "logger")) { + logname = value; + } else if (!std::strcmp(key, "logkinds") || !std::strcmp(key, "logkind")) { + if (!std::strcmp(value, "debug") || !std::strcmp(value, "debugging")) { + logkinds = Logger::DEBUG | Logger::INFO | Logger::WARN | Logger::ERROR; + } else if (!std::strcmp(value, "info") || !std::strcmp(value, "information")) { + logkinds = Logger::INFO | Logger::WARN | Logger::ERROR; + } else if (!std::strcmp(value, "warn") || !std::strcmp(value, "warning")) { + logkinds = Logger::WARN | Logger::ERROR; + } else if (!std::strcmp(value, "error") || !std::strcmp(value, "fatal")) { + logkinds = Logger::ERROR; + } else { + logkinds = atoix(value); + } + } else if (!std::strcmp(key, "logpx") || !std::strcmp(key, "lpx")) { + logpx = value; + } else if (!std::strcmp(key, "mtrg") || !std::strcmp(key, "metatrigger") || + !std::strcmp(key, "meta_trigger")) { + mtrgname = value; + } else if (!std::strcmp(key, "mtrgpx") || !std::strcmp(key, "mtpx")) { + mtrgpx = value; + } else if (!std::strcmp(key, "bnum") || !std::strcmp(key, "buckets")) { + bnum = atoix(value); + } else if (!std::strcmp(key, "capcnt") || !std::strcmp(key, "capcount") || + !std::strcmp(key, "cap_count")) { + capcnt = atoix(value); + } else if (!std::strcmp(key, "capsiz") || !std::strcmp(key, "capsize") || + !std::strcmp(key, "cap_size")) { + capsiz = atoix(value); + } else if (!std::strcmp(key, "apow") || !std::strcmp(key, "alignment")) { + apow = atoix(value); + } else if (!std::strcmp(key, "fpow") || !std::strcmp(key, "fbp")) { + fpow = atoix(value); + } else if (!std::strcmp(key, "opts") || !std::strcmp(key, "options")) { + if (std::strchr(value, 's')) tsmall = true; + if (std::strchr(value, 'l')) tlinear = true; + if (std::strchr(value, 'c')) tcompress = true; + } else if (!std::strcmp(key, "msiz") || !std::strcmp(key, "map")) { + msiz = atoix(value); + } else if (!std::strcmp(key, "dfunit") || !std::strcmp(key, "defrag")) { + dfunit = atoix(value); + } else if (!std::strcmp(key, "zcomp") || !std::strcmp(key, "compressor")) { + zcompname = value; + } else if (!std::strcmp(key, "psiz") || !std::strcmp(key, "page")) { + psiz = atoix(value); + } else if (!std::strcmp(key, "pccap") || !std::strcmp(key, "cache")) { + pccap = atoix(value); + } else if (!std::strcmp(key, "rcomp") || !std::strcmp(key, "comparator")) { + if (!std::strcmp(value, "lex") || !std::strcmp(value, "lexical")) { + rcomp = LEXICALCOMP; + } else if (!std::strcmp(value, "dec") || !std::strcmp(value, "decimal")) { + rcomp = DECIMALCOMP; + } else if (!std::strcmp(value, "lexdesc") || !std::strcmp(value, "lexicaldesc")) { + rcomp = LEXICALDESCCOMP; + } else if (!std::strcmp(value, "decdesc") || !std::strcmp(value, "decimaldesc")) { + rcomp = DECIMALDESCCOMP; + } + } else if (!std::strcmp(key, "zkey") || !std::strcmp(key, "pass") || + !std::strcmp(key, "password")) { + zkey = value; + } + } + ++it; + } + delete stdlogger_; + delete stdlogstrm_; + stdlogstrm_ = NULL; + stdlogger_ = NULL; + if (!logname.empty()) { + std::ostream* stdlogstrm = NULL; + if (logname == "-" || logname == "[stdout]" || logname == "[cout]") { + stdlogstrm = &std::cout; + } else if (logname == "+" || logname == "[stderr]" || logname == "[cerr]") { + stdlogstrm = &std::cerr; + } else { + std::ofstream *ofs = new std::ofstream; + ofs->open(logname.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::app); + if (!*ofs) ofs->open(logname.c_str(), std::ios_base::out | std::ios_base::binary); + if (ofs) { + stdlogstrm = ofs; + stdlogstrm_ = ofs; + } else { + delete ofs; + } + } + if (stdlogstrm) stdlogger_ = new StreamLogger(stdlogstrm, logpx.c_str()); + } + delete stdmtrigger_; + delete stdmtrgstrm_; + stdmtrgstrm_ = NULL; + stdmtrigger_ = NULL; + if (!mtrgname.empty()) { + std::ostream* stdmtrgstrm = NULL; + if (mtrgname == "-" || mtrgname == "[stdout]" || mtrgname == "[cout]") { + stdmtrgstrm = &std::cout; + } else if (mtrgname == "+" || mtrgname == "[stderr]" || mtrgname == "[cerr]") { + stdmtrgstrm = &std::cerr; + } else { + std::ofstream *ofs = new std::ofstream; + ofs->open(mtrgname.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::app); + if (!*ofs) ofs->open(mtrgname.c_str(), std::ios_base::out | std::ios_base::binary); + if (ofs) { + stdmtrgstrm = ofs; + stdmtrgstrm_ = ofs; + } else { + delete ofs; + } + } + if (stdmtrgstrm) stdmtrigger_ = new StreamMetaTrigger(stdmtrgstrm, mtrgpx.c_str()); + } + delete zcomp_; + zcomp_ = NULL; + ArcfourCompressor* arccomp = NULL; + if (!zcompname.empty()) { + if (zcompname == "zlib" || zcompname == "raw") { + zcomp_ = new ZLIBCompressor<ZLIB::RAW>; + } else if (zcompname == "def" || zcompname == "deflate") { + zcomp_ = new ZLIBCompressor<ZLIB::DEFLATE>; + } else if (zcompname == "gz" || zcompname == "gzip") { + zcomp_ = new ZLIBCompressor<ZLIB::GZIP>; + } else if (zcompname == "lzo" || zcompname == "oz") { + zcomp_ = new LZOCompressor<LZO::RAW>; + } else if (zcompname == "lzocrc" || zcompname == "ozcrc") { + zcomp_ = new LZOCompressor<LZO::CRC>; + } else if (zcompname == "lzma" || zcompname == "xz") { + zcomp_ = new LZMACompressor<LZMA::RAW>; + } else if (zcompname == "lzmacrc" || zcompname == "xzcrc") { + zcomp_ = new LZMACompressor<LZMA::CRC>; + } else if (zcompname == "lzmasha" || zcompname == "xzsha") { + zcomp_ = new LZMACompressor<LZMA::SHA>; + } else if (zcompname == "arc" || zcompname == "rc4") { + arccomp = new ArcfourCompressor(); + zcomp_ = arccomp; + } else if (zcompname == "arcz" || zcompname == "rc4z") { + arccomp = new ArcfourCompressor(); + arccomp->set_compressor(ZLIBRAWCOMP); + zcomp_ = arccomp; + } + } + BasicDB *db; + switch (type) { + default: { + set_error(_KCCODELINE_, Error::INVALID, "unknown database type"); + return false; + } + case TYPEPHASH: { + ProtoHashDB* phdb = new ProtoHashDB(); + if (stdlogger_) { + phdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + phdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + phdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + phdb->tune_meta_trigger(mtrigger_); + } + db = phdb; + break; + } + case TYPEPTREE: { + ProtoTreeDB *ptdb = new ProtoTreeDB(); + if (stdlogger_) { + ptdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + ptdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + ptdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + ptdb->tune_meta_trigger(mtrigger_); + } + db = ptdb; + break; + } + case TYPESTASH: { + StashDB* sdb = new StashDB(); + if (stdlogger_) { + sdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + sdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + sdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + sdb->tune_meta_trigger(mtrigger_); + } + if (bnum > 0) sdb->tune_buckets(bnum); + db = sdb; + break; + } + case TYPECACHE: { + int8_t opts = 0; + if (tcompress) opts |= CacheDB::TCOMPRESS; + CacheDB* cdb = new CacheDB(); + if (stdlogger_) { + cdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + cdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + cdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + cdb->tune_meta_trigger(mtrigger_); + } + if (opts > 0) cdb->tune_options(opts); + if (bnum > 0) cdb->tune_buckets(bnum); + if (zcomp_) cdb->tune_compressor(zcomp_); + if (capcnt > 0) cdb->cap_count(capcnt); + if (capsiz > 0) cdb->cap_size(capsiz); + db = cdb; + break; + } + case TYPEGRASS: { + int8_t opts = 0; + if (tcompress) opts |= GrassDB::TCOMPRESS; + GrassDB* gdb = new GrassDB(); + if (stdlogger_) { + gdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + gdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + gdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + gdb->tune_meta_trigger(mtrigger_); + } + if (opts > 0) gdb->tune_options(opts); + if (bnum > 0) gdb->tune_buckets(bnum); + if (psiz > 0) gdb->tune_page(psiz); + if (zcomp_) gdb->tune_compressor(zcomp_); + if (pccap > 0) gdb->tune_page_cache(pccap); + if (rcomp) gdb->tune_comparator(rcomp); + db = gdb; + break; + } + case TYPEHASH: { + int8_t opts = 0; + if (tsmall) opts |= HashDB::TSMALL; + if (tlinear) opts |= HashDB::TLINEAR; + if (tcompress) opts |= HashDB::TCOMPRESS; + HashDB* hdb = new HashDB(); + if (stdlogger_) { + hdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + hdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + hdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + hdb->tune_meta_trigger(mtrigger_); + } + if (apow >= 0) hdb->tune_alignment(apow); + if (fpow >= 0) hdb->tune_fbp(fpow); + if (opts > 0) hdb->tune_options(opts); + if (bnum > 0) hdb->tune_buckets(bnum); + if (msiz >= 0) hdb->tune_map(msiz); + if (dfunit > 0) hdb->tune_defrag(dfunit); + if (zcomp_) hdb->tune_compressor(zcomp_); + db = hdb; + break; + } + case TYPETREE: { + int8_t opts = 0; + if (tsmall) opts |= TreeDB::TSMALL; + if (tlinear) opts |= TreeDB::TLINEAR; + if (tcompress) opts |= TreeDB::TCOMPRESS; + TreeDB* tdb = new TreeDB(); + if (stdlogger_) { + tdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + tdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + tdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + tdb->tune_meta_trigger(mtrigger_); + } + if (apow >= 0) tdb->tune_alignment(apow); + if (fpow >= 0) tdb->tune_fbp(fpow); + if (opts > 0) tdb->tune_options(opts); + if (bnum > 0) tdb->tune_buckets(bnum); + if (psiz > 0) tdb->tune_page(psiz); + if (msiz >= 0) tdb->tune_map(msiz); + if (dfunit > 0) tdb->tune_defrag(dfunit); + if (zcomp_) tdb->tune_compressor(zcomp_); + if (pccap > 0) tdb->tune_page_cache(pccap); + if (rcomp) tdb->tune_comparator(rcomp); + db = tdb; + break; + } + case TYPEDIR: { + int8_t opts = 0; + if (tcompress) opts |= DirDB::TCOMPRESS; + DirDB* ddb = new DirDB(); + if (stdlogger_) { + ddb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + ddb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + ddb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + ddb->tune_meta_trigger(mtrigger_); + } + if (opts > 0) ddb->tune_options(opts); + if (zcomp_) ddb->tune_compressor(zcomp_); + db = ddb; + break; + } + case TYPEFOREST: { + int8_t opts = 0; + if (tcompress) opts |= TreeDB::TCOMPRESS; + ForestDB* fdb = new ForestDB(); + if (stdlogger_) { + fdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + fdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + fdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + fdb->tune_meta_trigger(mtrigger_); + } + if (opts > 0) fdb->tune_options(opts); + if (bnum > 0) fdb->tune_buckets(bnum); + if (psiz > 0) fdb->tune_page(psiz); + if (zcomp_) fdb->tune_compressor(zcomp_); + if (pccap > 0) fdb->tune_page_cache(pccap); + if (rcomp) fdb->tune_comparator(rcomp); + db = fdb; + break; + } + case TYPETEXT: { + TextDB* xdb = new TextDB(); + if (stdlogger_) { + xdb->tune_logger(stdlogger_, logkinds); + } else if (logger_) { + xdb->tune_logger(logger_, logkinds_); + } + if (stdmtrigger_) { + xdb->tune_meta_trigger(stdmtrigger_); + } else if (mtrigger_) { + xdb->tune_meta_trigger(mtrigger_); + } + db = xdb; + break; + } + } + if (arccomp) arccomp->set_key(zkey.c_str(), zkey.size()); + if (!db->open(fpath, mode)) { + const Error& error = db->error(); + set_error(_KCCODELINE_, error.code(), error.message()); + delete db; + return false; + } + if (arccomp) { + const std::string& apath = File::absolute_path(fpath); + uint64_t hash = (hashmurmur(apath.c_str(), apath.size()) >> 16) << 40; + hash += (uint64_t)(time() * 256); + arccomp->begin_cycle(hash); + } + type_ = type; + db_ = db; + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bool err = false; + if (!db_->close()) { + const Error& error = db_->error(); + set_error(_KCCODELINE_, error.code(), error.message()); + err = true; + } + delete zcomp_; + delete stdmtrigger_; + delete stdmtrgstrm_; + delete stdlogger_; + delete stdlogstrm_; + delete db_; + type_ = TYPEVOID; + db_ = NULL; + stdlogstrm_ = NULL; + stdlogger_ = NULL; + stdmtrgstrm_ = NULL; + stdmtrigger_ = NULL; + zcomp_ = NULL; + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->synchronize(hard, proc, checker); + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->occupy(writable, proc); + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->begin_transaction(hard); + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->begin_transaction_try(hard); + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->end_transaction(commit); + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->clear(); + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return db_->count(); + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return db_->size(); + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return ""; + } + return db_->path(); + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + return db_->status(strmap); + } + /** + * Reveal the inner database object. + * @return the inner database object, or NULL on failure. + */ + BasicDB* reveal_inner_db() { + _assert_(true); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return db_; + } + /** + * Get keys matching a prefix string. + * @param prefix the prefix string. + * @param strvec a string vector to contain the result. + * @param max the maximum number to retrieve. If it is negative, no limit is specified. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return the number of retrieved keys or -1 on failure. + */ + int64_t match_prefix(const std::string& prefix, std::vector<std::string>* strvec, + int64_t max = -1, ProgressChecker* checker = NULL) { + _assert_(strvec); + const char* pbuf = prefix.data(); + size_t psiz = prefix.size(); + if (max < 0) max = INT64MAX; + Comparator* comp; + switch (type_) { + case TYPEPTREE: { + comp = LEXICALCOMP; + break; + } + case TYPEGRASS: { + comp = ((GrassDB*)db_)->rcomp(); + break; + } + case TYPETREE: { + comp = ((TreeDB*)db_)->rcomp(); + break; + } + case TYPEFOREST: { + comp = ((ForestDB*)db_)->rcomp(); + break; + } + default: { + comp = NULL; + break; + } + } + bool err = false; + int64_t allcnt = count(); + if (checker && !checker->check("match_prefix", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + strvec->clear(); + Cursor* cur = cursor(); + int64_t curcnt = 0; + if (comp == LEXICALCOMP) { + if (cur->jump(pbuf, psiz)) { + while ((int64_t)strvec->size() < max) { + size_t ksiz; + char* kbuf = cur->get_key(&ksiz, true); + if (kbuf) { + if (ksiz >= psiz && !std::memcmp(kbuf, pbuf, psiz)) { + strvec->push_back(std::string(kbuf, ksiz)); + } else { + delete[] kbuf; + break; + } + delete[] kbuf; + } else { + if (cur->error() != Error::NOREC) err = true; + break; + } + curcnt++; + if (checker && !checker->check("match_prefix", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + } + } else if (cur->error() != Error::NOREC) { + err = true; + } + } else { + if (cur->jump()) { + while ((int64_t)strvec->size() < max) { + size_t ksiz; + char* kbuf = cur->get_key(&ksiz, true); + if (kbuf) { + if (ksiz >= psiz && !std::memcmp(kbuf, pbuf, psiz)) + strvec->push_back(std::string(kbuf, ksiz)); + delete[] kbuf; + } else { + if (cur->error() != Error::NOREC) err = true; + break; + } + curcnt++; + if (checker && !checker->check("match_prefix", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + } + } else if (cur->error() != Error::NOREC) { + err = true; + } + } + if (checker && !checker->check("match_prefix", "ending", strvec->size(), allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + delete cur; + return err ? -1 : strvec->size(); + } + /** + * Get keys matching a regular expression string. + * @param regex the regular expression string. + * @param strvec a string vector to contain the result. + * @param max the maximum number to retrieve. If it is negative, no limit is specified. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return the number of retrieved keys or -1 on failure. + */ + int64_t match_regex(const std::string& regex, std::vector<std::string>* strvec, + int64_t max = -1, ProgressChecker* checker = NULL) { + _assert_(strvec); + if (max < 0) max = INT64MAX; + Regex reg; + if (!reg.compile(regex, Regex::MATCHONLY)) { + set_error(_KCCODELINE_, Error::LOGIC, "compilation failed"); + return -1; + } + bool err = false; + int64_t allcnt = count(); + if (checker && !checker->check("match_regex", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + strvec->clear(); + Cursor* cur = cursor(); + int64_t curcnt = 0; + if (cur->jump()) { + while ((int64_t)strvec->size() < max) { + size_t ksiz; + char* kbuf = cur->get_key(&ksiz, true); + if (kbuf) { + std::string key(kbuf, ksiz); + if (reg.match(key)) strvec->push_back(key); + delete[] kbuf; + } else { + if (cur->error() != Error::NOREC) err = true; + break; + } + curcnt++; + if (checker && !checker->check("match_regex", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + } + } else if (cur->error() != Error::NOREC) { + err = true; + } + if (checker && !checker->check("match_regex", "ending", strvec->size(), allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + delete cur; + return err ? -1 : strvec->size(); + } + /** + * Get keys similar to a string in terms of the levenshtein distance. + * @param origin the origin string. + * @param range the maximum distance of keys to adopt. + * @param utf flag to treat keys as UTF-8 strings. + * @param strvec a string vector to contain the result. + * @param max the maximum number to retrieve. If it is negative, no limit is specified. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return the number of retrieved keys or -1 on failure. + */ + int64_t match_similar(const std::string& origin, size_t range, bool utf, + std::vector<std::string>* strvec, + int64_t max = -1, ProgressChecker* checker = NULL) { + _assert_(strvec); + if (max < 0) max = INT64MAX; + bool err = false; + int64_t allcnt = count(); + if (checker && !checker->check("match_similar", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + strvec->clear(); + uint32_t ostack[128]; + uint32_t* oary = NULL; + size_t onum = 0; + if (utf) { + const char* ostr = origin.c_str(); + onum = strutflen(ostr); + oary = onum > sizeof(ostack) / sizeof(*ostack) ? new uint32_t[onum] : ostack; + strutftoucs(ostr, oary, &onum); + } + Cursor* cur = cursor(); + int64_t curcnt = 0; + std::priority_queue<SimilarKey> queue; + if (cur->jump()) { + if (max > 0) { + while (true) { + size_t ksiz; + char* kbuf = cur->get_key(&ksiz, true); + if (kbuf) { + size_t kdist; + if (oary) { + uint32_t kstack[128]; + uint32_t* kary = ksiz > sizeof(kstack) / sizeof(*kstack) ? + new uint32_t[ksiz] : kstack; + size_t knum; + strutftoucs(kbuf, ksiz, kary, &knum); + kdist = std::labs((long)onum - (long)knum) > (long)range ? + UINT32MAX : strucsdist(oary, onum, kary, knum); + if (kary != kstack) delete[] kary; + } else { + kdist = std::labs((long)origin.size() - (long)ksiz) > (long)range ? + UINT32MAX : memdist(origin.data(), origin.size(), kbuf, ksiz); + } + if (kdist <= range) { + std::string key(kbuf, ksiz); + if ((int64_t)queue.size() < max) { + SimilarKey skey = { kdist, key, curcnt }; + queue.push(skey); + } else { + const SimilarKey& top = queue.top(); + if (!top.less(kdist, key, curcnt)) { + queue.pop(); + SimilarKey skey = { kdist, key, curcnt }; + queue.push(skey); + } + } + } + delete[] kbuf; + } else { + if (cur->error() != Error::NOREC) err = true; + break; + } + curcnt++; + if (checker && !checker->check("match_similar", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + } + while (!queue.empty()) { + const SimilarKey& top = queue.top(); + strvec->push_back(top.key); + queue.pop(); + } + size_t end = strvec->size() - 1; + size_t mid = strvec->size() / 2; + for (size_t i = 0; i < mid; i++) { + (*strvec)[i].swap((*strvec)[end-i]); + } + } + } else if (cur->error() != Error::NOREC) { + err = true; + } + if (checker && !checker->check("match_similar", "ending", strvec->size(), allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + delete cur; + if (oary && oary != ostack) delete[] oary; + return err ? -1 : strvec->size(); + } + /** + * Merge records from other databases. + * @param srcary an array of the source detabase objects. + * @param srcnum the number of the elements of the source array. + * @param mode the merge mode. PolyDB::MSET to overwrite the existing value, PolyDB::MADD to + * keep the existing value, PolyDB::MREPLACE to modify the existing record only, + * PolyDB::MAPPEND to append the new value. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + */ + bool merge(BasicDB** srcary, size_t srcnum, MergeMode mode = MSET, + ProgressChecker* checker = NULL) { + _assert_(srcary && srcnum <= MEMMAXSIZ); + if (type_ == TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bool err = false; + Comparator* comp; + switch (type_) { + case TYPEGRASS: { + comp = ((GrassDB*)db_)->rcomp(); + break; + } + case TYPETREE: { + comp = ((TreeDB*)db_)->rcomp(); + break; + } + case TYPEFOREST: { + comp = ((ForestDB*)db_)->rcomp(); + break; + } + default: { + comp = NULL; + break; + } + } + if (!comp) comp = LEXICALCOMP; + std::priority_queue<MergeLine> lines; + int64_t allcnt = 0; + for (size_t i = 0; i < srcnum; i++) { + MergeLine line; + line.cur = srcary[i]->cursor(); + line.comp = comp; + line.cur->jump(); + line.kbuf = line.cur->get(&line.ksiz, &line.vbuf, &line.vsiz, true); + if (line.kbuf) { + lines.push(line); + int64_t count = srcary[i]->count(); + if (count > 0) allcnt += count; + } else { + delete line.cur; + } + } + if (checker && !checker->check("merge", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + int64_t curcnt = 0; + while (!err && !lines.empty()) { + MergeLine line = lines.top(); + lines.pop(); + switch (mode) { + case MSET: { + if (!set(line.kbuf, line.ksiz, line.vbuf, line.vsiz)) err = true; + break; + } + case MADD: { + if (!add(line.kbuf, line.ksiz, line.vbuf, line.vsiz) && + error() != Error::DUPREC) err = true; + break; + } + case MREPLACE: { + if (!replace(line.kbuf, line.ksiz, line.vbuf, line.vsiz) && + error() != Error::NOREC) err = true; + break; + } + case MAPPEND: { + if (!append(line.kbuf, line.ksiz, line.vbuf, line.vsiz)) err = true; + break; + } + } + delete[] line.kbuf; + line.kbuf = line.cur->get(&line.ksiz, &line.vbuf, &line.vsiz, true); + if (line.kbuf) { + lines.push(line); + } else { + delete line.cur; + } + curcnt++; + if (checker && !checker->check("merge", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + break; + } + } + if (checker && !checker->check("merge", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + while (!lines.empty()) { + MergeLine line = lines.top(); + lines.pop(); + delete[] line.kbuf; + delete line.cur; + } + return !err; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + if (logger_) { + logger_->log(file, line, func, kind, message); + } else if (type_ != TYPEVOID) { + db_->log(file, line, func, kind, message); + } + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + if (type_ != TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + logger_ = logger; + logkinds_ = kinds; + return true; + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + if (type_ != TYPEVOID) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + private: + /** + * Stream logger implementation. + */ + class StreamLogger : public Logger { + public: + /** constructor */ + StreamLogger(std::ostream* strm, const char* prefix) : strm_(strm), prefix_(prefix) {} + /** print a log message */ + void log(const char* file, int32_t line, const char* func, Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + const char* kstr = "MISC"; + switch (kind) { + case Logger::DEBUG: kstr = "DEBUG"; break; + case Logger::INFO: kstr = "INFO"; break; + case Logger::WARN: kstr = "WARN"; break; + case Logger::ERROR: kstr = "ERROR"; break; + } + if (!prefix_.empty()) *strm_ << prefix_ << ": "; + *strm_ << "[" << kstr << "]: " << file << ": " << line << ": " << func << ": " << + message << std::endl; + } + private: + std::ostream* strm_; ///< output stream + std::string prefix_; ///< prefix of each message + }; + /** + * Stream meta operation trigger implementation. + */ + class StreamMetaTrigger : public MetaTrigger { + public: + /** constructor */ + StreamMetaTrigger(std::ostream* strm, const char* prefix) : strm_(strm), prefix_(prefix) {} + /** print a meta operation */ + void trigger(Kind kind, const char* message) { + _assert_(message); + const char* kstr = "unknown"; + switch (kind) { + case MetaTrigger::OPEN: kstr = "OPEN"; break; + case MetaTrigger::CLOSE: kstr = "CLOSE"; break; + case MetaTrigger::CLEAR: kstr = "CLEAR"; break; + case MetaTrigger::ITERATE: kstr = "ITERATE"; break; + case MetaTrigger::SYNCHRONIZE: kstr = "SYNCHRONIZE"; break; + case MetaTrigger::OCCUPY: kstr = "OCCUPY"; break; + case MetaTrigger::BEGINTRAN: kstr = "BEGINTRAN"; break; + case MetaTrigger::COMMITTRAN: kstr = "COMMITTRAN"; break; + case MetaTrigger::ABORTTRAN: kstr = "ABORTTRAN"; break; + case MetaTrigger::MISC: kstr = "MISC"; break; + } + if (!prefix_.empty()) *strm_ << prefix_ << ": "; + *strm_ << "[" << kstr << "]: " << message << std::endl; + } + private: + std::ostream* strm_; ///< output stream + std::string prefix_; ///< prefix of each message + }; + /** + * Key for similarity search. + */ + struct SimilarKey { + size_t dist; + std::string key; + int64_t order; + bool operator <(const SimilarKey& right) const { + if (dist != right.dist) return dist < right.dist; + if (key != right.key) return key < right.key; + return order < right.order; + } + bool less(size_t rdist, const std::string& rkey, uint32_t rorder) const { + if (dist != rdist) return dist < rdist; + if (key != rkey) return key < rkey; + return order < rorder; + } + }; + /** + * Front line of a merging list. + */ + struct MergeLine { + BasicDB::Cursor* cur; ///< cursor + Comparator* comp; ///< comparator + char* kbuf; ///< pointer to the key + size_t ksiz; ///< size of the key + const char* vbuf; ///< pointer to the value + size_t vsiz; ///< size of the value + /** comparing operator */ + bool operator <(const MergeLine& right) const { + return comp->compare(kbuf, ksiz, right.kbuf, right.ksiz) > 0; + } + }; + /** Dummy constructor to forbid the use. */ + PolyDB(const PolyDB&); + /** Dummy Operator to forbid the use. */ + PolyDB& operator =(const PolyDB&); + /** The database type. */ + Type type_; + /** The internal database. */ + BasicDB* db_; + /** The last happened error. */ + Error error_; + /** The standard log stream. */ + std::ostream* stdlogstrm_; + /** The standard logger. */ + Logger* stdlogger_; + /** The internal logger. */ + Logger* logger_; + /** The kinds of logged messages. */ + uint32_t logkinds_; + /** The standard meta operation trigger stream. */ + std::ostream* stdmtrgstrm_; + /** The standard meta operation trigger. */ + MetaTrigger* stdmtrigger_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The custom compressor. */ + Compressor* zcomp_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcpolymgr.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolymgr.cc new file mode 100644 index 0000000000..67dc84450d --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolymgr.cc @@ -0,0 +1,1573 @@ +/************************************************************************************************* + * The command line utility of the polymorphic database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcpolydb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, const char* info); +static int32_t runcreate(int argc, char** argv); +static int32_t runinform(int argc, char** argv); +static int32_t runset(int argc, char** argv); +static int32_t runremove(int argc, char** argv); +static int32_t runget(int argc, char** argv); +static int32_t runlist(int argc, char** argv); +static int32_t runclear(int argc, char** argv); +static int32_t runimport(int argc, char** argv); +static int32_t runcopy(int argc, char** argv); +static int32_t rundump(int argc, char** argv); +static int32_t runload(int argc, char** argv); +static int32_t runmerge(int argc, char** argv); +static int32_t runsetbulk(int argc, char** argv); +static int32_t runremovebulk(int argc, char** argv); +static int32_t rungetbulk(int argc, char** argv); +static int32_t runcheck(int argc, char** argv); +static int32_t proccreate(const char* path, int32_t oflags); +static int32_t procinform(const char* path, int32_t oflags, bool st); +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode); +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags); +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz); +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + int32_t mode, bool des, int64_t max, bool rm, bool pv, bool px); +static int32_t procclear(const char* path, int32_t oflags); +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx); +static int32_t proccopy(const char* path, const char* file, int32_t oflags); +static int32_t procdump(const char* path, const char* file, int32_t oflags); +static int32_t procload(const char* path, const char* file, int32_t oflags); +static int32_t procmerge(const char* path, int32_t oflags, kc::PolyDB::MergeMode mode, + const std::vector<std::string>& srcpaths); +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs); +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys); +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px); +static int32_t proccheck(const char* path, int32_t oflags); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "create")) { + rv = runcreate(argc, argv); + } else if (!std::strcmp(argv[1], "inform")) { + rv = runinform(argc, argv); + } else if (!std::strcmp(argv[1], "set")) { + rv = runset(argc, argv); + } else if (!std::strcmp(argv[1], "remove")) { + rv = runremove(argc, argv); + } else if (!std::strcmp(argv[1], "get")) { + rv = runget(argc, argv); + } else if (!std::strcmp(argv[1], "list")) { + rv = runlist(argc, argv); + } else if (!std::strcmp(argv[1], "clear")) { + rv = runclear(argc, argv); + } else if (!std::strcmp(argv[1], "import")) { + rv = runimport(argc, argv); + } else if (!std::strcmp(argv[1], "copy")) { + rv = runcopy(argc, argv); + } else if (!std::strcmp(argv[1], "dump")) { + rv = rundump(argc, argv); + } else if (!std::strcmp(argv[1], "load")) { + rv = runload(argc, argv); + } else if (!std::strcmp(argv[1], "merge")) { + rv = runmerge(argc, argv); + } else if (!std::strcmp(argv[1], "setbulk")) { + rv = runsetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "removebulk")) { + rv = runremovebulk(argc, argv); + } else if (!std::strcmp(argv[1], "getbulk")) { + rv = rungetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "check")) { + rv = runcheck(argc, argv); + } else if (!std::strcmp(argv[1], "version") || !std::strcmp(argv[1], "--version")) { + printversion(); + } else { + usage(); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: the command line utility of the polymorphic database of Kyoto Cabinet\n", + g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s create [-otr] [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s inform [-onl|-otl|-onr] [-st] path\n", g_progname); + eprintf(" %s set [-onl|-otl|-onr] [-add|-rep|-app|-inci|-incd] [-sx] path key value\n", + g_progname); + eprintf(" %s remove [-onl|-otl|-onr] [-sx] path key\n", g_progname); + eprintf(" %s get [-onl|-otl|-onr] [-rm] [-sx] [-px] [-pz] path key\n", g_progname); + eprintf(" %s list [-onl|-otl|-onr] [-mp|-mr|-ms] [-des] [-max num] [-rm] [-sx] [-pv] [-px]" + " path [key]\n", g_progname); + eprintf(" %s clear [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s import [-onl|-otl|-onr] [-sx] path [file]\n", g_progname); + eprintf(" %s copy [-onl|-otl|-onr] path file\n", g_progname); + eprintf(" %s dump [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s load [-otr] [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s merge [-onl|-otl|-onr] [-add|-rep|-app] path src...\n", g_progname); + eprintf(" %s setbulk [-onl|-otl|-onr] [-sx] path key value ...\n", g_progname); + eprintf(" %s removebulk [-onl|-otl|-onr] [-sx] path key ...\n", g_progname); + eprintf(" %s getbulk [-onl|-otl|-onr] [-sx] [-px] path key ...\n", g_progname); + eprintf(" %s check [-onl|-otl|-onr] path\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print error message of database +static void dberrprint(kc::BasicDB* db, const char* info) { + const kc::BasicDB::Error& err = db->error(); + eprintf("%s: %s: %s: %d: %s: %s\n", + g_progname, info, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// parse arguments of create command +static int32_t runcreate(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::PolyDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccreate(path, oflags); + return rv; +} + + +// parse arguments of inform command +static int32_t runinform(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + bool st = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-st")) { + st = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procinform(path, oflags, st); + return rv; +} + + +// parse arguments of set command +static int32_t runset(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + const char* vstr = NULL; + int32_t oflags = 0; + int32_t mode = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-add")) { + mode = 'a'; + } else if (!std::strcmp(argv[i], "-rep")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-app")) { + mode = 'c'; + } else if (!std::strcmp(argv[i], "-inci")) { + mode = 'i'; + } else if (!std::strcmp(argv[i], "-incd")) { + mode = 'd'; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else if (!vstr) { + vstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr || !vstr) usage(); + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + int32_t rv = procset(path, kstr, ksiz, vstr, vsiz, oflags, mode); + delete[] kbuf; + delete[] vbuf; + return rv; +} + + +// parse arguments of remove command +static int32_t runremove(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procremove(path, kstr, ksiz, oflags); + delete[] kbuf; + return rv; +} + + +// parse arguments of get command +static int32_t runget(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool rm = false; + bool sx = false; + bool px = false; + bool pz = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else if (!std::strcmp(argv[i], "-pz")) { + pz = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procget(path, kstr, ksiz, oflags, rm, px, pz); + delete[] kbuf; + return rv; +} + + +// parse arguments of list command +static int32_t runlist(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + int32_t mode = 0; + bool des = false; + int64_t max = -1; + bool rm = false; + bool sx = false; + bool pv = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-mp")) { + mode = 'p'; + } else if (!std::strcmp(argv[i], "-mr")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-ms")) { + mode = 's'; + } else if (!std::strcmp(argv[i], "-des")) { + des = true; + } else if (!std::strcmp(argv[i], "-max")) { + if (++i >= argc) usage(); + max = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-pv")) { + pv = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + char* kbuf = NULL; + size_t ksiz = 0; + if (kstr) { + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = new char[ksiz+1]; + std::memcpy(kbuf, kstr, ksiz); + kbuf[ksiz] = '\0'; + } + } + int32_t rv = proclist(path, kbuf, ksiz, oflags, mode, des, max, rm, pv, px); + delete[] kbuf; + return rv; +} + + +// parse arguments of clear command +static int32_t runclear(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procclear(path, oflags); + return rv; +} + + +// parse arguments of import command +static int32_t runimport(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procimport(path, file, oflags, sx); + return rv; +} + + +// parse arguments of copy command +static int32_t runcopy(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path || !file) usage(); + int32_t rv = proccopy(path, file, oflags); + return rv; +} + + +// parse arguments of dump command +static int32_t rundump(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procdump(path, file, oflags); + return rv; +} + + +// parse arguments of load command +static int32_t runload(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::PolyDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procload(path, file, oflags); + return rv; +} + + +// parse arguments of merge command +static int32_t runmerge(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + kc::PolyDB::MergeMode mode = kc::PolyDB::MSET; + std::vector<std::string> srcpaths; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-add")) { + mode = kc::PolyDB::MADD; + } else if (!std::strcmp(argv[i], "-rep")) { + mode = kc::PolyDB::MREPLACE; + } else if (!std::strcmp(argv[i], "-app")) { + mode = kc::PolyDB::MAPPEND; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + srcpaths.push_back(argv[i]); + } + } + if (!path && srcpaths.size() < 1) usage(); + int32_t rv = procmerge(path, oflags, mode, srcpaths); + return rv; +} + + +// parse arguments of setbulk command +static int32_t runsetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::map<std::string, std::string> recs; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + if (++i >= argc) usage(); + const char* vstr = argv[i]; + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + std::string key(kstr, ksiz); + std::string value(vstr, vsiz); + recs[key] = value; + delete[] kbuf; + delete[] vbuf; + } + } + if (!path) usage(); + int32_t rv = procsetbulk(path, oflags, recs); + return rv; +} + + +// parse arguments of removebulk command +static int32_t runremovebulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procremovebulk(path, oflags, keys); + return rv; +} + + +// parse arguments of getbulk command +static int32_t rungetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procgetbulk(path, oflags, keys, px); + return rv; +} + + +// parse arguments of check command +static int32_t runcheck(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccheck(path, oflags); + return rv; +} + + +// perform create command +static int32_t proccreate(const char* path, int32_t oflags) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | kc::PolyDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform inform command +static int32_t procinform(const char* path, int32_t oflags, bool st) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (st) { + std::map<std::string, std::string> status; + if (db.status(&status)) { + std::map<std::string, std::string>::iterator it = status.begin(); + std::map<std::string, std::string>::iterator itend = status.end(); + while (it != itend) { + oprintf("%s: %s\n", it->first.c_str(), it->second.c_str()); + ++it; + } + } else { + dberrprint(&db, "DB::status failed"); + err = true; + } + } else { + oprintf("count: %lld\n", (long long)db.count()); + oprintf("size: %lld\n", (long long)db.size()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform set command +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + switch (mode) { + default: { + if (!db.set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 'a': { + if (!db.add(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::add failed"); + err = true; + } + break; + } + case 'r': { + if (!db.replace(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::replace failed"); + err = true; + } + break; + } + case 'c': { + if (!db.append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::append failed"); + err = true; + } + break; + } + case 'i': { + int64_t onum = db.increment(kbuf, ksiz, kc::atoi(vbuf)); + if (onum == kc::INT64MIN) { + dberrprint(&db, "DB::increment failed"); + err = true; + } else { + oprintf("%lld\n", (long long)onum); + } + break; + } + case 'd': { + double onum = db.increment_double(kbuf, ksiz, kc::atof(vbuf)); + if (kc::chknan(onum)) { + dberrprint(&db, "DB::increment_double failed"); + err = true; + } else { + oprintf("%f\n", onum); + } + break; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform remove command +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.remove(kbuf, ksiz)) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform get command +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::PolyDB::OWRITER : kc::PolyDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + char* vbuf; + size_t vsiz; + if (rm) { + vbuf = db.seize(kbuf, ksiz, &vsiz); + } else { + vbuf = db.get(kbuf, ksiz, &vsiz); + } + if (vbuf) { + printdata(vbuf, vsiz, px); + if (!pz) oprintf("\n"); + delete[] vbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform list command +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + int32_t mode, bool des, int64_t max, bool rm, bool pv, bool px) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::PolyDB::OWRITER : kc::PolyDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(bool rm, bool pv, bool px) : rm_(rm), pv_(pv), px_(px) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + printdata(kbuf, ksiz, px_); + if (pv_) { + oprintf("\t"); + printdata(vbuf, vsiz, px_); + } + oprintf("\n"); + return rm_ ? REMOVE : NOP; + } + bool rm_; + bool pv_; + bool px_; + } visitor(rm, pv, px); + class Printer { + public: + static bool print(kc::BasicDB* db, const std::vector<std::string>& keys, + bool des, bool rm, bool pv, bool px) { + bool err = false; + if (des) { + for (int64_t i = (int64_t)keys.size() - 1; i >= 0; i--) { + if (!proc_one(db, keys[i], rm, pv, px)) err = true; + } + } else { + std::vector<std::string>::const_iterator it = keys.begin(); + std::vector<std::string>::const_iterator itend = keys.end(); + while (it != itend) { + if (!proc_one(db, *it, rm, pv, px)) err = true; + ++it; + } + } + return !err; + } + private: + static bool proc_one(kc::BasicDB* db, const std::string& key, bool rm, bool pv, bool px) { + bool err = false; + printdata(key.data(), key.size(), px); + if (pv) { + size_t vsiz; + char* vbuf = db->get(key.data(), key.size(), &vsiz); + if (vbuf) { + oprintf("\t"); + printdata(vbuf, vsiz, px); + delete[] vbuf; + } else { + dberrprint(db, "DB::get failed"); + err = true; + } + } + oprintf("\n"); + if (rm && !db->remove(key.data(), key.size())) { + dberrprint(db, "DB::remove failed"); + err = true; + } + return !err; + } + }; + if (mode == 'p') { + std::vector<std::string> keys; + if (db.match_prefix(std::string(kbuf, ksiz), &keys, max) >= 0) { + if (!Printer::print(&db, keys, des, rm, pv, px)) err = true; + } else { + dberrprint(&db, "DB::match_prefix failed"); + err = true; + } + } else if (mode == 'r') { + std::vector<std::string> keys; + if (db.match_regex(std::string(kbuf, ksiz), &keys, max) >= 0) { + if (!Printer::print(&db, keys, des, rm, pv, px)) err = true; + } else { + dberrprint(&db, "DB::match_regex failed"); + err = true; + } + } else if (mode == 's') { + size_t range = ksiz / 3 + 1; + std::vector<std::string> keys; + if (db.match_similar(std::string(kbuf, ksiz), range, false, &keys, max) >= 0) { + if (!Printer::print(&db, keys, des, rm, pv, px)) err = true; + } else { + dberrprint(&db, "DB::match_similar failed"); + err = true; + } + } else if (kbuf || des || max >= 0) { + if (max < 0) max = kc::INT64MAX; + kc::PolyDB::Cursor cur(&db); + if (des) { + if (kbuf) { + if (!cur.jump_back(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump_back() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } else { + if (kbuf) { + if (!cur.jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } + } else { + if (!db.iterate(&visitor, rm)) { + dberrprint(&db, "DB::iterate failed"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform clear command +static int32_t procclear(const char* path, int32_t oflags) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.clear()) { + dberrprint(&db, "DB::clear failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform import command +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx) { + std::istream *is = &std::cin; + std::ifstream ifs; + if (file) { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | kc::PolyDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + int64_t cnt = 0; + std::string line; + std::vector<std::string> fields; + while (!err && mygetline(is, &line)) { + cnt++; + kc::strsplit(line, '\t', &fields); + if (sx) { + std::vector<std::string>::iterator it = fields.begin(); + std::vector<std::string>::iterator itend = fields.end(); + while (it != itend) { + size_t esiz; + char* ebuf = kc::hexdecode(it->c_str(), &esiz); + it->clear(); + it->append(ebuf, esiz); + delete[] ebuf; + ++it; + } + } + switch (fields.size()) { + case 2: { + if (!db.set(fields[0], fields[1])) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 1: { + if (!db.remove(fields[0]) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + break; + } + } + oputchar('.'); + if (cnt % 50 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + if (cnt % 50 > 0) oprintf(" (%lld)\n", (long long)cnt); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform copy command +static int32_t proccopy(const char* path, const char* file, int32_t oflags) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + DotChecker checker(&std::cout, -100); + if (!db.copy(file, &checker)) { + dberrprint(&db, "DB::copy failed"); + err = true; + } + oprintf(" (end)\n"); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld blocks were copied successfully\n", (long long)checker.count()); + return err ? 1 : 0; +} + + +// perform dump command +static int32_t procdump(const char* path, const char* file, int32_t oflags) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, 1000); + if (!db.dump_snapshot(file, &checker)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were dumped successfully\n", (long long)checker.count()); + } else { + if (!db.dump_snapshot(&std::cout)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform load command +static int32_t procload(const char* path, const char* file, int32_t oflags) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | kc::PolyDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(file, &checker)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } else { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(&std::cin)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform merge command +static int32_t procmerge(const char* path, int32_t oflags, kc::PolyDB::MergeMode mode, + const std::vector<std::string>& srcpaths) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | kc::PolyDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + kc::BasicDB** srcary = new kc::BasicDB*[srcpaths.size()]; + size_t srcnum = 0; + std::vector<std::string>::const_iterator it = srcpaths.begin(); + std::vector<std::string>::const_iterator itend = srcpaths.end(); + while (it != itend) { + const std::string srcpath = *it; + kc::PolyDB* srcdb = new kc::PolyDB; + if (srcdb->open(srcpath, kc::PolyDB::OREADER | oflags)) { + srcary[srcnum++] = srcdb; + } else { + dberrprint(srcdb, "DB::open failed"); + err = true; + delete srcdb; + } + ++it; + } + DotChecker checker(&std::cout, 1000); + if (!db.merge(srcary, srcnum, mode, &checker)) { + dberrprint(&db, "DB::merge failed"); + err = true; + } + oprintf(" (end)\n"); + for (size_t i = 0; i < srcnum; i++) { + kc::BasicDB* srcdb = srcary[i]; + if (!srcdb->close()) { + dberrprint(srcdb, "DB::close failed"); + err = true; + } + delete srcdb; + } + delete[] srcary; + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld records were merged successfully\n", (long long)checker.count()); + return err ? 1 : 0; +} + + +// perform setbulk command +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.set_bulk(recs) != (int64_t)recs.size()) { + dberrprint(&db, "DB::set_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform removebulk command +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.remove_bulk(keys) < 0) { + dberrprint(&db, "DB::remove_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform getbulk command +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + std::map<std::string, std::string> recs; + if (db.get_bulk(keys, &recs) >= 0) { + std::map<std::string, std::string>::iterator it = recs.begin(); + std::map<std::string, std::string>::iterator itend = recs.end(); + while (it != itend) { + printdata(it->first.data(), it->first.size(), px); + oprintf("\t"); + printdata(it->second.data(), it->second.size(), px); + oprintf("\n"); + ++it; + } + } else { + dberrprint(&db, "DB::get_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform check command +static int32_t proccheck(const char* path, int32_t oflags) { + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::PolyDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + kc::PolyDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::jump failed"); + err = true; + } + int64_t cnt = 0; + while (!err) { + size_t ksiz; + const char* vbuf; + size_t vsiz; + char* kbuf = cur.get(&ksiz, &vbuf, &vsiz); + if (kbuf) { + cnt++; + size_t rsiz; + char* rbuf = db.get(kbuf, ksiz, &rsiz); + if (rbuf) { + if (rsiz != vsiz || std::memcmp(rbuf, vbuf, rsiz)) { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] rbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] kbuf; + if (cnt % 1000 == 0) { + oputchar('.'); + if (cnt % 50000 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + } else { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::get failed"); + err = true; + } + break; + } + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::step failed"); + err = true; + } + } + oprintf(" (end)\n"); + if (db.count() != cnt) { + dberrprint(&db, "DB::count failed"); + err = true; + } + const std::string& rpath = db.path(); + kc::File::Status sbuf; + if (kc::File::status(rpath, &sbuf)) { + if (!sbuf.isdir && db.size() != sbuf.size) { + dberrprint(&db, "DB::size failed"); + err = true; + } + } else { + dberrprint(&db, "File::status failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld records were checked successfully\n", (long long)cnt); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcpolytest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolytest.cc new file mode 100644 index 0000000000..93dd3366fb --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcpolytest.cc @@ -0,0 +1,3091 @@ +/************************************************************************************************* + * The test cases of the polymorphic database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcpolydb.h> +#include <kcdbext.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dberrprint(kc::IndexDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static void dbmetaprint(kc::IndexDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t runmapred(int argc, char** argv); +static int32_t runindex(int argc, char** argv); +static int32_t runmisc(int argc, char** argv); +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, bool lv); +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t oflags, bool lv); +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, bool lv); +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, bool lv); +static int32_t procmapred(const char* path, int64_t rnum, bool rnd, bool ru, int32_t oflags, + bool lv, const char* tmpdir, int64_t dbnum, + int64_t clim, int64_t cbnum, int32_t opts); +static int32_t procindex(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + int32_t oflags, bool lv); +static int32_t procmisc(const char* path); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else if (!std::strcmp(argv[1], "mapred")) { + rv = runmapred(argc, argv); + } else if (!std::strcmp(argv[1], "index")) { + rv = runindex(argc, argv); + } else if (!std::strcmp(argv[1], "misc")) { + rv = runmisc(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the polymorphic database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-set|-get|-getw|-rem|-etc] [-tran]" + " [-oat|-oas|-onl|-otl|-onr] [-lv] path rnum\n", g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-oat|-oas|-onl|-otl|-onr] [-lv]" + " path rnum\n", g_progname); + eprintf(" %s wicked [-th num] [-it num] [-oat|-oas|-onl|-otl|-onr] [-lv]" + " path rnum\n", g_progname); + eprintf(" %s tran [-th num] [-it num] [-hard] [-oat|-oas|-onl|-otl|-onr] [-lv]" + " path rnum\n", g_progname); + eprintf(" %s mapred [-rnd] [-ru] [-oat|-oas|-onl|-otl|-onr] [-lv] [-tmp str]" + " [-dbnum num] [-clim num] [-cbnum num] [-xnl] [-xpm] [-xpr] [-xpf] [-xnc]" + " path rnum\n", g_progname); + eprintf(" %s index [-th num] [-rnd] [-set|-get|-rem|-etc]" + " [-oat|-oas|-onl|-otl|-onr] [-lv] path rnum\n", g_progname); + eprintf(" %s misc path\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print the error message of a database +static void dberrprint(kc::IndexDB* idb, int32_t line, const char* func) { + dberrprint(idb->reveal_inner_db(), line, func); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + if (db->status(&status)) { + std::map<std::string, std::string>::iterator it = status.begin(); + std::map<std::string, std::string>::iterator itend = status.end(); + while (it != itend) { + oprintf("%s: %s\n", it->first.c_str(), it->second.c_str()); + ++it; + } + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// print members of a database +static void dbmetaprint(kc::IndexDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + if (db->status(&status)) { + std::map<std::string, std::string>::iterator it = status.begin(); + std::map<std::string, std::string>::iterator itend = status.end(); + while (it != itend) { + oprintf("%s: %s\n", it->first.c_str(), it->second.c_str()); + ++it; + } + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + int32_t mode = 0; + bool tran = false; + int32_t oflags = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-set")) { + mode = 's'; + } else if (!std::strcmp(argv[i], "-get")) { + mode = 'g'; + } else if (!std::strcmp(argv[i], "-getw")) { + mode = 'w'; + } else if (!std::strcmp(argv[i], "-rem")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-etc")) { + mode = 'e'; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::PolyDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::PolyDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(path, rnum, thnum, rnd, mode, tran, oflags, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int32_t oflags = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::PolyDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::PolyDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(path, rnum, thnum, itnum, rnd, oflags, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t oflags = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::PolyDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::PolyDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(path, rnum, thnum, itnum, oflags, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool hard = false; + int32_t oflags = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-hard")) { + hard = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::PolyDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::PolyDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(path, rnum, thnum, itnum, hard, oflags, lv); + return rv; +} + + +// parse arguments of mapred command +static int32_t runmapred(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + bool rnd = false; + bool ru = false; + int32_t oflags = 0; + bool lv = false; + const char* tmpdir = ""; + int64_t dbnum = -1; + int64_t clim = -1; + int64_t cbnum = -1; + int32_t opts = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-ru")) { + ru = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::PolyDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::PolyDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else if (!std::strcmp(argv[i], "-tmp")) { + if (++i >= argc) usage(); + tmpdir = argv[i]; + } else if (!std::strcmp(argv[i], "-dbnum")) { + if (++i >= argc) usage(); + dbnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-clim")) { + if (++i >= argc) usage(); + clim = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-cbnum")) { + if (++i >= argc) usage(); + cbnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-xnl")) { + opts |= kc::MapReduce::XNOLOCK; + } else if (!std::strcmp(argv[i], "-xpm")) { + opts |= kc::MapReduce::XPARAMAP; + } else if (!std::strcmp(argv[i], "-xpr")) { + opts |= kc::MapReduce::XPARARED; + } else if (!std::strcmp(argv[i], "-xpf")) { + opts |= kc::MapReduce::XPARAFLS; + } else if (!std::strcmp(argv[i], "-xnc")) { + opts |= kc::MapReduce::XNOCOMP; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1) usage(); + int32_t rv = procmapred(path, rnum, rnd, ru, oflags, lv, tmpdir, dbnum, clim, cbnum, opts); + return rv; +} + + +// parse arguments of index command +static int32_t runindex(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + int32_t mode = 0; + int32_t oflags = 0; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-set")) { + mode = 's'; + } else if (!std::strcmp(argv[i], "-get")) { + mode = 'g'; + } else if (!std::strcmp(argv[i], "-rem")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-etc")) { + mode = 'e'; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::PolyDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::PolyDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::PolyDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::PolyDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::PolyDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + int32_t rv = procindex(path, rnum, thnum, rnd, mode, oflags, lv); + return rv; +} + + +// parse arguments of misc command +static int32_t runmisc(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else usage(); + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procmisc(path); + return rv; +} + + +// perform order command +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, bool lv) { + oprintf("<In-order Test>\n seed=%u path=%s rnum=%lld thnum=%d rnd=%d mode=%d tran=%d" + " oflags=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, rnd, mode, tran, oflags, lv); + bool err = false; + kc::PolyDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + uint32_t omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE | kc::PolyDB::OTRUNCATE; + if (mode == 'r') { + omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE; + } else if (mode == 'g' || mode == 'w') { + omode = kc::PolyDB::OREADER; + } + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (mode == 0 || mode == 's' || mode == 'e') { + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 's'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'g' || mode == 'e') { + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'g'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'w' || mode == 'e') { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'w'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::PolyDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t size) : size_(size) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + int64_t msiz_; + } syncprocessor(db.size()); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e' && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'r' || mode == 'e') { + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && mode_ != 'e') || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, mode, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, mode, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'r' || mode == 'e'); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t oflags, bool lv) { + oprintf("<Queue Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d rnd=%d" + " oflags=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, rnd, oflags, lv); + bool err = false; + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE; + if (itcnt == 1) omode |= kc::PolyDB::OTRUNCATE; + if (!db.open(path, omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::PolyDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::PolyDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, bool lv) { + oprintf("<Wicked Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d" + " oflags=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, oflags, lv); + bool err = false; + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE; + if (itcnt == 1) omode |= kc::PolyDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::PolyDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::NOIMPL) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + tran = false; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC && + db_->error() != kc::BasicDB::Error::NOIMPL) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (myrand(4) == 0) { + if (!cur->jump_back(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOIMPL && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump_back"); + err_ = true; + } + } else { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(5) > 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + if (myrand(rnum_ / 50 + 1) == 0) { + std::vector<std::string> keys; + std::string prefix(kbuf, ksiz > 0 ? ksiz - 1 : 0); + if (db_->match_prefix(prefix, &keys, myrand(10)) == -1) { + dberrprint(db_, __LINE__, "DB::match_prefix"); + err_ = true; + } + } + if (myrand(rnum_ / 50 + 1) == 0) { + std::vector<std::string> keys; + std::string regex(kbuf, ksiz > 0 ? ksiz - 1 : 0); + if (db_->match_regex(regex, &keys, myrand(10)) == -1 && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::match_regex"); + err_ = true; + } + } + if (myrand(rnum_ / 50 + 1) == 0) { + std::vector<std::string> keys; + std::string origin(kbuf, ksiz > 0 ? ksiz - 1 : 0); + if (db_->match_similar(origin, 3, myrand(2) == 0, &keys, myrand(10)) == -1) { + dberrprint(db_, __LINE__, "DB::match_similar"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0 && !db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::PolyDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, bool lv) { + oprintf("<Transaction Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d hard=%d" + " oflags=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, hard, oflags, lv); + bool err = false; + kc::PolyDB db; + kc::PolyDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE; + if (itcnt == 1) omode |= kc::PolyDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + std::string parapath = db.path() + "-para.kch"; + if (!paradb.open(parapath, omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::PolyDB* db, kc::PolyDB* paradb, int64_t rnum, + int32_t thnum, bool hard, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + hard_ = hard; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::PolyDB* db_; + kc::PolyDB* paradb_; + int64_t rnum_; + int32_t thnum_; + bool hard_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, hard, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, hard, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform mapred command +static int32_t procmapred(const char* path, int64_t rnum, bool rnd, bool ru, int32_t oflags, + bool lv, const char* tmpdir, int64_t dbnum, + int64_t clim, int64_t cbnum, int32_t opts) { + oprintf("<MapReduce Test>\n seed=%u path=%s rnum=%lld rnd=%d ru=%d oflags=%d lv=%d" + " tmp=%s dbnum=%lld clim=%lld cbnum=%lld opts=%d\n\n", + g_randseed, path, (long long)rnum, rnd, ru, oflags, lv, + tmpdir, (long long)dbnum, (long long)clim, (long long)cbnum, opts); + bool err = false; + kc::PolyDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + double stime = kc::time(); + uint32_t omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE | kc::PolyDB::OTRUNCATE; + if (ru) omode = kc::PolyDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class MapReduceImpl : public kc::MapReduce { + public: + MapReduceImpl() : mapcnt_(0), redcnt_(0), lock_() {} + bool map(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + mapcnt_.add(1); + return emit(vbuf, vsiz, kbuf, ksiz); + } + bool reduce(const char* kbuf, size_t ksiz, ValueIterator* iter) { + const char* vbuf; + size_t vsiz; + while ((vbuf = iter->next(&vsiz)) != NULL) { + redcnt_.add(1); + } + return true; + } + bool preprocess() { + oprintf("preprocessing:\n"); + if (!emit("pre", 3, "process", 7)) return false; + if (!emit("PROCESS", 7, "PRE", 3)) return false; + return true; + } + bool midprocess() { + oprintf("midprocessing:\n"); + if (!emit("mid", 3, "process", 7)) return false; + if (!emit("PROCESS", 7, "MID", 3)) return false; + return true; + } + bool postprocess() { + oprintf("postprocessing:\n"); + return true; + } + bool log(const char* name, const char* message) { + kc::ScopedMutex lock(&lock_); + oprintf("%s: %s", name, message); + int64_t musage = memusage(); + if (musage > 0) oprintf(": memory=%lld", (long long)(musage - g_memusage)); + oprintf("\n"); + return true; + } + int64_t mapcnt() { + return mapcnt_; + } + int64_t redcnt() { + return redcnt_; + } + private: + kc::AtomicInt64 mapcnt_; + kc::AtomicInt64 redcnt_; + kc::Mutex lock_; + }; + MapReduceImpl mr; + mr.tune_storage(dbnum, clim, cbnum); + int64_t pnum = rnum / 100; + if (pnum < 1) pnum = 1; + if (!ru) { + mr.log("misc", "setting records"); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + char vbuf[RECBUFSIZ]; + size_t vsiz = std::sprintf(vbuf, "%lld", (long long)(rnd ? myrand(pnum) + 1 : i % pnum)); + if (!db.append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, __LINE__, "DB::append"); + err = true; + } + } + } + if (!mr.execute(&db, tmpdir, opts)) { + dberrprint(&db, __LINE__, "MapReduce::execute"); + err = true; + } + if (!rnd && mr.mapcnt() != rnum) { + dberrprint(&db, __LINE__, "MapReduce::mapcnt"); + err = true; + } + if (!rnd && rnum % 100 == 0 && mr.redcnt() != rnum + 4) { + dberrprint(&db, __LINE__, "MapReduce::redcnt"); + err = true; + } + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform index command +static int32_t procindex(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + int32_t oflags, bool lv) { + oprintf("<Index Database Test>\n seed=%u path=%s rnum=%lld thnum=%d rnd=%d mode=%d" + " oflags=%d lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, rnd, mode, oflags, lv); + bool err = false; + kc::IndexDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + uint32_t omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE | kc::PolyDB::OTRUNCATE; + if (mode == 'r') { + omode = kc::PolyDB::OWRITER | kc::PolyDB::OCREATE; + } else if (mode == 'g' || mode == 'w') { + omode = kc::PolyDB::OREADER; + } + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (mode == 0 || mode == 's' || mode == 'e') { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::IndexDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::IndexDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, mode); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, mode); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 's'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("wicked testing:\n"); + stime = kc::time(); + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::IndexDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + switch (i % 5) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + if (rnd_ && myrand(range) == 0 && !db_->synchronize(false, NULL)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::IndexDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + }; + ThreadWicked threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, mode); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, mode); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'g' || mode == 'e') { + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::IndexDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::IndexDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + }; + ThreadGet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, mode); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, mode); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'g'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'r' || mode == 'e') { + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::IndexDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && mode_ != 'e') || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::IndexDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + }; + ThreadRemove threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, mode); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, mode); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'r' || mode == 'e'); + oprintf("time: %.3f\n", etime - stime); + } + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform misc command +static int32_t procmisc(const char* path) { + oprintf("<Miscellaneous Test>\n seed=%u path=%s\n\n", g_randseed, path); + bool err = false; + oprintf("opening the database:\n"); + kc::BasicDB* db = new kc::PolyDB; + db->open(path, kc::BasicDB::OWRITER | kc::BasicDB::OCREATE | kc::BasicDB::OTRUNCATE); + oprintf("setting records:\n"); + int64_t rnum = 10000; + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)i); + if (!db->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db, __LINE__, "DB::set"); + err = true; + } + } + oprintf("deploying cursors:\n"); + const int32_t cnum = 100; + kc::BasicDB::Cursor* curs[cnum]; + for (int32_t i = 0; i < cnum; i++) { + curs[i] = db->cursor(); + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)i + 1); + if (!curs[i]->jump(kbuf, ksiz)) { + dberrprint(db, __LINE__, "Cursor::jump"); + err = true; + } + } + oprintf("accessing records:\n"); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)i); + if (i % 3 == 0) { + size_t vsiz; + char* vbuf = db->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "DB::get"); + err = true; + } + } else { + if (!db->remove(kbuf, ksiz)) { + dberrprint(db, __LINE__, "DB::remove"); + err = true; + } + } + } + oprintf("updating records with cursors:\n"); + for (int32_t i = 0; i < cnum; i++) { + kc::BasicDB::Cursor* cur = curs[i]; + switch (i % 9) { + default: { + size_t ksiz; + char* kbuf = cur->get_key(&ksiz, i % 2 == 0); + if (kbuf) { + std::string value; + if (!db->get(std::string(kbuf, ksiz), &value)) { + dberrprint(db, __LINE__, "DB::get"); + err = true; + } + delete[] kbuf; + } else if (cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::get_key"); + err = true; + } + break; + } + case 1: { + size_t vsiz; + char* vbuf = cur->get_value(&vsiz, i % 2 == 0); + if (vbuf) { + delete[] vbuf; + } else if (cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::get_value"); + err = true; + } + break; + } + case 2: { + size_t ksiz; + const char* vbuf; + size_t vsiz; + char* kbuf = cur->get(&ksiz, &vbuf, &vsiz, i % 2 == 0); + if (kbuf) { + if (ksiz != vsiz || std::memcmp(kbuf, vbuf, ksiz)) { + dberrprint(db, __LINE__, "Cursor::get"); + err = true; + } + delete[] kbuf; + } else if (cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::get"); + err = true; + } + break; + } + case 3: { + std::string key, value; + if (cur->get(&key, &value, i % 2 == 0)) { + if (key != value) { + dberrprint(db, __LINE__, "Cursor::get"); + err = true; + } + } else if (cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::get"); + err = true; + } + break; + } + case 4: { + char vbuf[RECBUFSIZ]; + size_t vsiz = std::sprintf(vbuf, "kyoto:%d", i); + if (!cur->set_value(vbuf, vsiz) && cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::set_value"); + err = true; + } + break; + } + case 5: { + if (!cur->remove() && cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::remove"); + err = true; + } + break; + } + case 6: { + size_t ksiz; + const char* vbuf; + size_t vsiz; + char* kbuf = cur->seize(&ksiz, &vbuf, &vsiz); + if (kbuf) { + delete[] kbuf; + } else if (cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::seize"); + err = true; + } + break; + } + case 7: { + std::string key, value; + if (!cur->seize(&key, &value) && cur->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db, __LINE__, "Cursor::get"); + err = true; + } + break; + } + } + } + oprintf("bulk operations:\n"); + class VisitorBulk : public kc::DB::Visitor { + public: + VisitorBulk() : before_(0), after_(0) {} + int64_t before() { + return before_; + } + int64_t after() { + return after_; + } + private: + void visit_before() { + before_++; + } + void visit_after() { + after_++; + } + int64_t before_; + int64_t after_; + }; + std::vector<std::string> keys; + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)i); + keys.push_back(std::string(kbuf, ksiz)); + if (i % 10 == 0) { + VisitorBulk visitor; + if (db->accept_bulk(keys, &visitor, i % 3 > 0)) { + if (visitor.before() != 1 || visitor.after() != 1) { + dberrprint(db, __LINE__, "DB::accept_bulk"); + err = true; + } + } else { + dberrprint(db, __LINE__, "DB::accept_bulk"); + err = true; + } + keys.clear(); + } + } + kc::PolyDB* pdb = dynamic_cast<kc::PolyDB*>(db); + if (pdb) { + kc::BasicDB* idb = pdb->reveal_inner_db(); + if (idb) { + const std::type_info& info = typeid(*idb); + const char* ext = NULL; + if (info == typeid(kc::HashDB)) { + ext = "kch"; + } else if (info == typeid(kc::TreeDB)) { + ext = "kct"; + } else if (info == typeid(kc::DirDB)) { + ext = "kcd"; + } else if (info == typeid(kc::ForestDB)) { + ext = "kcf"; + } + if (ext) { + const std::string& dpath = idb->path() + kc::File::EXTCHR + "tmp" + + kc::File::EXTCHR + ext; + oprintf("copying the database file:\n"); + if (!idb->copy(dpath)) { + dberrprint(db, __LINE__, "DB::copy"); + err = true; + } + oprintf("merging the database files:\n"); + kc::PolyDB srcdb; + if (!srcdb.open(dpath, kc::PolyDB::OREADER)) { + dberrprint(&srcdb, __LINE__, "DB::open"); + err = true; + } + kc::BasicDB* bdb = &srcdb; + if (!pdb->merge(&bdb, 1, kc::PolyDB::MAPPEND)) { + dberrprint(db, __LINE__, "DB::merge"); + err = true; + } + if (!srcdb.close()) { + dberrprint(&srcdb, __LINE__, "DB::close"); + err = true; + } + kc::File::remove_recursively(dpath.c_str()); + } + } + } + oprintf("scanning in parallel:\n"); + class VisitorCount : public kc::DB::Visitor { + public: + explicit VisitorCount() : cnt_(0) {} + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_.add(1); + return NOP; + } + kc::AtomicInt64 cnt_; + } visitorcount; + class ProgressCheckerCount : public kc::BasicDB::ProgressChecker { + public: + explicit ProgressCheckerCount() : cnt_(0) {} + int64_t cnt() { + return cnt_; + } + private: + bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) { + cnt_.add(1); + return true; + } + kc::AtomicInt64 cnt_; + } checkercount; + class ThreadWicked : public kc::Thread { + public: + explicit ThreadWicked(kc::BasicDB* db, int64_t rnum) : + db_(db), rnum_(rnum), err_(false) {} + bool error() { + return err_; + } + private: + void run() { + for (int64_t i = 0; i < rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)i); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + } + } + kc::BasicDB* db_; + int64_t rnum_; + bool err_; + } threadwicked(pdb, rnum); + threadwicked.start(); + if (!db->scan_parallel(&visitorcount, 4, &checkercount)) { + dberrprint(pdb, __LINE__, "DB::scan_parallel"); + err = true; + } + threadwicked.join(); + if (threadwicked.error()) err = true; + if (visitorcount.cnt() != db->count()) { + dberrprint(pdb, __LINE__, "DB::scan_parallel"); + err = true; + } + if (checkercount.cnt() < db->count() + 2) { + dberrprint(pdb, __LINE__, "DB::scan_parallel ss"); + err = true; + } + oprintf("deleting the database object:\n"); + delete db; + oprintf("deleting the cursor objects:\n"); + for (int32_t i = 0; i < cnum; i++) { + delete curs[i]; + } + oprintf("re-opening the database object with a external database:\n"); + pdb = new kc::PolyDB; + pdb->set_internal_db(new kc::PolyDB); + if (!pdb->open(path, kc::PolyDB::OREADER)) { + dberrprint(pdb, __LINE__, "DB::open"); + err = true; + } + oprintf("deleting the database object:\n"); + delete pdb; + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcprotodb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcprotodb.cc new file mode 100644 index 0000000000..1151e91f0a --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcprotodb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Prototype database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcprotodb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcprotodb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcprotodb.h new file mode 100644 index 0000000000..5350ced154 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcprotodb.h @@ -0,0 +1,1358 @@ +/************************************************************************************************* + * Prototype database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCPROTODB_H // duplication check +#define _KCPROTODB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> +#include <kcplantdb.h> + +namespace kyotocabinet { // common namespace + + +/** + * Prototype implementation of database with STL. + * @param STRMAP a class compatible with the map class of STL. + * @param DBTYPE the database type number of the class. + * @note This class template is a template for concrete classes which wrap data structures + * compatible with std::map. Template instance classes can be inherited but overwriting methods + * is forbidden. The class ProtoHashDB is the instance using std::unordered_map. The class + * ProtoTreeDB is the instance using std::map. Before every database operation, it is necessary + * to call the BasicDB::open method in order to open a database file and connect the database + * object to it. To avoid data missing or corruption, it is important to close every database + * file by the BasicDB::close method when the database is no longer in use. It is forbidden for + * multible database objects in a process to open the same database at the same time. It is + * forbidden to share a database object with child processes. + */ +template <class STRMAP, uint8_t DBTYPE> +class ProtoDB : public BasicDB { + public: + class Cursor; + private: + struct TranLog; + class ScopedVisitor; + /** An alias of list of cursors. */ + typedef std::list<Cursor*> CursorList; + /** An alias of list of transaction logs. */ + typedef std::list<TranLog> TranLogList; + /** The size of the opaque buffer. */ + static const size_t OPAQUESIZ = 16; + /** The threshold of busy loop and sleep for locking. */ + static const uint32_t LOCKBUSYLOOP = 8192; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class ProtoDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(ProtoDB* db) : db_(db), it_(db->recs_.end()) { + _assert_(db); + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.push_back(this); + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + if (!db_) return; + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.remove(this); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(db_->omode_ & OWRITER)) { + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + if (it_ == db_->recs_.end()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + const std::string& key = it_->first; + const std::string& value = it_->second; + size_t vsiz; + const char* vbuf = visitor->visit_full(key.data(), key.size(), + value.data(), value.size(), &vsiz); + if (vbuf == Visitor::REMOVE) { + if (db_->tran_) { + TranLog log(key, value); + db_->trlogs_.push_back(log); + } + db_->size_ -= key.size() + value.size(); + if (db_->curs_.size() > 1) { + typename CursorList::const_iterator cit = db_->curs_.begin(); + typename CursorList::const_iterator citend = db_->curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur != this && cur->it_ == it_) ++cur->it_; + ++cit; + } + } + db_->recs_.erase(it_++); + } else if (vbuf == Visitor::NOP) { + if (step) ++it_; + } else { + if (db_->tran_) { + TranLog log(key, value); + db_->trlogs_.push_back(log); + } + db_->size_ -= value.size(); + db_->size_ += vsiz; + it_->second = std::string(vbuf, vsiz); + if (step) ++it_; + } + return true; + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + it_ = db_->recs_.begin(); + if (it_ == db_->recs_.end()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + std::string key(kbuf, ksiz); + search(key); + if (it_ == db_->recs_.end()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + return jump(key.data(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @return true on success, or false on failure. + */ + bool jump_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + it_ = db_->recs_.end(); + if (it_ == db_->recs_.begin()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + if (!iter_back()) { + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + return true; + } + /** + * Jump the cursor to a record for backward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + std::string key(kbuf, ksiz); + search(key); + if (it_ == db_->recs_.end()) { + if (it_ == db_->recs_.begin()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + if (!iter_back()) { + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + } else { + std::string key(kbuf, ksiz); + if (key < it_->first) { + if (it_ == db_->recs_.begin()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + it_ = db_->recs_.end(); + return false; + } + if (!iter_back()) { + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + it_ = db_->recs_.end(); + return false; + } + } + } + return true; + } + /** + * Jump the cursor to a record for backward scan. + * @note Equal to the original Cursor::jump_back method except that the parameter is + * std::string. + */ + bool jump_back(const std::string& key) { + _assert_(true); + return jump_back(key.data(), key.size()); + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (it_ == db_->recs_.end()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + ++it_; + if (it_ == db_->recs_.end()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Step the cursor to the previous record. + * @return true on success, or false on failure. + */ + bool step_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (it_ == db_->recs_.begin()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + it_ = db_->recs_.end(); + return false; + } + if (!iter_back()) { + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + it_ = db_->recs_.end(); + return false; + } + return true; + } + /** + * Get the database object. + * @return the database object. + */ + ProtoDB* db() { + _assert_(true); + return db_; + } + private: + /** + * Search for a record. + */ + void search(const std::string& key); + /** + * Place back the inner iterator. + * @return true on success, or false on failure. + */ + bool iter_back(); + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + ProtoDB* db_; + /** The inner iterator. */ + typename STRMAP::iterator it_; + }; + /** + * Default constructor. + */ + explicit ProtoDB() : + mlock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL), + omode_(0), recs_(), curs_(), path_(""), size_(0), opaque_(), + tran_(false), trlogs_(), trsize_(0) { + _assert_(true); + map_tune(); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~ProtoDB() { + _assert_(true); + if (omode_ != 0) close(); + if (!curs_.empty()) { + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->db_ = NULL; + ++cit; + } + } + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + if (writable) { + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + std::string key(kbuf, ksiz); + typename STRMAP::iterator it = recs_.find(key); + if (it == recs_.end()) { + size_t vsiz; + const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + if (tran_) { + TranLog log(key); + trlogs_.push_back(log); + } + size_ += ksiz + vsiz; + recs_[key] = std::string(vbuf, vsiz); + } + } else { + const std::string& value = it->second; + size_t vsiz; + const char* vbuf = visitor->visit_full(kbuf, ksiz, value.data(), value.size(), &vsiz); + if (vbuf == Visitor::REMOVE) { + if (tran_) { + TranLog log(key, value); + trlogs_.push_back(log); + } + size_ -= ksiz + value.size(); + if (!curs_.empty()) { + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->it_ == it) ++cur->it_; + ++cit; + } + } + recs_.erase(it); + } else if (vbuf != Visitor::NOP) { + if (tran_) { + TranLog log(key, value); + trlogs_.push_back(log); + } + size_ -= value.size(); + size_ += vsiz; + it->second = std::string(vbuf, vsiz); + } + } + } else { + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + std::string key(kbuf, ksiz); + const STRMAP& rrecs = recs_; + typename STRMAP::const_iterator it = rrecs.find(key); + if (it == rrecs.end()) { + size_t vsiz; + const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + } else { + const std::string& value = it->second; + size_t vsiz; + const char* vbuf = visitor->visit_full(kbuf, ksiz, value.data(), value.size(), &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + } + } + return true; + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + if (keys.empty()) return true; + std::vector<std::string>::const_iterator kit = keys.begin(); + std::vector<std::string>::const_iterator kitend = keys.end(); + while (kit != kitend) { + const std::string& key = *kit; + typename STRMAP::iterator it = recs_.find(key); + if (it == recs_.end()) { + size_t vsiz; + const char* vbuf = visitor->visit_empty(key.data(), key.size(), &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + if (tran_) { + TranLog log(key); + trlogs_.push_back(log); + } + size_ += key.size() + vsiz; + recs_[key] = std::string(vbuf, vsiz); + } + } else { + const std::string& value = it->second; + size_t vsiz; + const char* vbuf = visitor->visit_full(key.data(), key.size(), + value.data(), value.size(), &vsiz); + if (vbuf == Visitor::REMOVE) { + if (tran_) { + TranLog log(key, value); + trlogs_.push_back(log); + } + size_ -= key.size() + value.size(); + if (!curs_.empty()) { + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->it_ == it) ++cur->it_; + ++cit; + } + } + recs_.erase(it); + } else if (vbuf != Visitor::NOP) { + if (tran_) { + TranLog log(key, value); + trlogs_.push_back(log); + } + size_ -= value.size(); + size_ += vsiz; + it->second = std::string(vbuf, vsiz); + } + } + ++kit; + } + return true; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + int64_t allcnt = recs_.size(); + if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + typename STRMAP::iterator it = recs_.begin(); + typename STRMAP::iterator itend = recs_.end(); + int64_t curcnt = 0; + while (it != itend) { + const std::string& key = it->first; + const std::string& value = it->second; + size_t vsiz; + const char* vbuf = visitor->visit_full(key.data(), key.size(), + value.data(), value.size(), &vsiz); + if (vbuf == Visitor::REMOVE) { + size_ -= key.size() + value.size(); + recs_.erase(it++); + } else if (vbuf == Visitor::NOP) { + ++it; + } else { + size_ -= value.size(); + size_ += vsiz; + it->second = std::string(vbuf, vsiz); + ++it; + } + curcnt++; + if (checker && !checker->check("iterate", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + } + if (checker && !checker->check("iterate", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + trigger_meta(MetaTrigger::ITERATE, "iterate"); + return true; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (thnum < 1) thnum = 1; + if (thnum > (size_t)INT8MAX) thnum = INT8MAX; + ScopedVisitor svis(visitor); + int64_t allcnt = recs_.size(); + if (checker && !checker->check("scan_parallel", "beginning", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + class ThreadImpl : public Thread { + public: + explicit ThreadImpl() : + db_(NULL), visitor_(NULL), checker_(NULL), allcnt_(0), + itp_(NULL), itend_(), itmtx_(NULL), error_() {} + void init(ProtoDB* db, Visitor* visitor, ProgressChecker* checker, int64_t allcnt, + typename STRMAP::const_iterator* itp, typename STRMAP::const_iterator itend, + Mutex* itmtx) { + db_ = db; + visitor_ = visitor; + checker_ = checker; + allcnt_ = allcnt; + itp_ = itp; + itend_ = itend; + itmtx_ = itmtx; + } + const Error& error() { + return error_; + } + private: + void run() { + ProtoDB* db = db_; + Visitor* visitor = visitor_; + ProgressChecker* checker = checker_; + int64_t allcnt = allcnt_; + typename STRMAP::const_iterator* itp = itp_; + typename STRMAP::const_iterator itend = itend_; + Mutex* itmtx = itmtx_; + while (true) { + itmtx->lock(); + if (*itp == itend) { + itmtx->unlock(); + break; + } + const std::string& key = (*itp)->first; + const std::string& value = (*itp)->second; + ++(*itp); + itmtx->unlock(); + size_t vsiz; + visitor->visit_full(key.data(), key.size(), value.data(), value.size(), &vsiz); + if (checker && !checker->check("scan_parallel", "processing", -1, allcnt)) { + db->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + error_ = db->error(); + break; + } + } + } + ProtoDB* db_; + Visitor* visitor_; + ProgressChecker* checker_; + int64_t allcnt_; + typename STRMAP::const_iterator* itp_; + typename STRMAP::const_iterator itend_; + Mutex* itmtx_; + Error error_; + }; + bool err = false; + typename STRMAP::const_iterator it = recs_.begin(); + typename STRMAP::const_iterator itend = recs_.end(); + Mutex itmtx; + ThreadImpl* threads = new ThreadImpl[thnum]; + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->init(this, visitor, checker, allcnt, &it, itend, &itmtx); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->start(); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->join(); + if (thread->error() != Error::SUCCESS) { + *error_ = thread->error(); + err = true; + } + } + delete[] threads; + if (err) return false; + if (checker && !checker->check("scan_parallel", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + trigger_meta(MetaTrigger::ITERATE, "scan_parallel"); + return true; + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + return error_; + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + error_->set(code, message); + if (logger_) { + Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ? + Logger::ERROR : Logger::INFO; + if (kind & logkinds_) + report(file, line, func, kind, "%d: %s: %s", code, Error::codename(code), message); + } + } + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. BasicDB::OWRITER as a writer, BasicDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: BasicDB::OCREATE, + * which means it creates a new database if the file does not exist, BasicDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, BasicDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, BasicDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: BasicDB::ONOLOCK, which means it opens the database file without file locking, + * BasicDB::OTRYLOCK, which means locking is performed without blocking, BasicDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the BasicDB::close method when it is no + * longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", path.c_str()); + omode_ = mode; + path_.append(path); + std::memset(opaque_, 0, sizeof(opaque_)); + trigger_meta(MetaTrigger::OPEN, "open"); + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", path_.c_str()); + tran_ = false; + trlogs_.clear(); + recs_.clear(); + if (!curs_.empty()) { + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->it_ = recs_.end(); + ++cit; + } + } + path_.clear(); + omode_ = 0; + trigger_meta(MetaTrigger::CLOSE, "close"); + return true; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bool err = false; + if ((omode_ & OWRITER) && checker && + !checker->check("synchronize", "nothing to be synchronized", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (proc) { + if (checker && !checker->check("synchronize", "running the post processor", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!proc->process(path_, recs_.size(), size_)) { + set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed"); + err = true; + } + } + trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize"); + return !err; + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, writable); + bool err = false; + if (proc && !proc->process(path_, recs_.size(), size_)) { + set_error(_KCCODELINE_, Error::LOGIC, "processing failed"); + err = true; + } + trigger_meta(MetaTrigger::OCCUPY, "occupy"); + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + uint32_t wcnt = 0; + while (true) { + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!tran_) break; + mlock_.unlock(); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + tran_ = true; + trsize_ = size_; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); + mlock_.unlock(); + return true; + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (tran_) { + set_error(_KCCODELINE_, Error::LOGIC, "competition avoided"); + mlock_.unlock(); + return false; + } + tran_ = true; + trsize_ = size_; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try"); + mlock_.unlock(); + return true; + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!tran_) { + set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); + return false; + } + if (!commit) { + if (!curs_.empty()) { + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->it_ = recs_.end(); + ++cit; + } + } + const TranLogList& logs = trlogs_; + typename TranLogList::const_iterator lit = logs.end(); + typename TranLogList::const_iterator litbeg = logs.begin(); + while (lit != litbeg) { + --lit; + if (lit->full) { + recs_[lit->key] = lit->value; + } else { + recs_.erase(lit->key); + } + } + size_ = trsize_; + } + trlogs_.clear(); + tran_ = false; + trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction"); + return true; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + recs_.clear(); + if (!curs_.empty()) { + typename CursorList::const_iterator cit = curs_.begin(); + typename CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->it_ = recs_.end(); + ++cit; + } + } + std::memset(opaque_, 0, sizeof(opaque_)); + trigger_meta(MetaTrigger::CLEAR, "clear"); + return true; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return recs_.size(); + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return size_; + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return ""; + } + return path_; + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + (*strmap)["type"] = strprintf("%u", (unsigned)DBTYPE); + (*strmap)["realtype"] = strprintf("%u", (unsigned)DBTYPE); + (*strmap)["path"] = path_; + if (strmap->count("opaque") > 0) + (*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_)); + (*strmap)["count"] = strprintf("%lld", (long long)recs_.size()); + (*strmap)["size"] = strprintf("%lld", (long long)size_); + return true; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + ScopedRWLock lock(&mlock_, false); + if (!logger_) return; + logger_->log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + logger_ = logger; + logkinds_ = kinds; + return true; + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + /** + * Get the opaque data. + * @return the pointer to the opaque data region, whose size is 16 bytes. + */ + char* opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return opaque_; + } + /** + * Synchronize the opaque data. + * @return true on success, or false on failure. + */ + bool synchronize_opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + return true; + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, ...) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + va_list ap; + va_start(ap, format); + vstrprintf(&message, format, ap); + va_end(ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report a message for debugging with variable number of arguments. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ap used according to the format string. + */ + void report_valist(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, va_list ap) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + vstrprintf(&message, format, ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report the content of a binary buffer for debugging. + * @param file the file name of the epicenter. + * @param line the line number of the epicenter. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param name the name of the information. + * @param buf the binary buffer. + * @param size the size of the binary buffer + */ + void report_binary(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* name, const char* buf, size_t size) { + _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ); + if (!logger_) return; + char* hex = hexencode(buf, size); + report(file, line, func, kind, "%s=%s", name, hex); + delete[] hex; + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN for beginning + * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaTrigger::ABORTTRAN + * for aborting transaction, and MetaTrigger::MISC for miscellaneous operations. + * @param message the supplement message. + */ + void trigger_meta(MetaTrigger::Kind kind, const char* message) { + _assert_(message); + if (mtrigger_) mtrigger_->trigger(kind, message); + } + private: + /** + * Transaction log. + */ + struct TranLog { + bool full; ///< flag whether full + std::string key; ///< old key + std::string value; ///< old value + /** constructor for a full record */ + explicit TranLog(const std::string& pkey, const std::string& pvalue) : + full(true), key(pkey), value(pvalue) { + _assert_(true); + } + /** constructor for an empty record */ + explicit TranLog(const std::string& pkey) : full(false), key(pkey) { + _assert_(true); + } + }; + /** + * Scoped visitor. + */ + class ScopedVisitor { + public: + /** constructor */ + explicit ScopedVisitor(Visitor* visitor) : visitor_(visitor) { + _assert_(visitor); + visitor_->visit_before(); + } + /** destructor */ + ~ScopedVisitor() { + _assert_(true); + visitor_->visit_after(); + } + private: + Visitor* visitor_; ///< visitor + }; + /** + * Tune the internal map object. + */ + void map_tune(); + /** Dummy constructor to forbid the use. */ + ProtoDB(const ProtoDB&); + /** Dummy Operator to forbid the use. */ + ProtoDB& operator =(const ProtoDB&); + /** The method lock. */ + RWLock mlock_; + /** The last happened error. */ + TSD<Error> error_; + /** The internal logger. */ + Logger* logger_; + /** The kinds of logged messages. */ + uint32_t logkinds_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The open mode. */ + uint32_t omode_; + /** The map of records. */ + STRMAP recs_; + /** The cursor objects. */ + CursorList curs_; + /** The path of the database file. */ + std::string path_; + /** The total size of records. */ + int64_t size_; + /** The opaque data. */ + char opaque_[OPAQUESIZ]; + /** The flag whether in transaction. */ + bool tran_; + /** The transaction logs. */ + TranLogList trlogs_; + /** The old size before transaction. */ + size_t trsize_; +}; + + +/** + * Search for a record. + */ +template <class STRMAP, uint8_t DBTYPE> +inline void ProtoDB<STRMAP, DBTYPE>::Cursor::search(const std::string& key) { + _assert_(true); + it_ = db_->recs_.find(key); +} + + +/** + * Search for a record. + */ +template <> /** specialization for StringTreeMap */ +inline void ProtoDB<StringTreeMap, BasicDB::TYPEPTREE>::Cursor::search(const std::string& key) { + _assert_(true); + it_ = db_->recs_.lower_bound(key); +} + + +/** + * Place back the inner iterator. + */ +template <class STRMAP, uint8_t DBTYPE> +inline bool ProtoDB<STRMAP, DBTYPE>::Cursor::iter_back() { + _assert_(true); + return false; +} + + +/** + * Place back the inner iterator. + */ +template <> /** specialization for StringTreeMap */ +inline bool ProtoDB<StringTreeMap, BasicDB::TYPEPTREE>::Cursor::iter_back() { + _assert_(true); + --it_; + return true; +} + + +/** + * Tune the internal map object. + */ +template <class STRMAP, uint8_t DBTYPE> +inline void ProtoDB<STRMAP, DBTYPE>::map_tune() { + _assert_(true); +} + + +/** + * Tune the internal map object. + */ +template <> /** specialization for StringTreeMap */ +inline void ProtoDB<StringHashMap, BasicDB::TYPEPHASH>::map_tune() { + _assert_(true); + recs_.rehash(1048583LL); + recs_.max_load_factor(FLTMAX); +} + + +/** An alias of the prototype hash database. */ +typedef ProtoDB<StringHashMap, BasicDB::TYPEPHASH> ProtoHashDB; + + +/** An alias of the prototype tree database. */ +typedef ProtoDB<StringTreeMap, BasicDB::TYPEPTREE> ProtoTreeDB; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcprototest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcprototest.cc new file mode 100644 index 0000000000..d4b6c16883 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcprototest.cc @@ -0,0 +1,2138 @@ +/************************************************************************************************* + * The test cases of the prototype database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcprotodb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +template <class PROTODB> +static int32_t procorder(const char* tname, int64_t rnum, int32_t thnum, bool rnd, + bool etc, bool tran); +template <class PROTODB> +static int32_t procqueue(const char* tname, int64_t rnum, int32_t thnum, int32_t itnum, + bool rnd); +template <class PROTODB> +static int32_t procwicked(const char* tname, int64_t rnum, int32_t thnum, int32_t itnum); +template <class PROTODB> +static int32_t proctran(const char* tname, int64_t rnum, int32_t thnum, int32_t itnum); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the prototype database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-tree] [-th num] [-rnd] [-etc] [-tran] rnum\n", g_progname); + eprintf(" %s queue [-tree] [-th num] [-it num] [-rnd] rnum\n", g_progname); + eprintf(" %s wicked [-tree] [-th num] [-it num] rnum\n", g_progname); + eprintf(" %s tran [-tree] [-th num] [-it num] rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t rtype = kc::atoi(status["realtype"].c_str()); + if (rtype > 0 && rtype != type) + oprintf("real type: %s (%s) (realtype=0x%02X)\n", + kc::BasicDB::typecname(rtype), kc::BasicDB::typestring(rtype), rtype); + oprintf("path: %s\n", status["path"].c_str()); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t count = kc::atoi(status["count"].c_str()); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s)\n", size, sizestr.c_str()); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + bool tree = false; + int32_t thnum = 1; + bool rnd = false; + bool etc = false; + bool tran = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-tree")) { + tree = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-etc")) { + etc = true; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = 0; + if (tree) { + rv = procorder<kc::ProtoTreeDB>("Tree", rnum, thnum, rnd, etc, tran); + } else { + rv = procorder<kc::ProtoHashDB>("Hash", rnum, thnum, rnd, etc, tran); + } + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + bool tree = false; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-tree")) { + tree = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = 0; + if (tree) { + rv = procqueue<kc::ProtoTreeDB>("Tree", rnum, thnum, itnum, rnd); + } else { + rv = procqueue<kc::ProtoHashDB>("Hash", rnum, thnum, itnum, rnd); + } + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + bool tree = false; + int32_t thnum = 1; + int32_t itnum = 1; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-tree")) { + tree = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = 0; + if (tree) { + rv = procwicked<kc::ProtoTreeDB>("Tree", rnum, thnum, itnum); + } else { + rv = procwicked<kc::ProtoHashDB>("Hash", rnum, thnum, itnum); + } + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + bool tree = false; + int32_t thnum = 1; + int32_t itnum = 1; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-tree")) { + tree = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = 0; + if (tree) { + rv = proctran<kc::ProtoTreeDB>("Tree", rnum, thnum, itnum); + } else { + rv = proctran<kc::ProtoHashDB>("Hash", rnum, thnum, itnum); + } + return rv; +} + + +// perform order command +template <class PROTODB> +static int32_t procorder(const char* tname, int64_t rnum, int32_t thnum, bool rnd, + bool etc, bool tran) { + oprintf("<%s In-order Test>\n seed=%u rnum=%lld thnum=%d rnd=%d etc=%d tran=%d\n\n", + tname, g_randseed, (long long)rnum, thnum, rnd, etc, tran); + bool err = false; + PROTODB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + if (!db.open("-", kc::BasicDB::OWRITER | kc::BasicDB::OCREATE | kc::BasicDB::OTRUNCATE)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + typename PROTODB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size) : + rnum_(rnum), rnd_(rnd), size_(size) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + } syncprocessor(rnum, rnd, db.size()); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool etc, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + etc_ = etc; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && !etc_) || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool etc_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, etc, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, etc, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, true); + oprintf("time: %.3f\n", etime - stime); + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +template <class PROTODB> +static int32_t procqueue(const char* tname, int64_t rnum, int32_t thnum, int32_t itnum, + bool rnd) { + oprintf("<%s Queue Test>\n seed=%u rnum=%lld thnum=%d itnum=%d rnd=%d\n\n", + tname, g_randseed, (long long)rnum, thnum, itnum, rnd); + bool err = false; + PROTODB db; + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = PROTODB::OWRITER | PROTODB::OCREATE; + if (itcnt == 1) omode |= PROTODB::OTRUNCATE; + if (!db.open("-", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, PROTODB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + PROTODB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +template <class PROTODB> +static int32_t procwicked(const char* tname, int64_t rnum, int32_t thnum, int32_t itnum) { + oprintf("<%s Wicked Test>\n seed=%u rnum=%lld thnum=%d itnum=%d\n\n", + tname, g_randseed, (long long)rnum, thnum, itnum); + bool err = false; + PROTODB db; + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::BasicDB::OWRITER | kc::BasicDB::OCREATE; + if (itcnt == 1) omode |= kc::BasicDB::OTRUNCATE; + if (!db.open("-", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, PROTODB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (myrand(4) == 0) { + if (!cur->jump_back(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOIMPL && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump_back"); + err_ = true; + } + } else { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(5) > 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + PROTODB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform order command +template <class PROTODB> +static int32_t proctran(const char* tname, int64_t rnum, int32_t thnum, int32_t itnum) { + oprintf("<%s Transaction Test>\n seed=%u rnum=%lld thnum=%d itnum=%d\n\n", + tname, g_randseed, (long long)rnum, thnum, itnum); + bool err = false; + PROTODB db; + PROTODB paradb; + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::BasicDB::OWRITER | kc::BasicDB::OCREATE; + if (itcnt == 1) omode |= kc::BasicDB::OTRUNCATE; + if (!db.open("-", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + if (!paradb.open("para", omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, PROTODB* db, PROTODB* paradb, int64_t rnum, + int32_t thnum, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + PROTODB* db_; + PROTODB* paradb_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcregex.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcregex.cc new file mode 100644 index 0000000000..e3e56e8a6e --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcregex.cc @@ -0,0 +1,195 @@ +/************************************************************************************************* + * Regular expression + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcregex.h" +#include "myconf.h" + +#if _KC_PXREGEX +extern "C" { +#include <regex.h> +} +#else +#include <regex> +#endif + + +namespace kyotocabinet { // common namespace + + +/** + * Regex internal. + */ +struct RegexCore { +#if _KC_PXREGEX + ::regex_t rbuf; + bool alive; + bool nosub; +#else + std::regex* rbuf; +#endif +}; + + +/** + * Default constructor. + */ +Regex::Regex() : opq_(NULL) { +#if _KC_PXREGEX + _assert_(true); + RegexCore* core = new RegexCore; + core->alive = false; + core->nosub = false; + opq_ = (void*)core; +#else + _assert_(true); + RegexCore* core = new RegexCore; + core->rbuf = NULL; + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +Regex::~Regex() { +#if _KC_PXREGEX + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + if (core->alive) ::regfree(&core->rbuf); + delete core; +#else + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + delete core->rbuf; + delete core; +#endif +} + + +/** + * Compile a string of regular expression. + */ +bool Regex::compile(const std::string& regex, uint32_t opts) { +#if _KC_PXREGEX + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + if (core->alive) { + ::regfree(&core->rbuf); + core->alive = false; + } + int32_t cflags = REG_EXTENDED; + if (opts & IGNCASE) cflags |= REG_ICASE; + if ((opts & MATCHONLY) || regex.empty()) { + cflags |= REG_NOSUB; + core->nosub = true; + } + if (::regcomp(&core->rbuf, regex.c_str(), cflags) != 0) return false; + core->alive = true; + return true; +#else + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + if (core->rbuf) { + delete core->rbuf; + core->rbuf = NULL; + } + int32_t cflags = std::regex::ECMAScript; + if (opts & IGNCASE) cflags |= std::regex::icase; + if ((opts & MATCHONLY) || regex.empty()) cflags |= std::regex::nosubs; + try { + core->rbuf = new std::regex(regex, (std::regex::flag_type)cflags); + } catch (...) { + core->rbuf = NULL; + return false; + } + return true; +#endif +} + + +/** + * Check whether a string matches the regular expression. + */ +bool Regex::match(const std::string& str) { +#if _KC_PXREGEX + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + if (!core->alive) return false; + if (core->nosub) return ::regexec(&core->rbuf, str.c_str(), 0, NULL, 0) == 0; + ::regmatch_t subs[1]; + return ::regexec(&core->rbuf, str.c_str(), 1, subs, 0) == 0; +#else + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + if (!core->rbuf) return false; + std::smatch res; + return std::regex_search(str, res, *core->rbuf); +#endif +} + + +/** + * Check whether a string matches the regular expression. + */ +std::string Regex::replace(const std::string& str, const std::string& alt) { +#if _KC_PXREGEX + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + if (!core->alive || core->nosub) return str; + regmatch_t subs[256]; + if (::regexec(&core->rbuf, str.c_str(), sizeof(subs) / sizeof(*subs), subs, 0) != 0) + return str; + const char* sp = str.c_str(); + std::string xstr; + bool first = true; + while (sp[0] != '\0' && ::regexec(&core->rbuf, sp, 10, subs, first ? 0 : REG_NOTBOL) == 0) { + first = false; + if (subs[0].rm_so == -1) break; + xstr.append(sp, subs[0].rm_so); + for (const char* rp = alt.c_str(); *rp != '\0'; rp++) { + if (*rp == '$') { + if (rp[1] >= '0' && rp[1] <= '9') { + int32_t num = rp[1] - '0'; + if (subs[num].rm_so != -1 && subs[num].rm_eo != -1) + xstr.append(sp + subs[num].rm_so, subs[num].rm_eo - subs[num].rm_so); + ++rp; + } else if (rp[1] == '&') { + xstr.append(sp + subs[0].rm_so, subs[0].rm_eo - subs[0].rm_so); + ++rp; + } else if (rp[1] != '\0') { + xstr.append(++rp, 1); + } + } else { + xstr.append(rp, 1); + } + } + sp += subs[0].rm_eo; + if (subs[0].rm_eo < 1) break; + } + xstr.append(sp); + return xstr; +#else + _assert_(true); + RegexCore* core = (RegexCore*)opq_; + if (!core->rbuf) return str; + return std::regex_replace(str, *core->rbuf, alt); +#endif +} + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcregex.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcregex.h new file mode 100644 index 0000000000..5c5f2fb58b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcregex.h @@ -0,0 +1,113 @@ +/************************************************************************************************* + * Regular expression + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCREGEX_H // duplication check +#define _KCREGEX_H + +#include <kccommon.h> +#include <kcutil.h> + +namespace kyotocabinet { // common namespace + + +/** + * Regular expression. + */ +class Regex { + public: + /** + * Options. + */ + enum Option { + IGNCASE = 1 << 0, ///< case-insensitive + MATCHONLY = 1 << 1, ///< matching only + }; + /** + * Default constructor. + */ + explicit Regex(); + /** + * Destructor. + */ + ~Regex(); + /** + * Compile a string of regular expression. + * @param regex the string of regular expression. + * @param opts the optional features by bitwise-or: Regex::IGNCASE for case-insensitive + * matching, Regex::MATCHONLY for matching only usage. + */ + bool compile(const std::string& regex, uint32_t opts = 0); + /** + * Check whether a string matches the regular expression. + * @param str the string. + * @return true if the string matches, or false if not. + */ + bool match(const std::string& str); + /** + * Check whether a string matches the regular expression. + * @param str the string. + * @param alt the alternative string with which each substring is replaced. Each "$" in the + * string escapes the following character. Special escapes "$1" through "$9" refer to partial + * substrings corresponding to sub-expressions in the regular expression. "$0" and "$&" refer + * to the whole matching substring. + * @return the result string. + */ + std::string replace(const std::string& str, const std::string& alt); + /** + * Check whether a string matches a regular expression. + * @param str the string. + * @param pattern the matching pattern. + * @param opts the optional features by bitwise-or: Regex::IGNCASE for case-insensitive + * matching, Regex::MATCHONLY for matching only usage. + * @return true if the string matches, or false if not. + */ + static bool match(const std::string& str, const std::string& pattern, uint32_t opts = 0) { + Regex regex; + if (!regex.compile(pattern, opts)) return false; + return regex.match(str); + } + /** + * Check whether a string matches the regular expression. + * @param str the string. + * @param pattern the matching pattern. + * @param alt the alternative string with which each substring is replaced. Each "$" in the + * string escapes the following character. Special escapes "$1" through "$9" refer to partial + * substrings corresponding to sub-expressions in the regular expression. "$0" and "$&" refer + * to the whole matching substring. + * @param opts the optional features by bitwise-or: Regex::IGNCASE for case-insensitive + * matching, Regex::MATCHONLY for matching only usage. + * @return the result string. + */ + static std::string replace(const std::string& str, const std::string& pattern, + const std::string& alt, uint32_t opts = 0) { + Regex regex; + if (!regex.compile(pattern, opts)) return str; + return regex.replace(str, alt); + } + private: + /** Dummy constructor to forbid the use. */ + Regex(const Regex&); + /** Dummy Operator to forbid the use. */ + Regex& operator =(const Regex&); + /** Opaque pointer. */ + void* opq_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcstashdb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcstashdb.cc new file mode 100644 index 0000000000..f2a765b370 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcstashdb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Stash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcstashdb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcstashdb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcstashdb.h new file mode 100644 index 0000000000..2ce5599e7c --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcstashdb.h @@ -0,0 +1,1482 @@ +/************************************************************************************************* + * Stash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCSTASHDB_H // duplication check +#define _KCSTASHDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> +#include <kcplantdb.h> + +namespace kyotocabinet { // common namespace + + +/** + * Economical on-memory hash database. + * @note This class is a concrete class to operate a hash database on memory. This class can be + * inherited but overwriting methods is forbidden. Before every database operation, it is + * necessary to call the StashDB::open method in order to open a database file and connect the + * database object to it. To avoid data missing or corruption, it is important to close every + * database file by the StashDB::close method when the database is no longer in use. It is + * forbidden for multible database objects in a process to open the same database at the same + * time. It is forbidden to share a database object with child processes. + */ +class StashDB : public BasicDB { + public: + class Cursor; + private: + struct Record; + struct TranLog; + class Repeater; + class Setter; + class Remover; + class ScopedVisitor; + /** An alias of list of cursors. */ + typedef std::list<Cursor*> CursorList; + /** An alias of list of transaction logs. */ + typedef std::list<TranLog> TranLogList; + /** The number of slots of the record lock. */ + static const int32_t RLOCKSLOT = 1024; + /** The default bucket number. */ + static const size_t DEFBNUM = 1048583LL; + /** The size of the opaque buffer. */ + static const size_t OPAQUESIZ = 16; + /** The threshold of busy loop and sleep for locking. */ + static const uint32_t LOCKBUSYLOOP = 8192; + /** The mininum number of buckets to use mmap. */ + static const size_t MAPZMAPBNUM = 32768; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class StashDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(StashDB* db) : db_(db), bidx_(-1), rbuf_(NULL) { + _assert_(db); + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.push_back(this); + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + if (!db_) return; + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.remove(this); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(db_->omode_ & OWRITER)) { + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + if (bidx_ < 0) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + Record rec(rbuf_); + size_t vsiz; + const char* vbuf = visitor->visit_full(rec.kbuf_, rec.ksiz_, rec.vbuf_, rec.vsiz_, &vsiz); + if (vbuf == Visitor::REMOVE) { + Repeater repeater(Visitor::REMOVE, 0); + db_->accept_impl(rec.kbuf_, rec.ksiz_, &repeater, bidx_); + } else if (vbuf == Visitor::NOP) { + if (step) step_impl(); + } else { + Repeater repeater(vbuf, vsiz); + db_->accept_impl(rec.kbuf_, rec.ksiz_, &repeater, bidx_); + if (step && rbuf_) step_impl(); + } + return true; + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bidx_ = 0; + rbuf_ = NULL; + while (bidx_ < (int64_t)db_->bnum_) { + if (db_->buckets_[bidx_]) { + rbuf_ = db_->buckets_[bidx_]; + return true; + } + bidx_++; + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + bidx_ = -1; + return false; + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bidx_ = -1; + rbuf_ = NULL; + size_t bidx = db_->hash_record(kbuf, ksiz) % db_->bnum_; + char* rbuf = db_->buckets_[bidx]; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) { + bidx_ = bidx; + rbuf_ = rbuf; + return true; + } + rbuf = rec.child_; + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + return jump(key.c_str(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const std::string& key) { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (bidx_ < 0) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + bool err = false; + if (!step_impl()) err = true; + return !err; + } + /** + * Step the cursor to the previous record. + * @note This is a dummy implementation for compatibility. + */ + bool step_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Get the database object. + * @return the database object. + */ + StashDB* db() { + _assert_(true); + return db_; + } + private: + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step_impl() { + _assert_(true); + Record rec(rbuf_); + rbuf_ = rec.child_; + if (!rbuf_) { + while (++bidx_ < (int64_t)db_->bnum_) { + if (db_->buckets_[bidx_]) { + rbuf_ = db_->buckets_[bidx_]; + return true; + } + } + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + bidx_ = -1; + return false; + } + return true; + } + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + StashDB* db_; + /** The index of the current bucket. */ + int64_t bidx_; + /** The buffer of the current record. */ + char* rbuf_; + }; + /** + * Default constructor. + */ + explicit StashDB() : + mlock_(), rlock_(RLOCKSLOT), flock_(), error_(), + logger_(NULL), logkinds_(0), mtrigger_(NULL), + omode_(0), curs_(), path_(""), bnum_(DEFBNUM), opaque_(), + count_(0), size_(0), buckets_(NULL), + tran_(false), trlogs_(), trcount_(0), trsize_(0) { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + ~StashDB() { + _assert_(true); + if (omode_ != 0) close(); + if (!curs_.empty()) { + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->db_ = NULL; + ++cit; + } + } + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operation for each record is performed atomically and other threads accessing the + * same record are blocked. To avoid deadlock, any explicit database operation must not be + * performed in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + size_t lidx = bidx % RLOCKSLOT; + if (writable) { + rlock_.lock_writer(lidx); + } else { + rlock_.lock_reader(lidx); + } + accept_impl(kbuf, ksiz, visitor, bidx); + rlock_.unlock(lidx); + return true; + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note The operations for specified records are performed atomically and other threads + * accessing the same records are blocked. To avoid deadlock, any explicit database operation + * must not be performed in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + size_t knum = keys.size(); + if (knum < 1) return true; + struct RecordKey { + const char* kbuf; + size_t ksiz; + size_t bidx; + }; + RecordKey* rkeys = new RecordKey[knum]; + std::set<size_t> lidxs; + for (size_t i = 0; i < knum; i++) { + const std::string& key = keys[i]; + RecordKey* rkey = rkeys + i; + rkey->kbuf = key.data(); + rkey->ksiz = key.size(); + rkey->bidx = hash_record(rkey->kbuf, rkey->ksiz) % bnum_; + lidxs.insert(rkey->bidx % RLOCKSLOT); + } + std::set<size_t>::iterator lit = lidxs.begin(); + std::set<size_t>::iterator litend = lidxs.end(); + while (lit != litend) { + if (writable) { + rlock_.lock_writer(*lit); + } else { + rlock_.lock_reader(*lit); + } + ++lit; + } + for (size_t i = 0; i < knum; i++) { + RecordKey* rkey = rkeys + i; + accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->bidx); + } + lit = lidxs.begin(); + litend = lidxs.end(); + while (lit != litend) { + rlock_.unlock(*lit); + ++lit; + } + delete[] rkeys; + return true; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The whole iteration is performed atomically and other threads are blocked. To avoid + * deadlock, any explicit database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + int64_t allcnt = count_; + if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + int64_t curcnt = 0; + for (size_t i = 0; i < bnum_; i++) { + char* rbuf = buckets_[i]; + while (rbuf) { + curcnt++; + Record rec(rbuf); + rbuf = rec.child_; + size_t vsiz; + const char* vbuf = visitor->visit_full(rec.kbuf_, rec.ksiz_, + rec.vbuf_, rec.vsiz_, &vsiz); + if (vbuf == Visitor::REMOVE) { + Repeater repeater(Visitor::REMOVE, 0); + accept_impl(rec.kbuf_, rec.ksiz_, &repeater, i); + } else if (vbuf != Visitor::NOP) { + Repeater repeater(vbuf, vsiz); + accept_impl(rec.kbuf_, rec.ksiz_, &repeater, i); + } + if (checker && !checker->check("iterate", "processing", curcnt, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + } + } + if (checker && !checker->check("iterate", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + trigger_meta(MetaTrigger::ITERATE, "iterate"); + return true; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (thnum < 1) thnum = 1; + if (thnum > (size_t)INT8MAX) thnum = INT8MAX; + if (thnum > bnum_) thnum = bnum_; + ScopedVisitor svis(visitor); + int64_t allcnt = count_; + if (checker && !checker->check("scan_parallel", "beginning", 0, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + class ThreadImpl : public Thread { + public: + explicit ThreadImpl() : + db_(NULL), visitor_(NULL), checker_(NULL), allcnt_(0), + begidx_(0), endidx_(0), error_() {} + void init(StashDB* db, Visitor* visitor, ProgressChecker* checker, int64_t allcnt, + size_t begidx, size_t endidx) { + db_ = db; + visitor_ = visitor; + checker_ = checker; + allcnt_ = allcnt; + begidx_ = begidx; + endidx_ = endidx; + } + const Error& error() { + return error_; + } + private: + void run() { + StashDB* db = db_; + Visitor* visitor = visitor_; + ProgressChecker* checker = checker_; + int64_t allcnt = allcnt_; + size_t endidx = endidx_; + char** buckets = db->buckets_; + for (size_t i = begidx_; i < endidx; i++) { + char* rbuf = buckets[i]; + while (rbuf) { + Record rec(rbuf); + rbuf = rec.child_; + size_t vsiz; + visitor->visit_full(rec.kbuf_, rec.ksiz_, rec.vbuf_, rec.vsiz_, &vsiz); + if (checker && !checker->check("scan_parallel", "processing", -1, allcnt)) { + db->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + error_ = db->error(); + break; + } + } + } + } + StashDB* db_; + Visitor* visitor_; + ProgressChecker* checker_; + int64_t allcnt_; + size_t begidx_; + size_t endidx_; + Error error_; + }; + bool err = false; + rlock_.lock_reader_all(); + ThreadImpl* threads = new ThreadImpl[thnum]; + double range = (double)bnum_ / thnum; + for (size_t i = 0; i < thnum; i++) { + size_t cidx = i * range; + size_t nidx = (i + 1) * range; + if (i < 1) cidx = 0; + if (i >= thnum - 1) nidx = bnum_; + ThreadImpl* thread = threads + i; + thread->init(this, visitor, checker, allcnt, cidx, nidx); + thread->start(); + } + for (size_t i = 0; i < thnum; i++) { + ThreadImpl* thread = threads + i; + thread->join(); + if (thread->error() != Error::SUCCESS) { + *error_ = thread->error(); + err = true; + } + } + delete[] threads; + rlock_.unlock_all(); + if (err) return false; + if (checker && !checker->check("scan_parallel", "ending", -1, allcnt)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + trigger_meta(MetaTrigger::ITERATE, "scan_parallel"); + return true; + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + return error_; + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + error_->set(code, message); + if (logger_) { + Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ? + Logger::ERROR : Logger::INFO; + if (kind & logkinds_) + report(file, line, func, kind, "%d: %s: %s", code, Error::codename(code), message); + } + } + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. StashDB::OWRITER as a writer, StashDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: StashDB::OCREATE, + * which means it creates a new database if the file does not exist, StashDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, StashDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, StashDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: StashDB::ONOLOCK, which means it opens the database file without file locking, + * StashDB::OTRYLOCK, which means locking is performed without blocking, StashDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the StashDB::close method when it is no + * longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", path.c_str()); + omode_ = mode; + path_.append(path); + if (bnum_ >= MAPZMAPBNUM) { + buckets_ = (char**)mapalloc(sizeof(*buckets_) * bnum_); + } else { + buckets_ = new char*[bnum_]; + for (size_t i = 0; i < bnum_; i++) { + buckets_[i] = NULL; + } + } + std::memset(opaque_, 0, sizeof(opaque_)); + trigger_meta(MetaTrigger::OPEN, "open"); + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", path_.c_str()); + tran_ = false; + trlogs_.clear(); + for (size_t i = 0; i < bnum_; i++) { + char* rbuf = buckets_[i]; + while (rbuf) { + Record rec(rbuf); + char* child = rec.child_; + delete[] rbuf; + rbuf = child; + } + } + if (bnum_ >= MAPZMAPBNUM) { + mapfree(buckets_); + } else { + delete[] buckets_; + } + path_.clear(); + omode_ = 0; + trigger_meta(MetaTrigger::CLOSE, "close"); + return true; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bool err = false; + if ((omode_ & OWRITER) && checker && + !checker->check("synchronize", "nothing to be synchronized", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (proc) { + if (checker && !checker->check("synchronize", "running the post processor", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!proc->process(path_, count_, size_impl())) { + set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed"); + err = true; + } + } + trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize"); + return !err; + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, writable); + bool err = false; + if (proc && !proc->process(path_, count_, size_impl())) { + set_error(_KCCODELINE_, Error::LOGIC, "processing failed"); + err = true; + } + trigger_meta(MetaTrigger::OCCUPY, "occupy"); + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + uint32_t wcnt = 0; + while (true) { + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (!tran_) break; + mlock_.unlock(); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + tran_ = true; + trcount_ = count_; + trsize_ = size_; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); + mlock_.unlock(); + return true; + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + mlock_.lock_writer(); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + mlock_.unlock(); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + mlock_.unlock(); + return false; + } + if (tran_) { + set_error(_KCCODELINE_, Error::LOGIC, "competition avoided"); + mlock_.unlock(); + return false; + } + tran_ = true; + trcount_ = count_; + trsize_ = size_; + trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try"); + mlock_.unlock(); + return true; + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!tran_) { + set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); + return false; + } + if (!commit) { + disable_cursors(); + apply_trlogs(); + count_ = trcount_; + size_ = trsize_; + } + trlogs_.clear(); + tran_ = false; + trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction"); + return true; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + disable_cursors(); + if (count_ > 0) { + for (size_t i = 0; i < bnum_; i++) { + char* rbuf = buckets_[i]; + while (rbuf) { + Record rec(rbuf); + char* child = rec.child_; + delete[] rbuf; + rbuf = child; + } + buckets_[i] = NULL; + } + count_ = 0; + size_ = 0; + } + std::memset(opaque_, 0, sizeof(opaque_)); + trigger_meta(MetaTrigger::CLEAR, "clear"); + return true; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return count_; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return size_impl(); + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return ""; + } + return path_; + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + (*strmap)["type"] = strprintf("%u", (unsigned)TYPESTASH); + (*strmap)["realtype"] = strprintf("%u", (unsigned)TYPESTASH); + (*strmap)["path"] = path_; + if (strmap->count("opaque") > 0) + (*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_)); + if (strmap->count("bnum_used") > 0) { + int64_t cnt = 0; + for (size_t i = 0; i < bnum_; i++) { + if (buckets_[i]) cnt++; + } + (*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt); + } + (*strmap)["count"] = strprintf("%lld", (long long)count_); + (*strmap)["size"] = strprintf("%lld", (long long)size_impl()); + return true; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + ScopedRWLock lock(&mlock_, false); + if (!logger_) return; + logger_->log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + logger_ = logger; + logkinds_ = kinds; + return true; + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + /** + * Set the number of buckets of the hash table. + * @param bnum the number of buckets of the hash table. + * @return true on success, or false on failure. + */ + bool tune_buckets(int64_t bnum) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + bnum_ = bnum >= 0 ? bnum : DEFBNUM; + if (bnum_ > (size_t)INT16MAX) bnum_ = nearbyprime(bnum_); + return true; + } + /** + * Get the opaque data. + * @return the pointer to the opaque data region, whose size is 16 bytes. + */ + char* opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return NULL; + } + return opaque_; + } + /** + * Synchronize the opaque data. + * @return true on success, or false on failure. + */ + bool synchronize_opaque() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!(omode_ & OWRITER)) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + return true; + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, ...) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + va_list ap; + va_start(ap, format); + vstrprintf(&message, format, ap); + va_end(ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report a message for debugging with variable number of arguments. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ap used according to the format string. + */ + void report_valist(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, va_list ap) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + vstrprintf(&message, format, ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report the content of a binary buffer for debugging. + * @param file the file name of the epicenter. + * @param line the line number of the epicenter. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param name the name of the information. + * @param buf the binary buffer. + * @param size the size of the binary buffer + */ + void report_binary(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* name, const char* buf, size_t size) { + _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ); + if (!logger_) return; + char* hex = hexencode(buf, size); + report(file, line, func, kind, "%s=%s", name, hex); + delete[] hex; + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN for beginning + * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaTrigger::ABORTTRAN + * for aborting transaction, and MetaTrigger::MISC for miscellaneous operations. + * @param message the supplement message. + */ + void trigger_meta(MetaTrigger::Kind kind, const char* message) { + _assert_(message); + if (mtrigger_) mtrigger_->trigger(kind, message); + } + private: + /** + * Record data. + */ + struct Record { + /** constructor */ + Record(char* child, const char* kbuf, uint64_t ksiz, const char* vbuf, uint64_t vsiz) : + child_(child), kbuf_(kbuf), ksiz_(ksiz), vbuf_(vbuf), vsiz_(vsiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); + } + /** constructor */ + Record(const char* rbuf) : + child_(NULL), kbuf_(NULL), ksiz_(0), vbuf_(NULL), vsiz_(0) { + _assert_(rbuf); + deserialize(rbuf); + } + /** overwrite the buffer */ + void overwrite(char* rbuf, const char* vbuf, size_t vsiz) { + _assert_(rbuf && vbuf && vsiz <= MEMMAXSIZ); + char* wp = rbuf + sizeof(child_) + sizevarnum(ksiz_) + ksiz_; + wp += writevarnum(wp, vsiz); + std::memcpy(wp, vbuf, vsiz); + } + /** serialize data into a buffer */ + char* serialize() { + _assert_(true); + uint64_t rsiz = sizeof(child_) + sizevarnum(ksiz_) + ksiz_ + sizevarnum(vsiz_) + vsiz_; + char* rbuf = new char[rsiz]; + char* wp = rbuf; + *(char**)wp = child_; + wp += sizeof(child_); + wp += writevarnum(wp, ksiz_); + std::memcpy(wp, kbuf_, ksiz_); + wp += ksiz_; + wp += writevarnum(wp, vsiz_); + std::memcpy(wp, vbuf_, vsiz_); + return rbuf; + } + /** deserialize a buffer into object */ + void deserialize(const char* rbuf) { + _assert_(rbuf); + const char* rp = rbuf; + child_ = *(char**)rp; + rp += sizeof(child_); + rp += readvarnum(rp, sizeof(ksiz_), &ksiz_); + kbuf_ = rp; + rp += ksiz_; + rp += readvarnum(rp, sizeof(vsiz_), &vsiz_); + vbuf_ = rp; + } + /** print debug info */ + void print() { + std::cout << "child:" << (void*)child_ << std::endl; + std::cout << "key:" << std::string(kbuf_, ksiz_) << std::endl; + std::cout << "value:" << std::string(vbuf_, vsiz_) << std::endl; + std::cout << "ksiz:" << ksiz_ << std::endl; + std::cout << "vsiz:" << vsiz_ << std::endl; + } + char* child_; ///< region of the child + const char* kbuf_; ///< region of the key + uint64_t ksiz_; ///< size of the key + const char* vbuf_; ///< region of the value + uint64_t vsiz_; ///< size of the key + }; + /** + * Transaction log. + */ + struct TranLog { + bool full; ///< flag whether full + std::string key; ///< old key + std::string value; ///< old value + /** constructor for a full record */ + explicit TranLog(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) : + full(true), key(kbuf, ksiz), value(vbuf, vsiz) { + _assert_(true); + } + /** constructor for an empty record */ + explicit TranLog(const char* kbuf, size_t ksiz) : full(false), key(kbuf, ksiz) { + _assert_(true); + } + }; + /** + * Repeating visitor. + */ + class Repeater : public Visitor { + public: + /** constructor */ + explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsiz) {} + private: + /** process a full record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; ///< region of the value + size_t vsiz_; ///< size of the value + }; + /** + * Setting visitor. + */ + class Setter : public Visitor { + public: + /** constructor */ + explicit Setter(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsiz) {} + private: + /** process a full record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + *sp = vsiz_; + return vbuf_; + } + /** process an empty record */ + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && sp); + *sp = vsiz_; + return vbuf_; + } + const char* vbuf_; ///< region of the value + size_t vsiz_; ///< size of the value + }; + /** + * Removing visitor. + */ + class Remover : public Visitor { + private: + /** visit a record */ + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp); + return REMOVE; + } + }; + /** + * Scoped visitor. + */ + class ScopedVisitor { + public: + /** constructor */ + explicit ScopedVisitor(Visitor* visitor) : visitor_(visitor) { + _assert_(visitor); + visitor_->visit_before(); + } + /** destructor */ + ~ScopedVisitor() { + _assert_(true); + visitor_->visit_after(); + } + private: + Visitor* visitor_; ///< visitor + }; + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param bidx the bucket index. + */ + void accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, size_t bidx) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + char* rbuf = buckets_[bidx]; + char** entp = buckets_ + bidx; + while (rbuf) { + Record rec(rbuf); + if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) { + size_t vsiz; + const char* vbuf = visitor->visit_full(rec.kbuf_, rec.ksiz_, + rec.vbuf_, rec.vsiz_, &vsiz); + if (vbuf == Visitor::REMOVE) { + if (tran_) { + ScopedMutex lock(&flock_); + TranLog log(rec.kbuf_, rec.ksiz_, rec.vbuf_, rec.vsiz_); + trlogs_.push_back(log); + } + count_ -= 1; + size_ -= rec.ksiz_ + rec.vsiz_; + escape_cursors(rbuf); + *entp = rec.child_; + delete[] rbuf; + } else if (vbuf != Visitor::NOP) { + if (tran_) { + ScopedMutex lock(&flock_); + TranLog log(rec.kbuf_, rec.ksiz_, rec.vbuf_, rec.vsiz_); + trlogs_.push_back(log); + } + int32_t oh = (int32_t)sizevarnum(vsiz) - (int32_t)sizevarnum(rec.vsiz_); + int64_t diff = (int64_t)rec.vsiz_ - (int64_t)(vsiz + oh); + size_ += (int64_t)vsiz - (int64_t)rec.vsiz_; + if (diff >= 0) { + rec.overwrite(rbuf, vbuf, vsiz); + } else { + Record nrec(rec.child_, kbuf, ksiz, vbuf, vsiz); + char* nbuf = nrec.serialize(); + adjust_cursors(rbuf, nbuf); + *entp = nbuf; + delete[] rbuf; + } + } + return; + } + entp = (char**)rbuf; + rbuf = rec.child_; + } + size_t vsiz; + const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); + if (vbuf != Visitor::REMOVE && vbuf != Visitor::NOP) { + if (tran_) { + ScopedMutex lock(&flock_); + TranLog log(kbuf, ksiz); + trlogs_.push_back(log); + } + Record nrec(NULL, kbuf, ksiz, vbuf, vsiz); + *entp = nrec.serialize(); + count_ += 1; + size_ += ksiz + vsiz; + } + } + /** + * Get the hash value of a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the hash value. + */ + size_t hash_record(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + return hashmurmur(kbuf, ksiz); + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes. + */ + int64_t size_impl() { + _assert_(true); + return bnum_ * sizeof(*buckets_) + count_ * (4 + sizeof(void*)) + size_; + } + /** + * Escape cursors on a shifted or removed records. + * @param rbuf the record buffer. + */ + void escape_cursors(char* rbuf) { + _assert_(rbuf); + ScopedMutex lock(&flock_); + if (curs_.empty()) return; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->rbuf_ == rbuf) cur->step_impl(); + ++cit; + } + } + /** + * Adjust cursors on re-allocated records. + * @param obuf the old address. + * @param nbuf the new address. + */ + void adjust_cursors(char* obuf, char* nbuf) { + _assert_(obuf && nbuf); + ScopedMutex lock(&flock_); + if (curs_.empty()) return; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + if (cur->rbuf_ == obuf) cur->rbuf_ = nbuf; + ++cit; + } + } + /** + * Disable all cursors. + */ + void disable_cursors() { + _assert_(true); + ScopedMutex lock(&flock_); + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->bidx_ = -1; + cur->rbuf_ = NULL; + ++cit; + } + } + /** + * Apply transaction logs. + */ + void apply_trlogs() { + _assert_(true); + TranLogList::const_iterator it = trlogs_.end(); + TranLogList::const_iterator itbeg = trlogs_.begin(); + while (it != itbeg) { + --it; + const char* kbuf = it->key.c_str(); + size_t ksiz = it->key.size(); + const char* vbuf = it->value.c_str(); + size_t vsiz = it->value.size(); + size_t bidx = hash_record(kbuf, ksiz) % bnum_; + if (it->full) { + Setter setter(vbuf, vsiz); + accept_impl(kbuf, ksiz, &setter, bidx); + } else { + Remover remover; + accept_impl(kbuf, ksiz, &remover, bidx); + } + } + } + /** Dummy constructor to forbid the use. */ + StashDB(const StashDB&); + /** Dummy Operator to forbid the use. */ + StashDB& operator =(const StashDB&); + /** The method lock. */ + RWLock mlock_; + /** The record locks. */ + SlottedRWLock rlock_; + /** The file lock. */ + Mutex flock_; + /** The last happened error. */ + TSD<Error> error_; + /** The internal logger. */ + Logger* logger_; + /** The kinds of logged messages. */ + uint32_t logkinds_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The open mode. */ + uint32_t omode_; + /** The cursor objects. */ + CursorList curs_; + /** The path of the database file. */ + std::string path_; + /** The number of buckets. */ + size_t bnum_; + /** The opaque data. */ + char opaque_[OPAQUESIZ]; + /** The record number. */ + AtomicInt64 count_; + /** The total size of records. */ + AtomicInt64 size_; + /** The bucket array. */ + char** buckets_; + /** The flag whether in transaction. */ + bool tran_; + /** The list of transaction logs. */ + TranLogList trlogs_; + /** The count history for transaction. */ + int64_t trcount_; + /** The size history for transaction. */ + int64_t trsize_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcstashtest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcstashtest.cc new file mode 100644 index 0000000000..9ec9591d4b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcstashtest.cc @@ -0,0 +1,2142 @@ +/************************************************************************************************* + * The test cases of the stash database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kcstashdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t procorder(int64_t rnum, int32_t thnum, bool rnd, bool etc, bool tran, + int64_t bnum, bool lv); +static int32_t procqueue(int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int64_t bnum, bool lv); +static int32_t procwicked(int64_t rnum, int32_t thnum, int32_t itnum, int64_t bnum, bool lv); +static int32_t proctran(int64_t rnum, int32_t thnum, int32_t itnum, int64_t bnum, bool lv); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the stash database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-etc] [-tran] [-bnum num] [-lv] rnum\n", + g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-bnum num] [-lv] rnum\n", g_progname); + eprintf(" %s wicked [-th num] [-it num] [-bnum num] [-lv] rnum\n", g_progname); + eprintf(" %s tran [-th num] [-it num] [-bnum num] [-lv] rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["bnum_used"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t rtype = kc::atoi(status["realtype"].c_str()); + if (rtype > 0 && rtype != type) + oprintf("real type: %s (%s) (realtype=0x%02X)\n", + kc::BasicDB::typecname(rtype), kc::BasicDB::typestring(rtype), rtype); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t bnumused = kc::atoi(status["bnum_used"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + double load = 0; + if (count > 0 && bnumused > 0) load = (double)count / bnumused; + oprintf("buckets: %lld (used=%lld) (load=%.2f)\n", + (long long)bnum, (long long)bnumused, load); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s)\n", size, sizestr.c_str()); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + bool etc = false; + bool tran = false; + int64_t bnum = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-etc")) { + etc = true; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(rnum, thnum, rnd, etc, tran, bnum, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int64_t bnum = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(rnum, thnum, itnum, rnd, bnum, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int64_t bnum = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(rnum, thnum, itnum, bnum, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int64_t bnum = -1; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(rnum, thnum, itnum, bnum, lv); + return rv; +} + + +// perform order command +static int32_t procorder(int64_t rnum, int32_t thnum, bool rnd, bool etc, bool tran, + int64_t bnum, bool lv) { + oprintf("<In-order Test>\n seed=%u rnum=%lld thnum=%d rnd=%d etc=%d tran=%d" + " bnum=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, rnd, etc, tran, (long long)bnum, lv); + bool err = false; + kc::StashDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (bnum > 0) db.tune_buckets(bnum); + if (!db.open(":", kc::StashDB::OWRITER | kc::StashDB::OCREATE | kc::StashDB::OTRUNCATE)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (etc) { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::StashDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc) { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size) : + rnum_(rnum), rnd_(rnd), size_(size) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + } syncprocessor(rnum, rnd, db.size()); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (etc && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool etc, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + etc_ = etc; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && !etc_) || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool etc_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, etc, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, etc, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, true); + oprintf("time: %.3f\n", etime - stime); + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int64_t bnum, bool lv) { + oprintf("<Queue Test>\n seed=%u rnum=%lld thnum=%d itnum=%d rnd=%d" + " bnum=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, rnd, (long long)bnum, lv); + bool err = false; + kc::StashDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (bnum > 0) db.tune_buckets(bnum); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::StashDB::OWRITER | kc::StashDB::OCREATE; + if (itcnt == 1) omode |= kc::StashDB::OTRUNCATE; + if (!db.open(":", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::StashDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::StashDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(int64_t rnum, int32_t thnum, int32_t itnum, int64_t bnum, bool lv) { + oprintf("<Wicked Test>\n seed=%u rnum=%lld thnum=%d itnum=%d" + " bnum=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, (long long)bnum, lv); + bool err = false; + kc::StashDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (bnum > 0) db.tune_buckets(bnum); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::StashDB::OWRITER | kc::StashDB::OCREATE; + if (itcnt == 1) omode |= kc::StashDB::OTRUNCATE; + if (!db.open(":", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::StashDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(9)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(5) > 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::StashDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(int64_t rnum, int32_t thnum, int32_t itnum, int64_t bnum, bool lv) { + oprintf("<Transaction Test>\n seed=%u rnum=%lld thnum=%d itnum=%d" + " bnum=%lld lv=%d\n\n", + g_randseed, (long long)rnum, thnum, itnum, (long long)bnum, lv); + bool err = false; + kc::StashDB db; + kc::StashDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (bnum > 0) db.tune_buckets(bnum); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::StashDB::OWRITER | kc::StashDB::OCREATE; + if (itcnt == 1) omode |= kc::StashDB::OTRUNCATE; + if (!db.open(":", omode)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + if (!paradb.open("para", omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::StashDB* db, kc::StashDB* paradb, int64_t rnum, + int32_t thnum, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::StashDB* db_; + kc::StashDB* paradb_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kctextdb.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kctextdb.cc new file mode 100644 index 0000000000..0e98b6185e --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kctextdb.cc @@ -0,0 +1,27 @@ +/************************************************************************************************* + * Plain text database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kctextdb.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +// There is no implementation now. + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kctextdb.h b/plugins/Dbx_kyoto/src/kyotocabinet/kctextdb.h new file mode 100644 index 0000000000..324af3d8f1 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kctextdb.h @@ -0,0 +1,1249 @@ +/************************************************************************************************* + * Plain text database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCTEXTDB_H // duplication check +#define _KCTEXTDB_H + +#include <kccommon.h> +#include <kcutil.h> +#include <kcthread.h> +#include <kcfile.h> +#include <kccompress.h> +#include <kccompare.h> +#include <kcmap.h> +#include <kcregex.h> +#include <kcdb.h> + +namespace kyotocabinet { // common namespace + + +/** + * Plain text database. + * @note Although this class is designed to use a text file as a database file, not all methods + * are implemented. Each line in the text is treated as a record. When storing a record, the + * key is ignored and the value only is appended at the end of the file. Records can be + * retrieved only by the iterator and the cursor mechanisms. No record can be retrieved by + * specifying the key. When accessing a record by the iterator, the key is given as the offset + * from the beginning of the file for descriptive purposes. Any existing record cannot be + * modified and deleted. + */ +class TextDB : public BasicDB { + public: + class Cursor; + private: + class ScopedVisitor; + /** An alias of list of cursors. */ + typedef std::list<Cursor*> CursorList; + /** An alias of a past record. */ + typedef std::pair<int64_t, std::string> Record; + /** The size of the IO buffer. */ + static const size_t IOBUFSIZ = 1024; + public: + /** + * Cursor to indicate a record. + */ + class Cursor : public BasicDB::Cursor { + friend class TextDB; + public: + /** + * Constructor. + * @param db the container database object. + */ + explicit Cursor(TextDB* db) : db_(db), off_(INT64MAX), end_(0), queue_(), line_() { + _assert_(db); + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.push_back(this); + } + /** + * Destructor. + */ + virtual ~Cursor() { + _assert_(true); + if (!db_) return; + ScopedRWLock lock(&db_->mlock_, true); + db_->curs_.remove(this); + } + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + * @note The key is generated from the offset of each record. To avoid deadlock, any + * explicit database operation must not be performed in this function. + */ + bool accept(Visitor* visitor, bool writable = true, bool step = false) { + _assert_(visitor); + ScopedRWLock lock(&db_->mlock_, false); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !db_->writer_) { + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + bool err = false; + if (!accept_impl(visitor, step)) err = true; + return !err; + } + /** + * Jump the cursor to the first record for forward scan. + * @return true on success, or false on failure. + */ + bool jump() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, false); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + off_ = 0; + end_ = db_->file_.size(); + queue_.clear(); + line_.clear(); + if (off_ >= end_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Jump the cursor to a record for forward scan. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return true on success, or false on failure. + */ + bool jump(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + off_ = atoin(kbuf, ksiz); + end_ = db_->file_.size(); + queue_.clear(); + line_.clear(); + if (off_ >= end_) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + return true; + } + /** + * Jump the cursor to a record for forward scan. + * @note Equal to the original Cursor::jump method except that the parameter is std::string. + */ + bool jump(const std::string& key) { + _assert_(true); + return jump(key.c_str(), key.size()); + } + /** + * Jump the cursor to the last record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Jump the cursor to a record for backward scan. + * @note This is a dummy implementation for compatibility. + */ + bool jump_back(const std::string& key) { + _assert_(true); + return jump_back(key.c_str(), key.size()); + } + /** + * Step the cursor to the next record. + * @return true on success, or false on failure. + */ + bool step() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, false); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (queue_.empty() && !read_next()) return false; + if (queue_.empty()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + queue_.pop_front(); + return true; + } + /** + * Step the cursor to the previous record. + * @note This is a dummy implementation for compatibility. + */ + bool step_back() { + _assert_(true); + ScopedRWLock lock(&db_->mlock_, true); + if (db_->omode_ == 0) { + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Get the database object. + * @return the database object. + */ + TextDB* db() { + _assert_(true); + return db_; + } + private: + /** + * Accept a visitor to the current record. + * @param visitor a visitor object. + * @param step true to move the cursor to the next record, or false for no move. + * @return true on success, or false on failure. + */ + bool accept_impl(Visitor* visitor, bool step) { + _assert_(visitor); + if (queue_.empty() && !read_next()) return false; + if (queue_.empty()) { + db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); + return false; + } + bool err = false; + const Record& rec = queue_.front(); + char kbuf[NUMBUFSIZ]; + size_t ksiz = db_->write_key(kbuf, rec.first); + size_t vsiz; + const char* vbuf = visitor->visit_full(kbuf, ksiz, + rec.second.data(), rec.second.size(), &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + char stack[IOBUFSIZ]; + size_t rsiz = vsiz + 1; + char* rbuf = rsiz > sizeof(stack) ? new char[rsiz] : stack; + std::memcpy(rbuf, vbuf, vsiz); + rbuf[vsiz] = '\n'; + if (!db_->file_.append(rbuf, rsiz)) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, db_->file_.error()); + err = true; + } + if (rbuf != stack) delete[] rbuf; + if (db_->autosync_ && !db_->file_.synchronize(true)) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, db_->file_.error()); + err = true; + } + } + if (step) queue_.pop_front(); + return !err; + } + /** + * Read the next record. + * @return true on success, or false on failure. + */ + bool read_next() { + _assert_(true); + while (off_ < end_) { + char stack[IOBUFSIZ]; + int64_t rsiz = end_ - off_; + if (rsiz > (int64_t)sizeof(stack)) rsiz = sizeof(stack); + if (!db_->file_.read_fast(off_, stack, rsiz)) { + db_->set_error(_KCCODELINE_, Error::SYSTEM, db_->file_.error()); + return false; + } + const char* rp = stack; + const char* pv = rp; + const char* ep = rp + rsiz; + while (rp < ep) { + if (*rp == '\n') { + line_.append(pv, rp - pv); + Record rec; + rec.first = off_ + pv - stack; + rec.second = line_; + queue_.push_back(rec); + line_.clear(); + rp++; + pv = rp; + } else { + rp++; + } + } + line_.append(pv, rp - pv); + off_ += rsiz; + if (!queue_.empty()) break; + } + return true; + } + /** Dummy constructor to forbid the use. */ + Cursor(const Cursor&); + /** Dummy Operator to forbid the use. */ + Cursor& operator =(const Cursor&); + /** The inner database. */ + TextDB* db_; + /** The current offset. */ + int64_t off_; + /** The end offset. */ + int64_t end_; + /** The queue of read lines. */ + std::deque<Record> queue_; + /** The current line. */ + std::string line_; + }; + /** + * Default constructor. + */ + explicit TextDB() : + error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL), + omode_(0), writer_(false), autotran_(false), autosync_(false), + file_(), curs_(), path_("") { + _assert_(true); + } + /** + * Destructor. + * @note If the database is not closed, it is closed implicitly. + */ + virtual ~TextDB() { + _assert_(true); + if (omode_ != 0) close(); + if (!curs_.empty()) { + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->db_ = NULL; + ++cit; + } + } + } + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note No record can be retrieved by specifying the key and the Visitor::visit_empty method + * is always called. To avoid deadlock, any explicit database operation must not be performed + * in this function. + */ + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + bool err = false; + if (!accept_impl(kbuf, ksiz, visitor)) err = true; + return !err; + } + /** + * Accept a visitor to multiple records at once. + * @param keys specifies a string vector of the keys. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @return true on success, or false on failure. + * @note No record can be retrieved by specifying the key and the Visitor::visit_empty method + * is always called. To avoid deadlock, any explicit database operation must not be performed + * in this function. + */ + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, + bool writable = true) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + bool err = false; + std::vector<std::string>::const_iterator kit = keys.begin(); + std::vector<std::string>::const_iterator kitend = keys.end(); + while (kit != kitend) { + if (!accept_impl(kit->data(), kit->size(), visitor)) err = true; + ++kit; + } + return !err; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param writable true for writable operation, or false for read-only operation. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The key is generated from the offset of each record. To avoid deadlock, any explicit + * database operation must not be performed in this function. + */ + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) { + _assert_(visitor); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (writable && !writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + ScopedVisitor svis(visitor); + bool err = false; + if (!iterate_impl(visitor, checker)) err = true; + trigger_meta(MetaTrigger::ITERATE, "iterate"); + return !err; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note This function is for reading records and not for updating ones. The return value of + * the visitor is just ignored. The key is generated from the offset of each record. To + * avoid deadlock, any explicit database operation must not be performed in this function. + */ + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) { + _assert_(visitor && thnum <= MEMMAXSIZ); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (thnum < 1) thnum = 1; + if (thnum > (size_t)INT8MAX) thnum = INT8MAX; + ScopedVisitor svis(visitor); + bool err = false; + if (!scan_parallel_impl(visitor, thnum, checker)) err = true; + trigger_meta(MetaTrigger::ITERATE, "scan_parallel"); + return !err; + } + /** + * Get the last happened error. + * @return the last happened error. + */ + Error error() const { + _assert_(true); + return error_; + } + /** + * Set the error information. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param code an error code. + * @param message a supplement message. + */ + void set_error(const char* file, int32_t line, const char* func, + Error::Code code, const char* message) { + _assert_(file && line > 0 && func && message); + error_->set(code, message); + if (logger_) { + Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ? + Logger::ERROR : Logger::INFO; + if (kind & logkinds_) + report(file, line, func, kind, "%d: %s: %s", code, Error::codename(code), message); + } + } + /** + * Open a database file. + * @param path the path of a database file. + * @param mode the connection mode. TextDB::OWRITER as a writer, TextDB::OREADER as a + * reader. The following may be added to the writer mode by bitwise-or: TextDB::OCREATE, + * which means it creates a new database if the file does not exist, TextDB::OTRUNCATE, which + * means it creates a new database regardless if the file exists, TextDB::OAUTOTRAN, which + * means each updating operation is performed in implicit transaction, TextDB::OAUTOSYNC, + * which means each updating operation is followed by implicit synchronization with the file + * system. The following may be added to both of the reader mode and the writer mode by + * bitwise-or: TextDB::ONOLOCK, which means it opens the database file without file locking, + * TextDB::OTRYLOCK, which means locking is performed without blocking, TextDB::ONOREPAIR, + * which means the database file is not repaired implicitly even if file destruction is + * detected. + * @return true on success, or false on failure. + * @note Every opened database must be closed by the TextDB::close method when it is no + * longer in use. It is not allowed for two or more database objects in the same process to + * keep their connections to the same database file at the same time. + */ + bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", path.c_str()); + writer_ = false; + autotran_ = false; + autosync_ = false; + uint32_t fmode = File::OREADER; + if (mode & OWRITER) { + writer_ = true; + fmode = File::OWRITER; + if (mode & OCREATE) fmode |= File::OCREATE; + if (mode & OTRUNCATE) fmode |= File::OTRUNCATE; + if (mode & OAUTOTRAN) autotran_ = true; + if (mode & OAUTOSYNC) autosync_ = true; + } + if (mode & ONOLOCK) fmode |= File::ONOLOCK; + if (mode & OTRYLOCK) fmode |= File::OTRYLOCK; + if (!file_.open(path, fmode, 0)) { + const char* emsg = file_.error(); + Error::Code code = Error::SYSTEM; + if (std::strstr(emsg, "(permission denied)") || std::strstr(emsg, "(directory)")) { + code = Error::NOPERM; + } else if (std::strstr(emsg, "(file not found)") || std::strstr(emsg, "(invalid path)")) { + code = Error::NOREPOS; + } + set_error(_KCCODELINE_, code, emsg); + return false; + } + if (autosync_ && !File::synchronize_whole()) { + set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); + file_.close(); + return false; + } + path_.append(path); + omode_ = mode; + trigger_meta(MetaTrigger::OPEN, "open"); + return true; + } + /** + * Close the database file. + * @return true on success, or false on failure. + */ + bool close() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", path_.c_str()); + bool err = false; + disable_cursors(); + if (!file_.close()) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + omode_ = 0; + path_.clear(); + trigger_meta(MetaTrigger::CLOSE, "close"); + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. If it is NULL, no postprocessing is performed. + * @param checker a progress checker object. If it is NULL, no checking is performed. + * @return true on success, or false on failure. + * @note The operation of the postprocessor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool synchronize(bool hard = false, FileProcessor* proc = NULL, + ProgressChecker* checker = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + bool err = false; + if (!synchronize_impl(hard, proc, checker)) err = true; + trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize"); + return !err; + } + /** + * Occupy database by locking and do something meanwhile. + * @param writable true to use writer lock, or false to use reader lock. + * @param proc a processor object. If it is NULL, no processing is performed. + * @return true on success, or false on failure. + * @note The operation of the processor is performed atomically and other threads accessing + * the same record are blocked. To avoid deadlock, any explicit database operation must not + * be performed in this function. + */ + bool occupy(bool writable = true, FileProcessor* proc = NULL) { + _assert_(true); + ScopedRWLock lock(&mlock_, writable); + bool err = false; + if (proc && !proc->process(path_, -1, file_.size())) { + set_error(_KCCODELINE_, Error::LOGIC, "processing failed"); + err = true; + } + trigger_meta(MetaTrigger::OCCUPY, "occupy"); + return !err; + } + /** + * Begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction(bool hard = false) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Try to begin transaction. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @return true on success, or false on failure. + */ + bool begin_transaction_try(bool hard = false) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * End transaction. + * @param commit true to commit the transaction, or false to abort the transaction. + * @return true on success, or false on failure. + */ + bool end_transaction(bool commit = true) { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return false; + } + /** + * Remove all records. + * @return true on success, or false on failure. + */ + bool clear() { + _assert_(true); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + if (!writer_) { + set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); + return false; + } + disable_cursors(); + if (!file_.truncate(0)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + if (autosync_ && !file_.synchronize(true)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + trigger_meta(MetaTrigger::CLEAR, "clear"); + return true; + } + /** + * Get the number of records. + * @return the number of records, or -1 on failure. + */ + int64_t count() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); + return -1; + } + /** + * Get the size of the database file. + * @return the size of the database file in bytes, or -1 on failure. + */ + int64_t size() { + _assert_(true); + ScopedRWLock lock(&mlock_, false); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return -1; + } + return file_.size(); + } + /** + * Get the path of the database file. + * @return the path of the database file, or an empty string on failure. + */ + std::string path() { + _assert_(true); + return path_; + } + /** + * Get the miscellaneous status information. + * @param strmap a string map to contain the result. + * @return true on success, or false on failure. + */ + bool status(std::map<std::string, std::string>* strmap) { + _assert_(strmap); + ScopedRWLock lock(&mlock_, true); + if (omode_ == 0) { + set_error(_KCCODELINE_, Error::INVALID, "not opened"); + return false; + } + (*strmap)["type"] = strprintf("%u", (unsigned)TYPETEXT); + (*strmap)["realtype"] = strprintf("%u", (unsigned)TYPETEXT); + (*strmap)["path"] = path_; + (*strmap)["size"] = strprintf("%lld", (long long)file_.size()); + return true; + } + /** + * Create a cursor object. + * @return the return value is the created cursor object. + * @note Because the object of the return value is allocated by the constructor, it should be + * released with the delete operator when it is no longer in use. + */ + Cursor* cursor() { + _assert_(true); + return new Cursor(this); + } + /** + * Write a log message. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param message the supplement message. + */ + void log(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* message) { + _assert_(file && line > 0 && func && message); + ScopedRWLock lock(&mlock_, false); + if (!logger_) return; + logger_->log(file, line, func, kind, message); + } + /** + * Set the internal logger. + * @param logger the logger object. + * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for debugging, + * Logger::INFO for normal information, Logger::WARN for warning, and Logger::ERROR for fatal + * error. + * @return true on success, or false on failure. + */ + bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::ERROR) { + _assert_(logger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + logger_ = logger; + logkinds_ = kinds; + return true; + } + /** + * Set the internal meta operation trigger. + * @param trigger the trigger object. + * @return true on success, or false on failure. + */ + bool tune_meta_trigger(MetaTrigger* trigger) { + _assert_(trigger); + ScopedRWLock lock(&mlock_, true); + if (omode_ != 0) { + set_error(_KCCODELINE_, Error::INVALID, "already opened"); + return false; + } + mtrigger_ = trigger; + return true; + } + protected: + /** + * Report a message for debugging. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ... used according to the format string. + */ + void report(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, ...) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + va_list ap; + va_start(ap, format); + vstrprintf(&message, format, ap); + va_end(ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report a message for debugging with variable number of arguments. + * @param file the file name of the program source code. + * @param line the line number of the program source code. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param format the printf-like format string. + * @param ap used according to the format string. + */ + void report_valist(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* format, va_list ap) { + _assert_(file && line > 0 && func && format); + if (!logger_ || !(kind & logkinds_)) return; + std::string message; + strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str()); + vstrprintf(&message, format, ap); + logger_->log(file, line, func, kind, message.c_str()); + } + /** + * Report the content of a binary buffer for debugging. + * @param file the file name of the epicenter. + * @param line the line number of the epicenter. + * @param func the function name of the program source code. + * @param kind the kind of the event. Logger::DEBUG for debugging, Logger::INFO for normal + * information, Logger::WARN for warning, and Logger::ERROR for fatal error. + * @param name the name of the information. + * @param buf the binary buffer. + * @param size the size of the binary buffer + */ + void report_binary(const char* file, int32_t line, const char* func, Logger::Kind kind, + const char* name, const char* buf, size_t size) { + _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ); + if (!logger_) return; + char* hex = hexencode(buf, size); + report(file, line, func, kind, "%s=%s", name, hex); + delete[] hex; + } + /** + * Trigger a meta database operation. + * @param kind the kind of the event. MetaTrigger::OPEN for opening, MetaTrigger::CLOSE for + * closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for iteration, + * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN for beginning + * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaTrigger::ABORTTRAN + * for aborting transaction, and MetaTrigger::MISC for miscellaneous operations. + * @param message the supplement message. + */ + void trigger_meta(MetaTrigger::Kind kind, const char* message) { + _assert_(message); + if (mtrigger_) mtrigger_->trigger(kind, message); + } + private: + /** + * Scoped visitor. + */ + class ScopedVisitor { + public: + /** constructor */ + explicit ScopedVisitor(Visitor* visitor) : visitor_(visitor) { + _assert_(visitor); + visitor_->visit_before(); + } + /** destructor */ + ~ScopedVisitor() { + _assert_(true); + visitor_->visit_after(); + } + private: + Visitor* visitor_; ///< visitor + }; + /** + * Accept a visitor to a record. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param visitor a visitor object. + * @return true on success, or false on failure. + */ + bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor) { + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); + bool err = false; + size_t vsiz; + const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + size_t rsiz = vsiz + 1; + char stack[IOBUFSIZ]; + char* rbuf = rsiz > sizeof(stack) ? new char[rsiz] : stack; + std::memcpy(rbuf, vbuf, vsiz); + rbuf[vsiz] = '\n'; + if (!file_.append(rbuf, rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + if (rbuf != stack) delete[] rbuf; + if (autosync_ && !file_.synchronize(true)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + } + return !err; + } + /** + * Iterate to accept a visitor for each record. + * @param visitor a visitor object. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool iterate_impl(Visitor* visitor, ProgressChecker* checker) { + _assert_(visitor); + if (checker && !checker->check("iterate", "beginning", 0, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + int64_t off = 0; + int64_t end = file_.size(); + int64_t curcnt = 0; + std::string line; + char stack[IOBUFSIZ*4]; + while (off < end) { + int64_t rsiz = end - off; + if (rsiz > (int64_t)sizeof(stack)) rsiz = sizeof(stack); + if (!file_.read_fast(off, stack, rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + const char* rp = stack; + const char* pv = rp; + const char* ep = rp + rsiz; + while (rp < ep) { + if (*rp == '\n') { + char kbuf[NUMBUFSIZ]; + size_t ksiz = write_key(kbuf, off + pv - stack); + const char* vbuf; + size_t vsiz; + if (line.empty()) { + vbuf = visitor->visit_full(kbuf, ksiz, pv, rp - pv, &vsiz); + } else { + line.append(pv, rp - pv); + vbuf = visitor->visit_full(kbuf, ksiz, line.data(), line.size(), &vsiz); + line.clear(); + } + if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { + char tstack[IOBUFSIZ]; + size_t trsiz = vsiz + 1; + char* trbuf = trsiz > sizeof(tstack) ? new char[trsiz] : tstack; + std::memcpy(trbuf, vbuf, vsiz); + trbuf[vsiz] = '\n'; + if (!file_.append(trbuf, trsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + if (trbuf != stack) delete[] trbuf; + return false; + } + if (trbuf != tstack) delete[] trbuf; + } + curcnt++; + if (checker && !checker->check("iterate", "processing", curcnt, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + rp++; + pv = rp; + } else { + rp++; + } + } + line.append(pv, rp - pv); + off += rsiz; + } + if (checker && !checker->check("iterate", "ending", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + return true; + } + /** + * Scan each record in parallel. + * @param visitor a visitor object. + * @param thnum the number of worker threads. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool scan_parallel_impl(Visitor *visitor, size_t thnum, ProgressChecker* checker) { + _assert_(visitor && thnum <= MEMMAXSIZ); + if (checker && !checker->check("scan_parallel", "beginning", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + int64_t off = 0; + int64_t end = file_.size(); + int64_t gap = (end - off) / thnum; + std::vector<int64_t> offs; + while (off < end) { + offs.push_back(off); + int64_t edge = off + gap; + while (true) { + if (edge >= end) { + off = end; + break; + } + char rbuf[IOBUFSIZ]; + int64_t rsiz = end - edge; + if (rsiz > (int64_t)sizeof(rbuf)) rsiz = sizeof(rbuf); + if (!file_.read_fast(edge, rbuf, rsiz)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + return false; + } + int64_t noff = -1; + const char* rp = rbuf; + const char* ep = rp + rsiz; + while (rp < ep) { + if (*rp == '\n') { + noff = edge + (rp - rbuf); + break; + } + rp++; + } + if (noff >= 0) { + off = noff + 1; + break; + } + edge += rsiz; + } + } + bool err = false; + size_t onum = offs.size(); + if (onum > 0) { + class ThreadImpl : public Thread { + public: + explicit ThreadImpl() : + db_(NULL), visitor_(NULL), checker_(NULL), begoff_(0), endoff_(0), error_() {} + void init(TextDB* db, Visitor* visitor, ProgressChecker* checker, + int64_t begoff, int64_t endoff) { + db_ = db; + visitor_ = visitor; + checker_ = checker; + begoff_ = begoff; + endoff_ = endoff; + } + const Error& error() { + return error_; + } + private: + void run() { + TextDB* db = db_; + File* file = &db->file_; + Visitor* visitor = visitor_; + ProgressChecker* checker = checker_; + int64_t off = begoff_; + int64_t end = endoff_; + std::string line; + char stack[IOBUFSIZ*4]; + while (off < end) { + int64_t rsiz = end - off; + if (rsiz > (int64_t)sizeof(stack)) rsiz = sizeof(stack); + if (!file->read_fast(off, stack, rsiz)) { + db->set_error(_KCCODELINE_, Error::SYSTEM, file->error()); + return; + } + const char* rp = stack; + const char* pv = rp; + const char* ep = rp + rsiz; + while (rp < ep) { + if (*rp == '\n') { + char kbuf[NUMBUFSIZ]; + size_t ksiz = db->write_key(kbuf, off + pv - stack); + if (line.empty()) { + size_t vsiz; + visitor->visit_full(kbuf, ksiz, pv, rp - pv, &vsiz); + } else { + line.append(pv, rp - pv); + size_t vsiz; + visitor->visit_full(kbuf, ksiz, line.data(), line.size(), &vsiz); + line.clear(); + } + if (checker && !checker->check("iterate", "processing", -1, -1)) { + db->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return; + } + rp++; + pv = rp; + } else { + rp++; + } + } + line.append(pv, rp - pv); + off += rsiz; + } + } + TextDB* db_; + Visitor* visitor_; + ProgressChecker* checker_; + int64_t begoff_; + int64_t endoff_; + Error error_; + }; + ThreadImpl* threads = new ThreadImpl[onum]; + for (size_t i = 0; i < onum; i++) { + int64_t begoff = offs[i]; + int64_t endoff = i < onum - 1 ? offs[i+1] : end; + ThreadImpl* thread = threads + i; + thread->init(this, visitor, checker, begoff, endoff); + thread->start(); + } + for (size_t i = 0; i < onum; i++) { + ThreadImpl* thread = threads + i; + thread->join(); + if (thread->error() != Error::SUCCESS) { + *error_ = thread->error(); + err = true; + } + } + delete[] threads; + } + if (checker && !checker->check("scan_parallel", "ending", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + err = true; + } + return !err; + } + /** + * Synchronize updated contents with the file and the device. + * @param hard true for physical synchronization with the device, or false for logical + * synchronization with the file system. + * @param proc a postprocessor object. + * @param checker a progress checker object. + * @return true on success, or false on failure. + */ + bool synchronize_impl(bool hard, FileProcessor* proc, ProgressChecker* checker) { + _assert_(true); + bool err = false; + if (writer_) { + if (checker && !checker->check("synchronize", "synchronizing the file", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!file_.synchronize(hard)) { + set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); + err = true; + } + } + if (proc) { + if (checker && !checker->check("synchronize", "running the post processor", -1, -1)) { + set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); + return false; + } + if (!proc->process(path_, -1, file_.size())) { + set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed"); + err = true; + } + } + return !err; + } + /** + * Disable all cursors. + */ + void disable_cursors() { + _assert_(true); + if (curs_.empty()) return; + CursorList::const_iterator cit = curs_.begin(); + CursorList::const_iterator citend = curs_.end(); + while (cit != citend) { + Cursor* cur = *cit; + cur->off_ = INT64MAX; + ++cit; + } + } + /** + * Write the key pattern into a buffer. + * @param kbuf the destination buffer. + * @param off the offset of the record. + * @return the size of the key pattern. + */ + size_t write_key(char* kbuf, int64_t off) { + _assert_(kbuf && off >= 0); + for (size_t i = 0; i < sizeof(off); i++) { + uint8_t c = off >> ((sizeof(off) - 1 - i) * 8); + uint8_t h = c >> 4; + if (h < 10) { + *(kbuf++) = '0' + h; + } else { + *(kbuf++) = 'A' - 10 + h; + } + uint8_t l = c & 0xf; + if (l < 10) { + *(kbuf++) = '0' + l; + } else { + *(kbuf++) = 'A' - 10 + l; + } + } + return sizeof(off) * 2; + } + /** Dummy constructor to forbid the use. */ + TextDB(const TextDB&); + /** Dummy Operator to forbid the use. */ + TextDB& operator =(const TextDB&); + /** The method lock. */ + RWLock mlock_; + /** The last happened error. */ + TSD<Error> error_; + /** The internal logger. */ + Logger* logger_; + /** The kinds of logged messages. */ + uint32_t logkinds_; + /** The internal meta operation trigger. */ + MetaTrigger* mtrigger_; + /** The open mode. */ + uint32_t omode_; + /** The flag for writer. */ + bool writer_; + /** The flag for auto transaction. */ + bool autotran_; + /** The flag for auto synchronization. */ + bool autosync_; + /** The file for data. */ + File file_; + /** The cursor objects. */ + CursorList curs_; + /** The path of the database file. */ + std::string path_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcthread.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcthread.cc new file mode 100644 index 0000000000..82d1bb9f27 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcthread.cc @@ -0,0 +1,2146 @@ +/************************************************************************************************* + * Threading devices + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcthread.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +/** + * Constants for implementation. + */ +namespace { +const uint32_t LOCKBUSYLOOP = 8192; ///< threshold of busy loop and sleep for locking +const size_t LOCKSEMNUM = 256; ///< number of semaphores for locking +} + + +/** + * Thread internal. + */ +struct ThreadCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + ::HANDLE th; ///< handle +#else + ::pthread_t th; ///< identifier + bool alive; ///< alive flag +#endif +}; + + +/** + * CondVar internal. + */ +struct CondVarCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + ::CRITICAL_SECTION mutex; ///< mutex + uint32_t wait; ///< wait count + uint32_t wake; ///< wake count + ::HANDLE sev; ///< signal event handle + ::HANDLE fev; ///< finish event handle +#else + ::pthread_cond_t cond; ///< condition +#endif +}; + + +/** + * Call the running thread. + * @param arg the thread. + * @return always NULL. + */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static ::DWORD threadrun(::LPVOID arg); +#else +static void* threadrun(void* arg); +#endif + + +/** + * Default constructor. + */ +Thread::Thread() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ThreadCore* core = new ThreadCore; + core->th = NULL; + opq_ = (void*)core; +#else + _assert_(true); + ThreadCore* core = new ThreadCore; + core->alive = false; + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +Thread::~Thread() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ThreadCore* core = (ThreadCore*)opq_; + if (core->th) join(); + delete core; +#else + _assert_(true); + ThreadCore* core = (ThreadCore*)opq_; + if (core->alive) join(); + delete core; +#endif +} + + +/** + * Start the thread. + */ +void Thread::start() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ThreadCore* core = (ThreadCore*)opq_; + if (core->th) throw std::invalid_argument("already started"); + ::DWORD id; + core->th = ::CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)threadrun, this, 0, &id); + if (!core->th) throw std::runtime_error("CreateThread"); +#else + _assert_(true); + ThreadCore* core = (ThreadCore*)opq_; + if (core->alive) throw std::invalid_argument("already started"); + if (::pthread_create(&core->th, NULL, threadrun, this) != 0) + throw std::runtime_error("pthread_create"); + core->alive = true; +#endif +} + + +/** + * Wait for the thread to finish. + */ +void Thread::join() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ThreadCore* core = (ThreadCore*)opq_; + if (!core->th) throw std::invalid_argument("not alive"); + if (::WaitForSingleObject(core->th, INFINITE) == WAIT_FAILED) + throw std::runtime_error("WaitForSingleObject"); + ::CloseHandle(core->th); + core->th = NULL; +#else + _assert_(true); + ThreadCore* core = (ThreadCore*)opq_; + if (!core->alive) throw std::invalid_argument("not alive"); + core->alive = false; + if (::pthread_join(core->th, NULL) != 0) throw std::runtime_error("pthread_join"); +#endif +} + + +/** + * Put the thread in the detached state. + */ +void Thread::detach() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); +#else + _assert_(true); + ThreadCore* core = (ThreadCore*)opq_; + if (!core->alive) throw std::invalid_argument("not alive"); + core->alive = false; + if (::pthread_detach(core->th) != 0) throw std::runtime_error("pthread_detach"); +#endif +} + + +/** + * Terminate the running thread. + */ +void Thread::exit() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::ExitThread(0); +#else + _assert_(true); + ::pthread_exit(NULL); +#endif +} + + +/** + * Yield the processor from the current thread. + */ +void Thread::yield() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::Sleep(0); +#else + _assert_(true); + ::sched_yield(); +#endif +} + + +/** + * Chill the processor by suspending execution for a quick moment. + */ +void Thread::chill() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::Sleep(21); +#else + _assert_(true); + struct ::timespec req; + req.tv_sec = 0; + req.tv_nsec = 21 * 1000 * 1000; + ::nanosleep(&req, NULL); +#endif +} + + + +/** + * Suspend execution of the current thread. + */ +bool Thread::sleep(double sec) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(sec >= 0.0); + if (sec <= 0.0) { + yield(); + return true; + } + if (sec > INT32MAX) sec = INT32MAX; + ::Sleep(sec * 1000); + return true; +#else + _assert_(sec >= 0.0); + if (sec <= 0.0) { + yield(); + return true; + } + if (sec > INT32MAX) sec = INT32MAX; + double integ, fract; + fract = std::modf(sec, &integ); + struct ::timespec req, rem; + req.tv_sec = (time_t)integ; + req.tv_nsec = (long)(fract * 999999000); + while (::nanosleep(&req, &rem) != 0) { + if (errno != EINTR) return false; + req = rem; + } + return true; +#endif +} + + +/** + * Get the hash value of the current thread. + */ +int64_t Thread::hash() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + return ::GetCurrentThreadId(); +#else + _assert_(true); + pthread_t tid = pthread_self(); + int64_t num; + if (sizeof(tid) == sizeof(num)) { + std::memcpy(&num, &tid, sizeof(num)); + } else if (sizeof(tid) == sizeof(int32_t)) { + uint32_t inum; + std::memcpy(&inum, &tid, sizeof(inum)); + num = inum; + } else { + num = hashmurmur(&tid, sizeof(tid)); + } + return num & INT64MAX; +#endif +} + + +/** + * Call the running thread. + */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static ::DWORD threadrun(::LPVOID arg) { + _assert_(true); + Thread* thread = (Thread*)arg; + thread->run(); + return NULL; +} +#else +static void* threadrun(void* arg) { + _assert_(true); + Thread* thread = (Thread*)arg; + thread->run(); + return NULL; +} +#endif + + +/** + * Default constructor. + */ +Mutex::Mutex() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::CRITICAL_SECTION* mutex = new ::CRITICAL_SECTION; + ::InitializeCriticalSection(mutex); + opq_ = (void*)mutex; +#else + _assert_(true); + ::pthread_mutex_t* mutex = new ::pthread_mutex_t; + if (::pthread_mutex_init(mutex, NULL) != 0) throw std::runtime_error("pthread_mutex_init"); + opq_ = (void*)mutex; +#endif +} + + +/** + * Constructor with the specifications. + */ +Mutex::Mutex(Type type) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::CRITICAL_SECTION* mutex = new ::CRITICAL_SECTION; + ::InitializeCriticalSection(mutex); + opq_ = (void*)mutex; +#else + _assert_(true); + ::pthread_mutexattr_t attr; + if (::pthread_mutexattr_init(&attr) != 0) throw std::runtime_error("pthread_mutexattr_init"); + switch (type) { + case FAST: { + break; + } + case ERRORCHECK: { + if (::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK) != 0) + throw std::runtime_error("pthread_mutexattr_settype"); + break; + } + case RECURSIVE: { + if (::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0) + throw std::runtime_error("pthread_mutexattr_settype"); + break; + } + } + ::pthread_mutex_t* mutex = new ::pthread_mutex_t; + if (::pthread_mutex_init(mutex, &attr) != 0) throw std::runtime_error("pthread_mutex_init"); + ::pthread_mutexattr_destroy(&attr); + opq_ = (void*)mutex; +#endif +} + + +/** + * Destructor. + */ +Mutex::~Mutex() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::CRITICAL_SECTION* mutex = (::CRITICAL_SECTION*)opq_; + ::DeleteCriticalSection(mutex); + delete mutex; +#else + _assert_(true); + ::pthread_mutex_t* mutex = (::pthread_mutex_t*)opq_; + ::pthread_mutex_destroy(mutex); + delete mutex; +#endif +} + + +/** + * Get the lock. + */ +void Mutex::lock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::CRITICAL_SECTION* mutex = (::CRITICAL_SECTION*)opq_; + ::EnterCriticalSection(mutex); +#else + _assert_(true); + ::pthread_mutex_t* mutex = (::pthread_mutex_t*)opq_; + if (::pthread_mutex_lock(mutex) != 0) throw std::runtime_error("pthread_mutex_lock"); +#endif +} + + +/** + * Try to get the lock. + */ +bool Mutex::lock_try() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::CRITICAL_SECTION* mutex = (::CRITICAL_SECTION*)opq_; + if (!::TryEnterCriticalSection(mutex)) return false; + return true; +#else + _assert_(true); + ::pthread_mutex_t* mutex = (::pthread_mutex_t*)opq_; + int32_t ecode = ::pthread_mutex_trylock(mutex); + if (ecode == 0) return true; + if (ecode != EBUSY) throw std::runtime_error("pthread_mutex_trylock"); + return false; +#endif +} + + +/** + * Try to get the lock. + */ +bool Mutex::lock_try(double sec) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || defined(_SYS_CYGWIN_) || defined(_SYS_MACOSX_) + _assert_(sec >= 0.0); + if (lock_try()) return true; + double end = time() + sec; + Thread::yield(); + uint32_t wcnt = 0; + while (!lock_try()) { + if (time() > end) return false; + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + return true; +#else + _assert_(sec >= 0.0); + ::pthread_mutex_t* mutex = (::pthread_mutex_t*)opq_; + struct ::timeval tv; + struct ::timespec ts; + if (::gettimeofday(&tv, NULL) == 0) { + double integ; + double fract = std::modf(sec, &integ); + ts.tv_sec = tv.tv_sec + (time_t)integ; + ts.tv_nsec = (long)(tv.tv_usec * 1000.0 + fract * 999999000); + if (ts.tv_nsec >= 1000000000) { + ts.tv_nsec -= 1000000000; + ts.tv_sec++; + } + } else { + ts.tv_sec = std::time(NULL) + 1; + ts.tv_nsec = 0; + } + int32_t ecode = ::pthread_mutex_timedlock(mutex, &ts); + if (ecode == 0) return true; + if (ecode != ETIMEDOUT) throw std::runtime_error("pthread_mutex_timedlock"); + return false; +#endif +} + + +/** + * Release the lock. + */ +void Mutex::unlock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::CRITICAL_SECTION* mutex = (::CRITICAL_SECTION*)opq_; + ::LeaveCriticalSection(mutex); +#else + _assert_(true); + ::pthread_mutex_t* mutex = (::pthread_mutex_t*)opq_; + if (::pthread_mutex_unlock(mutex) != 0) throw std::runtime_error("pthread_mutex_unlock"); +#endif +} + + +/** + * SlottedMutex internal. + */ +struct SlottedMutexCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + ::CRITICAL_SECTION* mutexes; ///< primitives + size_t slotnum; ///< number of slots +#else + ::pthread_mutex_t* mutexes; ///< primitives + size_t slotnum; ///< number of slots +#endif +}; + + +/** + * Constructor. + */ +SlottedMutex::SlottedMutex(size_t slotnum) : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedMutexCore* core = new SlottedMutexCore; + ::CRITICAL_SECTION* mutexes = new ::CRITICAL_SECTION[slotnum]; + for (size_t i = 0; i < slotnum; i++) { + ::InitializeCriticalSection(mutexes + i); + } + core->mutexes = mutexes; + core->slotnum = slotnum; + opq_ = (void*)core; +#else + _assert_(true); + SlottedMutexCore* core = new SlottedMutexCore; + ::pthread_mutex_t* mutexes = new ::pthread_mutex_t[slotnum]; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_mutex_init(mutexes + i, NULL) != 0) + throw std::runtime_error("pthread_mutex_init"); + } + core->mutexes = mutexes; + core->slotnum = slotnum; + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +SlottedMutex::~SlottedMutex() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::CRITICAL_SECTION* mutexes = core->mutexes; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + ::DeleteCriticalSection(mutexes + i); + } + delete[] mutexes; + delete core; +#else + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::pthread_mutex_t* mutexes = core->mutexes; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + ::pthread_mutex_destroy(mutexes + i); + } + delete[] mutexes; + delete core; +#endif +} + + +/** + * Get the lock of a slot. + */ +void SlottedMutex::lock(size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::EnterCriticalSection(core->mutexes + idx); +#else + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + if (::pthread_mutex_lock(core->mutexes + idx) != 0) + throw std::runtime_error("pthread_mutex_lock"); +#endif +} + + +/** + * Release the lock of a slot. + */ +void SlottedMutex::unlock(size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::LeaveCriticalSection(core->mutexes + idx); +#else + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + if (::pthread_mutex_unlock(core->mutexes + idx) != 0) + throw std::runtime_error("pthread_mutex_unlock"); +#endif +} + + +/** + * Get the locks of all slots. + */ +void SlottedMutex::lock_all() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::CRITICAL_SECTION* mutexes = core->mutexes; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + ::EnterCriticalSection(mutexes + i); + } +#else + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::pthread_mutex_t* mutexes = core->mutexes; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_mutex_lock(mutexes + i) != 0) + throw std::runtime_error("pthread_mutex_lock"); + } +#endif +} + + +/** + * Release the locks of all slots. + */ +void SlottedMutex::unlock_all() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::CRITICAL_SECTION* mutexes = core->mutexes; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + ::LeaveCriticalSection(mutexes + i); + } +#else + _assert_(true); + SlottedMutexCore* core = (SlottedMutexCore*)opq_; + ::pthread_mutex_t* mutexes = core->mutexes; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_mutex_unlock(mutexes + i) != 0) + throw std::runtime_error("pthread_mutex_unlock"); + } +#endif +} + + +/** + * Default constructor. + */ +SpinLock::SpinLock() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); +#elif _KC_GCCATOMIC + _assert_(true); +#else + _assert_(true); + ::pthread_spinlock_t* spin = new ::pthread_spinlock_t; + if (::pthread_spin_init(spin, PTHREAD_PROCESS_PRIVATE) != 0) + throw std::runtime_error("pthread_spin_init"); + opq_ = (void*)spin; +#endif +} + + +/** + * Destructor. + */ +SpinLock::~SpinLock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); +#elif _KC_GCCATOMIC + _assert_(true); +#else + _assert_(true); + ::pthread_spinlock_t* spin = (::pthread_spinlock_t*)opq_; + ::pthread_spin_destroy(spin); + delete spin; +#endif +} + + +/** + * Get the lock. + */ +void SpinLock::lock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + uint32_t wcnt = 0; + while (::InterlockedCompareExchange((LONG*)&opq_, 1, 0) != 0) { + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } +#elif _KC_GCCATOMIC + _assert_(true); + uint32_t wcnt = 0; + while (!__sync_bool_compare_and_swap(&opq_, 0, 1)) { + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } +#else + _assert_(true); + ::pthread_spinlock_t* spin = (::pthread_spinlock_t*)opq_; + if (::pthread_spin_lock(spin) != 0) throw std::runtime_error("pthread_spin_lock"); +#endif +} + + +/** + * Try to get the lock. + */ +bool SpinLock::lock_try() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + return ::InterlockedCompareExchange((LONG*)&opq_, 1, 0) == 0; +#elif _KC_GCCATOMIC + _assert_(true); + return __sync_bool_compare_and_swap(&opq_, 0, 1); +#else + _assert_(true); + ::pthread_spinlock_t* spin = (::pthread_spinlock_t*)opq_; + int32_t ecode = ::pthread_spin_trylock(spin); + if (ecode == 0) return true; + if (ecode != EBUSY) throw std::runtime_error("pthread_spin_trylock"); + return false; +#endif +} + + +/** + * Release the lock. + */ +void SpinLock::unlock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::InterlockedExchange((LONG*)&opq_, 0); +#elif _KC_GCCATOMIC + _assert_(true); + __sync_lock_release(&opq_); +#else + _assert_(true); + ::pthread_spinlock_t* spin = (::pthread_spinlock_t*)opq_; + if (::pthread_spin_unlock(spin) != 0) throw std::runtime_error("pthread_spin_unlock"); +#endif +} + + +/** + * SlottedSpinLock internal. + */ +struct SlottedSpinLockCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || _KC_GCCATOMIC + uint32_t* locks; ///< primitives + size_t slotnum; ///< number of slots +#else + ::pthread_spinlock_t* spins; ///< primitives + size_t slotnum; ///< number of slots +#endif +}; + + +/** + * Constructor. + */ +SlottedSpinLock::SlottedSpinLock(size_t slotnum) : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || _KC_GCCATOMIC + _assert_(true); + SlottedSpinLockCore* core = new SlottedSpinLockCore; + uint32_t* locks = new uint32_t[slotnum]; + for (size_t i = 0; i < slotnum; i++) { + locks[i] = 0; + } + core->locks = locks; + core->slotnum = slotnum; + opq_ = (void*)core; +#else + _assert_(true); + SlottedSpinLockCore* core = new SlottedSpinLockCore; + ::pthread_spinlock_t* spins = new ::pthread_spinlock_t[slotnum]; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_spin_init(spins + i, PTHREAD_PROCESS_PRIVATE) != 0) + throw std::runtime_error("pthread_spin_init"); + } + core->spins = spins; + core->slotnum = slotnum; + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +SlottedSpinLock::~SlottedSpinLock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || _KC_GCCATOMIC + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + delete[] core->locks; + delete core; +#else + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + ::pthread_spinlock_t* spins = core->spins; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + ::pthread_spin_destroy(spins + i); + } + delete[] spins; + delete core; +#endif +} + + +/** + * Get the lock of a slot. + */ +void SlottedSpinLock::lock(size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* lock = core->locks + idx; + uint32_t wcnt = 0; + while (::InterlockedCompareExchange((LONG*)lock, 1, 0) != 0) { + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } +#elif _KC_GCCATOMIC + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* lock = core->locks + idx; + uint32_t wcnt = 0; + while (!__sync_bool_compare_and_swap(lock, 0, 1)) { + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } +#else + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + if (::pthread_spin_lock(core->spins + idx) != 0) + throw std::runtime_error("pthread_spin_lock"); +#endif +} + + +/** + * Release the lock of a slot. + */ +void SlottedSpinLock::unlock(size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* lock = core->locks + idx; + ::InterlockedExchange((LONG*)lock, 0); +#elif _KC_GCCATOMIC + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* lock = core->locks + idx; + __sync_lock_release(lock); +#else + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + if (::pthread_spin_unlock(core->spins + idx) != 0) + throw std::runtime_error("pthread_spin_unlock"); +#endif +} + + +/** + * Get the locks of all slots. + */ +void SlottedSpinLock::lock_all() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* locks = core->locks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + uint32_t* lock = locks + i; + uint32_t wcnt = 0; + while (::InterlockedCompareExchange((LONG*)lock, 1, 0) != 0) { + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + } +#elif _KC_GCCATOMIC + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* locks = core->locks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + uint32_t* lock = locks + i; + uint32_t wcnt = 0; + while (!__sync_bool_compare_and_swap(lock, 0, 1)) { + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + } + } +#else + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + ::pthread_spinlock_t* spins = core->spins; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_spin_lock(spins + i) != 0) + throw std::runtime_error("pthread_spin_lock"); + } +#endif +} + + +/** + * Release the locks of all slots. + */ +void SlottedSpinLock::unlock_all() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* locks = core->locks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + uint32_t* lock = locks + i; + ::InterlockedExchange((LONG*)lock, 0); + } +#elif _KC_GCCATOMIC + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + uint32_t* locks = core->locks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + uint32_t* lock = locks + i; + __sync_lock_release(lock); + } +#else + _assert_(true); + SlottedSpinLockCore* core = (SlottedSpinLockCore*)opq_; + ::pthread_spinlock_t* spins = core->spins; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_spin_unlock(spins + i) != 0) + throw std::runtime_error("pthread_spin_unlock"); + } +#endif +} + + +/** + * Default constructor. + */ +RWLock::RWLock() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SpinRWLock* rwlock = new SpinRWLock; + opq_ = (void*)rwlock; +#else + _assert_(true); + ::pthread_rwlock_t* rwlock = new ::pthread_rwlock_t; + if (::pthread_rwlock_init(rwlock, NULL) != 0) throw std::runtime_error("pthread_rwlock_init"); + opq_ = (void*)rwlock; +#endif +} + + +/** + * Destructor. + */ +RWLock::~RWLock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SpinRWLock* rwlock = (SpinRWLock*)opq_; + delete rwlock; +#else + _assert_(true); + ::pthread_rwlock_t* rwlock = (::pthread_rwlock_t*)opq_; + ::pthread_rwlock_destroy(rwlock); + delete rwlock; +#endif +} + + +/** + * Get the writer lock. + */ +void RWLock::lock_writer() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SpinRWLock* rwlock = (SpinRWLock*)opq_; + rwlock->lock_writer(); +#else + _assert_(true); + ::pthread_rwlock_t* rwlock = (::pthread_rwlock_t*)opq_; + if (::pthread_rwlock_wrlock(rwlock) != 0) throw std::runtime_error("pthread_rwlock_lock"); +#endif +} + + +/** + * Try to get the writer lock. + */ +bool RWLock::lock_writer_try() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SpinRWLock* rwlock = (SpinRWLock*)opq_; + return rwlock->lock_writer_try(); +#else + _assert_(true); + ::pthread_rwlock_t* rwlock = (::pthread_rwlock_t*)opq_; + int32_t ecode = ::pthread_rwlock_trywrlock(rwlock); + if (ecode == 0) return true; + if (ecode != EBUSY) throw std::runtime_error("pthread_rwlock_trylock"); + return false; +#endif +} + + +/** + * Get a reader lock. + */ +void RWLock::lock_reader() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SpinRWLock* rwlock = (SpinRWLock*)opq_; + rwlock->lock_reader(); +#else + _assert_(true); + ::pthread_rwlock_t* rwlock = (::pthread_rwlock_t*)opq_; + if (::pthread_rwlock_rdlock(rwlock) != 0) throw std::runtime_error("pthread_rwlock_lock"); +#endif +} + + +/** + * Try to get a reader lock. + */ +bool RWLock::lock_reader_try() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SpinRWLock* rwlock = (SpinRWLock*)opq_; + return rwlock->lock_reader_try(); +#else + _assert_(true); + ::pthread_rwlock_t* rwlock = (::pthread_rwlock_t*)opq_; + int32_t ecode = ::pthread_rwlock_tryrdlock(rwlock); + if (ecode == 0) return true; + if (ecode != EBUSY) throw std::runtime_error("pthread_rwlock_trylock"); + return false; +#endif +} + + +/** + * Release the lock. + */ +void RWLock::unlock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SpinRWLock* rwlock = (SpinRWLock*)opq_; + rwlock->unlock(); +#else + _assert_(true); + ::pthread_rwlock_t* rwlock = (::pthread_rwlock_t*)opq_; + if (::pthread_rwlock_unlock(rwlock) != 0) throw std::runtime_error("pthread_rwlock_unlock"); +#endif +} + + +/** + * SlottedRWLock internal. + */ +struct SlottedRWLockCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + RWLock* rwlocks; ///< primitives + size_t slotnum; ///< number of slots +#else + ::pthread_rwlock_t* rwlocks; ///< primitives + size_t slotnum; ///< number of slots +#endif +}; + + +/** + * Constructor. + */ +SlottedRWLock::SlottedRWLock(size_t slotnum) : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedRWLockCore* core = new SlottedRWLockCore; + RWLock* rwlocks = new RWLock[slotnum]; + core->rwlocks = rwlocks; + core->slotnum = slotnum; + opq_ = (void*)core; +#else + _assert_(true); + SlottedRWLockCore* core = new SlottedRWLockCore; + ::pthread_rwlock_t* rwlocks = new ::pthread_rwlock_t[slotnum]; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_rwlock_init(rwlocks + i, NULL) != 0) + throw std::runtime_error("pthread_rwlock_init"); + } + core->rwlocks = rwlocks; + core->slotnum = slotnum; + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +SlottedRWLock::~SlottedRWLock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + delete[] core->rwlocks; + delete core; +#else + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + ::pthread_rwlock_t* rwlocks = core->rwlocks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + ::pthread_rwlock_destroy(rwlocks + i); + } + delete[] rwlocks; + delete core; +#endif +} + + +/** + * Get the writer lock of a slot. + */ +void SlottedRWLock::lock_writer(size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + core->rwlocks[idx].lock_writer(); +#else + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + if (::pthread_rwlock_wrlock(core->rwlocks + idx) != 0) + throw std::runtime_error("pthread_rwlock_wrlock"); +#endif +} + + +/** + * Get the reader lock of a slot. + */ +void SlottedRWLock::lock_reader(size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + core->rwlocks[idx].lock_reader(); +#else + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + if (::pthread_rwlock_rdlock(core->rwlocks + idx) != 0) + throw std::runtime_error("pthread_rwlock_rdlock"); +#endif +} + + +/** + * Release the lock of a slot. + */ +void SlottedRWLock::unlock(size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + core->rwlocks[idx].unlock(); +#else + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + if (::pthread_rwlock_unlock(core->rwlocks + idx) != 0) + throw std::runtime_error("pthread_rwlock_unlock"); +#endif +} + + +/** + * Get the writer locks of all slots. + */ +void SlottedRWLock::lock_writer_all() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + RWLock* rwlocks = core->rwlocks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + rwlocks[i].lock_writer(); + } +#else + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + ::pthread_rwlock_t* rwlocks = core->rwlocks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_rwlock_wrlock(rwlocks + i) != 0) + throw std::runtime_error("pthread_rwlock_wrlock"); + } +#endif +} + + +/** + * Get the reader locks of all slots. + */ +void SlottedRWLock::lock_reader_all() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + RWLock* rwlocks = core->rwlocks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + rwlocks[i].lock_reader(); + } +#else + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + ::pthread_rwlock_t* rwlocks = core->rwlocks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_rwlock_rdlock(rwlocks + i) != 0) + throw std::runtime_error("pthread_rwlock_rdlock"); + } +#endif +} + + +/** + * Release the locks of all slots. + */ +void SlottedRWLock::unlock_all() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + RWLock* rwlocks = core->rwlocks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + rwlocks[i].unlock(); + } +#else + _assert_(true); + SlottedRWLockCore* core = (SlottedRWLockCore*)opq_; + ::pthread_rwlock_t* rwlocks = core->rwlocks; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + if (::pthread_rwlock_unlock(rwlocks + i) != 0) + throw std::runtime_error("pthread_rwlock_unlock"); + } +#endif +} + + +/** + * SpinRWLock internal. + */ +struct SpinRWLockCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + LONG sem; ///< semaphore + uint32_t cnt; ///< count of threads +#elif _KC_GCCATOMIC + int32_t sem; ///< semaphore + uint32_t cnt; ///< count of threads +#else + ::pthread_spinlock_t sem; ///< semaphore + uint32_t cnt; ///< count of threads +#endif +}; + + +/** + * Lock the semephore of SpinRWLock. + * @param core the internal fields. + */ +static void spinrwlocklock(SpinRWLockCore* core); + + +/** + * Unlock the semephore of SpinRWLock. + * @param core the internal fields. + */ +static void spinrwlockunlock(SpinRWLockCore* core); + + +/** + * Default constructor. + */ +SpinRWLock::SpinRWLock() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || _KC_GCCATOMIC + _assert_(true); + SpinRWLockCore* core = new SpinRWLockCore; + core->sem = 0; + core->cnt = 0; + opq_ = (void*)core; +#else + _assert_(true); + SpinRWLockCore* core = new SpinRWLockCore; + if (::pthread_spin_init(&core->sem, PTHREAD_PROCESS_PRIVATE) != 0) + throw std::runtime_error("pthread_spin_init"); + core->cnt = 0; + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +SpinRWLock::~SpinRWLock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || _KC_GCCATOMIC + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + delete core; +#else + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + ::pthread_spin_destroy(&core->sem); + delete core; +#endif +} + + +/** + * Get the writer lock. + */ +void SpinRWLock::lock_writer() { + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + spinrwlocklock(core); + uint32_t wcnt = 0; + while (core->cnt > 0) { + spinrwlockunlock(core); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + spinrwlocklock(core); + } + core->cnt = INT32MAX; + spinrwlockunlock(core); +} + + +/** + * Try to get the writer lock. + */ +bool SpinRWLock::lock_writer_try() { + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + spinrwlocklock(core); + if (core->cnt > 0) { + spinrwlockunlock(core); + return false; + } + core->cnt = INT32MAX; + spinrwlockunlock(core); + return true; +} + + +/** + * Get a reader lock. + */ +void SpinRWLock::lock_reader() { + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + spinrwlocklock(core); + uint32_t wcnt = 0; + while (core->cnt >= (uint32_t)INT32MAX) { + spinrwlockunlock(core); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + spinrwlocklock(core); + } + core->cnt++; + spinrwlockunlock(core); +} + + +/** + * Try to get a reader lock. + */ +bool SpinRWLock::lock_reader_try() { + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + spinrwlocklock(core); + if (core->cnt >= (uint32_t)INT32MAX) { + spinrwlockunlock(core); + return false; + } + core->cnt++; + spinrwlockunlock(core); + return true; +} + + +/** + * Release the lock. + */ +void SpinRWLock::unlock() { + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + spinrwlocklock(core); + if (core->cnt >= (uint32_t)INT32MAX) { + core->cnt = 0; + } else { + core->cnt--; + } + spinrwlockunlock(core); +} + + +/** + * Promote a reader lock to the writer lock. + */ +bool SpinRWLock::promote() { + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + spinrwlocklock(core); + if (core->cnt > 1) { + spinrwlockunlock(core); + return false; + } + core->cnt = INT32MAX; + spinrwlockunlock(core); + return true; +} + + +/** + * Demote the writer lock to a reader lock. + */ +void SpinRWLock::demote() { + _assert_(true); + SpinRWLockCore* core = (SpinRWLockCore*)opq_; + spinrwlocklock(core); + core->cnt = 1; + spinrwlockunlock(core); +} + + +/** + * Lock the semephore of SpinRWLock. + */ +static void spinrwlocklock(SpinRWLockCore* core) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(core); + while (::InterlockedCompareExchange(&core->sem, 1, 0) != 0) { + ::Sleep(0); + } +#elif _KC_GCCATOMIC + _assert_(core); + while (!__sync_bool_compare_and_swap(&core->sem, 0, 1)) { + ::sched_yield(); + } +#else + _assert_(core); + if (::pthread_spin_lock(&core->sem) != 0) throw std::runtime_error("pthread_spin_lock"); +#endif +} + + +/** + * Unlock the semephore of SpinRWLock. + */ +static void spinrwlockunlock(SpinRWLockCore* core) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(core); + ::InterlockedExchange(&core->sem, 0); +#elif _KC_GCCATOMIC + _assert_(core); + __sync_lock_release(&core->sem); +#else + _assert_(core); + if (::pthread_spin_unlock(&core->sem) != 0) throw std::runtime_error("pthread_spin_unlock"); +#endif +} + + +/** + * SlottedRWLock internal. + */ +struct SlottedSpinRWLockCore { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + LONG sems[LOCKSEMNUM]; ///< semaphores +#elif _KC_GCCATOMIC + int32_t sems[LOCKSEMNUM]; ///< semaphores +#else + ::pthread_spinlock_t sems[LOCKSEMNUM]; ///< semaphores +#endif + uint32_t* cnts; ///< counts of threads + size_t slotnum; ///< number of slots +}; + + +/** + * Lock the semephore of SlottedSpinRWLock. + * @param core the internal fields. + * @param idx the index of the semaphoe. + */ +static void slottedspinrwlocklock(SlottedSpinRWLockCore* core, size_t idx); + + +/** + * Unlock the semephore of SlottedSpinRWLock. + * @param core the internal fields. + * @param idx the index of the semaphoe. + */ +static void slottedspinrwlockunlock(SlottedSpinRWLockCore* core, size_t idx); + + +/** + * Constructor. + */ +SlottedSpinRWLock::SlottedSpinRWLock(size_t slotnum) : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + SlottedSpinRWLockCore* core = new SlottedSpinRWLockCore; + LONG* sems = core->sems; + uint32_t* cnts = new uint32_t[slotnum]; + for (size_t i = 0; i < LOCKSEMNUM; i++) { + sems[i] = 0; + } + for (size_t i = 0; i < slotnum; i++) { + cnts[i] = 0; + } + core->cnts = cnts; + core->slotnum = slotnum; + opq_ = (void*)core; +#elif _KC_GCCATOMIC + SlottedSpinRWLockCore* core = new SlottedSpinRWLockCore; + int32_t* sems = core->sems; + uint32_t* cnts = new uint32_t[slotnum]; + for (size_t i = 0; i < LOCKSEMNUM; i++) { + sems[i] = 0; + } + for (size_t i = 0; i < slotnum; i++) { + cnts[i] = 0; + } + core->cnts = cnts; + core->slotnum = slotnum; + opq_ = (void*)core; +#else + _assert_(true); + SlottedSpinRWLockCore* core = new SlottedSpinRWLockCore; + ::pthread_spinlock_t* sems = core->sems; + uint32_t* cnts = new uint32_t[slotnum]; + for (size_t i = 0; i < LOCKSEMNUM; i++) { + if (::pthread_spin_init(sems + i, PTHREAD_PROCESS_PRIVATE) != 0) + throw std::runtime_error("pthread_spin_init"); + } + for (size_t i = 0; i < slotnum; i++) { + cnts[i] = 0; + } + core->cnts = cnts; + core->slotnum = slotnum; + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +SlottedSpinRWLock::~SlottedSpinRWLock() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) || _KC_GCCATOMIC + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + delete[] core->cnts; + delete core; +#else + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + ::pthread_spinlock_t* sems = core->sems; + for (size_t i = 0; i < LOCKSEMNUM; i++) { + ::pthread_spin_destroy(sems + i); + } + delete[] core->cnts; + delete core; +#endif +} + + +/** + * Get the writer lock of a slot. + */ +void SlottedSpinRWLock::lock_writer(size_t idx) { + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + size_t semidx = idx % LOCKSEMNUM; + slottedspinrwlocklock(core, semidx); + uint32_t wcnt = 0; + while (core->cnts[idx] > 0) { + slottedspinrwlockunlock(core, semidx); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + slottedspinrwlocklock(core, semidx); + } + core->cnts[idx] = INT32MAX; + slottedspinrwlockunlock(core, semidx); +} + + +/** + * Get the reader lock of a slot. + */ +void SlottedSpinRWLock::lock_reader(size_t idx) { + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + size_t semidx = idx % LOCKSEMNUM; + slottedspinrwlocklock(core, semidx); + uint32_t wcnt = 0; + while (core->cnts[idx] >= (uint32_t)INT32MAX) { + slottedspinrwlockunlock(core, semidx); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + slottedspinrwlocklock(core, semidx); + } + core->cnts[idx]++; + slottedspinrwlockunlock(core, semidx); +} + + +/** + * Release the lock of a slot. + */ +void SlottedSpinRWLock::unlock(size_t idx) { + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + size_t semidx = idx % LOCKSEMNUM; + slottedspinrwlocklock(core, semidx); + if (core->cnts[idx] >= (uint32_t)INT32MAX) { + core->cnts[idx] = 0; + } else { + core->cnts[idx]--; + } + slottedspinrwlockunlock(core, semidx); +} + + +/** + * Get the writer locks of all slots. + */ +void SlottedSpinRWLock::lock_writer_all() { + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + uint32_t* cnts = core->cnts; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + size_t semidx = i % LOCKSEMNUM; + slottedspinrwlocklock(core, semidx); + uint32_t wcnt = 0; + while (cnts[i] > 0) { + slottedspinrwlockunlock(core, semidx); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + slottedspinrwlocklock(core, semidx); + } + cnts[i] = INT32MAX; + slottedspinrwlockunlock(core, semidx); + } +} + + +/** + * Get the reader locks of all slots. + */ +void SlottedSpinRWLock::lock_reader_all() { + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + uint32_t* cnts = core->cnts; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + size_t semidx = i % LOCKSEMNUM; + slottedspinrwlocklock(core, semidx); + uint32_t wcnt = 0; + while (cnts[i] >= (uint32_t)INT32MAX) { + slottedspinrwlockunlock(core, semidx); + if (wcnt >= LOCKBUSYLOOP) { + Thread::chill(); + } else { + Thread::yield(); + wcnt++; + } + slottedspinrwlocklock(core, semidx); + } + cnts[i]++; + slottedspinrwlockunlock(core, semidx); + } +} + + +/** + * Release the locks of all slots. + */ +void SlottedSpinRWLock::unlock_all() { + _assert_(true); + SlottedSpinRWLockCore* core = (SlottedSpinRWLockCore*)opq_; + uint32_t* cnts = core->cnts; + size_t slotnum = core->slotnum; + for (size_t i = 0; i < slotnum; i++) { + size_t semidx = i % LOCKSEMNUM; + slottedspinrwlocklock(core, semidx); + if (cnts[i] >= (uint32_t)INT32MAX) { + cnts[i] = 0; + } else { + cnts[i]--; + } + slottedspinrwlockunlock(core, semidx); + } +} + + +/** + * Lock the semephore of SlottedSpinRWLock. + */ +static void slottedspinrwlocklock(SlottedSpinRWLockCore* core, size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(core); + while (::InterlockedCompareExchange(core->sems + idx, 1, 0) != 0) { + ::Sleep(0); + } +#elif _KC_GCCATOMIC + _assert_(core); + while (!__sync_bool_compare_and_swap(core->sems + idx, 0, 1)) { + ::sched_yield(); + } +#else + _assert_(core); + if (::pthread_spin_lock(core->sems + idx) != 0) throw std::runtime_error("pthread_spin_lock"); +#endif +} + + +/** + * Unlock the semephore of SlottedSpinRWLock. + */ +static void slottedspinrwlockunlock(SlottedSpinRWLockCore* core, size_t idx) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(core); + ::InterlockedExchange(core->sems + idx, 0); +#elif _KC_GCCATOMIC + _assert_(core); + __sync_lock_release(core->sems + idx); +#else + _assert_(core); + if (::pthread_spin_unlock(core->sems + idx) != 0) + throw std::runtime_error("pthread_spin_unlock"); +#endif +} + + +/** + * Default constructor. + */ +CondVar::CondVar() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + CondVarCore* core = new CondVarCore; + ::InitializeCriticalSection(&core->mutex); + core->wait = 0; + core->wake = 0; + core->sev = ::CreateEvent(NULL, true, false, NULL); + if (!core->sev) throw std::runtime_error("CreateEvent"); + core->fev = ::CreateEvent(NULL, false, false, NULL); + if (!core->fev) throw std::runtime_error("CreateEvent"); + opq_ = (void*)core; +#else + _assert_(true); + CondVarCore* core = new CondVarCore; + if (::pthread_cond_init(&core->cond, NULL) != 0) + throw std::runtime_error("pthread_cond_init"); + opq_ = (void*)core; +#endif +} + + +/** + * Destructor. + */ +CondVar::~CondVar() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + CondVarCore* core = (CondVarCore*)opq_; + ::CloseHandle(core->fev); + ::CloseHandle(core->sev); + ::DeleteCriticalSection(&core->mutex); +#else + _assert_(true); + CondVarCore* core = (CondVarCore*)opq_; + ::pthread_cond_destroy(&core->cond); + delete core; +#endif +} + + +/** + * Wait for the signal. + */ +void CondVar::wait(Mutex* mutex) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(mutex); + CondVarCore* core = (CondVarCore*)opq_; + ::CRITICAL_SECTION* mymutex = (::CRITICAL_SECTION*)mutex->opq_; + ::EnterCriticalSection(&core->mutex); + core->wait++; + ::LeaveCriticalSection(&core->mutex); + ::LeaveCriticalSection(mymutex); + while (true) { + ::WaitForSingleObject(core->sev, INFINITE); + ::EnterCriticalSection(&core->mutex); + if (core->wake > 0) { + core->wait--; + core->wake--; + if (core->wake < 1) { + ::ResetEvent(core->sev); + ::SetEvent(core->fev); + } + ::LeaveCriticalSection(&core->mutex); + break; + } + ::LeaveCriticalSection(&core->mutex); + } + ::EnterCriticalSection(mymutex); +#else + _assert_(mutex); + CondVarCore* core = (CondVarCore*)opq_; + ::pthread_mutex_t* mymutex = (::pthread_mutex_t*)mutex->opq_; + if (::pthread_cond_wait(&core->cond, mymutex) != 0) + throw std::runtime_error("pthread_cond_wait"); +#endif +} + + +/** + * Wait for the signal. + */ +bool CondVar::wait(Mutex* mutex, double sec) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(mutex && sec >= 0); + if (sec <= 0) return false; + CondVarCore* core = (CondVarCore*)opq_; + ::CRITICAL_SECTION* mymutex = (::CRITICAL_SECTION*)mutex->opq_; + ::EnterCriticalSection(&core->mutex); + core->wait++; + ::LeaveCriticalSection(&core->mutex); + ::LeaveCriticalSection(mymutex); + while (true) { + if (::WaitForSingleObject(core->sev, sec * 1000) == WAIT_TIMEOUT) { + ::EnterCriticalSection(&core->mutex); + if (::WaitForSingleObject(core->sev, 0) == WAIT_TIMEOUT) { + core->wait--; + ::SetEvent(core->fev); + ::LeaveCriticalSection(&core->mutex); + ::EnterCriticalSection(mymutex); + return false; + } + ::LeaveCriticalSection(&core->mutex); + } + ::EnterCriticalSection(&core->mutex); + if (core->wake > 0) { + core->wait--; + core->wake--; + if (core->wake < 1) { + ::ResetEvent(core->sev); + ::SetEvent(core->fev); + } + ::LeaveCriticalSection(&core->mutex); + break; + } + ::LeaveCriticalSection(&core->mutex); + } + ::EnterCriticalSection(mymutex); + return true; +#else + _assert_(mutex && sec >= 0); + if (sec <= 0) return false; + CondVarCore* core = (CondVarCore*)opq_; + ::pthread_mutex_t* mymutex = (::pthread_mutex_t*)mutex->opq_; + struct ::timeval tv; + struct ::timespec ts; + if (::gettimeofday(&tv, NULL) == 0) { + double integ; + double fract = std::modf(sec, &integ); + ts.tv_sec = tv.tv_sec + (time_t)integ; + ts.tv_nsec = (long)(tv.tv_usec * 1000.0 + fract * 999999000); + if (ts.tv_nsec >= 1000000000) { + ts.tv_nsec -= 1000000000; + ts.tv_sec++; + } + } else { + ts.tv_sec = std::time(NULL) + 1; + ts.tv_nsec = 0; + } + int32_t ecode = ::pthread_cond_timedwait(&core->cond, mymutex, &ts); + if (ecode == 0) return true; + if (ecode != ETIMEDOUT && ecode != EINTR) + throw std::runtime_error("pthread_cond_timedwait"); + return false; +#endif +} + + +/** + * Send the wake-up signal to another waiting thread. + */ +void CondVar::signal() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + CondVarCore* core = (CondVarCore*)opq_; + ::EnterCriticalSection(&core->mutex); + if (core->wait > 0) { + core->wake = 1; + ::SetEvent(core->sev); + ::LeaveCriticalSection(&core->mutex); + ::WaitForSingleObject(core->fev, INFINITE); + } else { + ::LeaveCriticalSection(&core->mutex); + } +#else + _assert_(true); + CondVarCore* core = (CondVarCore*)opq_; + if (::pthread_cond_signal(&core->cond) != 0) + throw std::runtime_error("pthread_cond_signal"); +#endif +} + + +/** + * Send the wake-up signals to all waiting threads. + */ +void CondVar::broadcast() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + CondVarCore* core = (CondVarCore*)opq_; + ::EnterCriticalSection(&core->mutex); + if (core->wait > 0) { + core->wake = core->wait; + ::SetEvent(core->sev); + ::LeaveCriticalSection(&core->mutex); + ::WaitForSingleObject(core->fev, INFINITE); + } else { + ::LeaveCriticalSection(&core->mutex); + } +#else + _assert_(true); + CondVarCore* core = (CondVarCore*)opq_; + if (::pthread_cond_broadcast(&core->cond) != 0) + throw std::runtime_error("pthread_cond_broadcast"); +#endif +} + + +/** + * Default constructor. + */ +TSDKey::TSDKey() : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::DWORD key = ::TlsAlloc(); + if (key == 0xFFFFFFFF) throw std::runtime_error("TlsAlloc"); + opq_ = (void*)key; +#else + _assert_(true); + ::pthread_key_t* key = new ::pthread_key_t; + if (::pthread_key_create(key, NULL) != 0) + throw std::runtime_error("pthread_key_create"); + opq_ = (void*)key; +#endif +} + + +/** + * Constructor with the specifications. + */ +TSDKey::TSDKey(void (*dstr)(void*)) : opq_(NULL) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::DWORD key = ::TlsAlloc(); + if (key == 0xFFFFFFFF) throw std::runtime_error("TlsAlloc"); + opq_ = (void*)key; +#else + _assert_(true); + ::pthread_key_t* key = new ::pthread_key_t; + if (::pthread_key_create(key, dstr) != 0) + throw std::runtime_error("pthread_key_create"); + opq_ = (void*)key; +#endif +} + + +/** + * Destructor. + */ +TSDKey::~TSDKey() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::DWORD key = (::DWORD)opq_; + ::TlsFree(key); +#else + _assert_(true); + ::pthread_key_t* key = (::pthread_key_t*)opq_; + ::pthread_key_delete(*key); + delete key; +#endif +} + + +/** + * Set the value. + */ +void TSDKey::set(void* ptr) { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::DWORD key = (::DWORD)opq_; + if (!::TlsSetValue(key, ptr)) std::runtime_error("TlsSetValue"); +#else + _assert_(true); + ::pthread_key_t* key = (::pthread_key_t*)opq_; + if (::pthread_setspecific(*key, ptr) != 0) + throw std::runtime_error("pthread_setspecific"); +#endif +} + + +/** + * Get the value. + */ +void* TSDKey::get() const { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::DWORD key = (::DWORD)opq_; + return ::TlsGetValue(key); +#else + _assert_(true); + ::pthread_key_t* key = (::pthread_key_t*)opq_; + return ::pthread_getspecific(*key); +#endif +} + + +/** + * Set the new value. + */ +int64_t AtomicInt64::set(int64_t val) { +#if (defined(_SYS_MSVC_) || defined(_SYS_MINGW_)) && defined(_SYS_WIN64_) + _assert_(true); + return ::InterlockedExchange((uint64_t*)&value_, val); +#elif _KC_GCCATOMIC + _assert_(true); + int64_t oval = __sync_lock_test_and_set(&value_, val); + __sync_synchronize(); + return oval; +#else + _assert_(true); + lock_.lock(); + int64_t oval = value_; + value_ = val; + lock_.unlock(); + return oval; +#endif +} + + +/** + * Add a value. + */ +int64_t AtomicInt64::add(int64_t val) { +#if (defined(_SYS_MSVC_) || defined(_SYS_MINGW_)) && defined(_SYS_WIN64_) + _assert_(true); + return ::InterlockedExchangeAdd((uint64_t*)&value_, val); +#elif _KC_GCCATOMIC + _assert_(true); + int64_t oval = __sync_fetch_and_add(&value_, val); + __sync_synchronize(); + return oval; +#else + _assert_(true); + lock_.lock(); + int64_t oval = value_; + value_ += val; + lock_.unlock(); + return oval; +#endif +} + + +/** + * Perform compare-and-swap. + */ +bool AtomicInt64::cas(int64_t oval, int64_t nval) { +#if (defined(_SYS_MSVC_) || defined(_SYS_MINGW_)) && defined(_SYS_WIN64_) + _assert_(true); + return ::InterlockedCompareExchange((uint64_t*)&value_, nval, oval) == oval; +#elif _KC_GCCATOMIC + _assert_(true); + bool rv = __sync_bool_compare_and_swap(&value_, oval, nval); + __sync_synchronize(); + return rv; +#else + _assert_(true); + bool rv = false; + lock_.lock(); + if (value_ == oval) { + value_ = nval; + rv = true; + } + lock_.unlock(); + return rv; +#endif +} + + +/** + * Get the current value. + */ +int64_t AtomicInt64::get() const { +#if (defined(_SYS_MSVC_) || defined(_SYS_MINGW_)) && defined(_SYS_WIN64_) + _assert_(true); + return ::InterlockedExchangeAdd((uint64_t*)&value_, 0); +#elif _KC_GCCATOMIC + _assert_(true); + return __sync_fetch_and_add((int64_t*)&value_, 0); +#else + _assert_(true); + lock_.lock(); + int64_t oval = value_; + lock_.unlock(); + return oval; +#endif +} + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcthread.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcthread.h new file mode 100644 index 0000000000..7b77bbd909 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcthread.h @@ -0,0 +1,1302 @@ +/************************************************************************************************* + * Threading devices + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCTHREAD_H // duplication check +#define _KCTHREAD_H + +#include <kccommon.h> +#include <kcutil.h> + +namespace kyotocabinet { // common namespace + + +/** + * Threading device. + */ +class Thread { + public: + /** + * Default constructor. + */ + explicit Thread(); + /** + * Destructor. + */ + virtual ~Thread(); + /** + * Perform the concrete process. + */ + virtual void run() = 0; + /** + * Start the thread. + */ + void start(); + /** + * Wait for the thread to finish. + */ + void join(); + /** + * Put the thread in the detached state. + */ + void detach(); + /** + * Terminate the running thread. + */ + static void exit(); + /** + * Yield the processor from the current thread. + */ + static void yield(); + /** + * Chill the processor by suspending execution for a quick moment. + */ + static void chill(); + /** + * Suspend execution of the current thread. + * @param sec the interval of the suspension in seconds. + * @return true on success, or false on failure. + */ + static bool sleep(double sec); + /** + * Get the hash value of the current thread. + * @return the hash value of the current thread. + */ + static int64_t hash(); + private: + /** Dummy constructor to forbid the use. */ + Thread(const Thread&); + /** Dummy Operator to forbid the use. */ + Thread& operator =(const Thread&); + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Basic mutual exclusion device. + */ +class Mutex { + friend class CondVar; + public: + /** + * Type of the behavior for double locking. + */ + enum Type { + FAST, ///< no operation + ERRORCHECK, ///< check error + RECURSIVE ///< allow recursive locking + }; + /** + * Default constructor. + */ + explicit Mutex(); + /** + * Constructor. + * @param type the behavior for double locking. + */ + explicit Mutex(Type type); + /** + * Destructor. + */ + ~Mutex(); + /** + * Get the lock. + */ + void lock(); + /** + * Try to get the lock. + * @return true on success, or false on failure. + */ + bool lock_try(); + /** + * Try to get the lock. + * @param sec the interval of the suspension in seconds. + * @return true on success, or false on failure. + */ + bool lock_try(double sec); + /** + * Release the lock. + */ + void unlock(); + private: + /** Dummy constructor to forbid the use. */ + Mutex(const Mutex&); + /** Dummy Operator to forbid the use. */ + Mutex& operator =(const Mutex&); + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Scoped mutex device. + */ +class ScopedMutex { + public: + /** + * Constructor. + * @param mutex a mutex to lock the block. + */ + explicit ScopedMutex(Mutex* mutex) : mutex_(mutex) { + _assert_(mutex); + mutex_->lock(); + } + /** + * Destructor. + */ + ~ScopedMutex() { + _assert_(true); + mutex_->unlock(); + } + private: + /** Dummy constructor to forbid the use. */ + ScopedMutex(const ScopedMutex&); + /** Dummy Operator to forbid the use. */ + ScopedMutex& operator =(const ScopedMutex&); + /** The inner device. */ + Mutex* mutex_; +}; + + +/** + * Slotted mutex device. + */ +class SlottedMutex { + public: + /** + * Constructor. + * @param slotnum the number of slots. + */ + explicit SlottedMutex(size_t slotnum); + /** + * Destructor. + */ + ~SlottedMutex(); + /** + * Get the lock of a slot. + * @param idx the index of a slot. + */ + void lock(size_t idx); + /** + * Release the lock of a slot. + * @param idx the index of a slot. + */ + void unlock(size_t idx); + /** + * Get the locks of all slots. + */ + void lock_all(); + /** + * Release the locks of all slots. + */ + void unlock_all(); + private: + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Lightweight mutual exclusion device. + */ +class SpinLock { + public: + /** + * Default constructor. + */ + explicit SpinLock(); + /** + * Destructor. + */ + ~SpinLock(); + /** + * Get the lock. + */ + void lock(); + /** + * Try to get the lock. + * @return true on success, or false on failure. + */ + bool lock_try(); + /** + * Release the lock. + */ + void unlock(); + private: + /** Dummy constructor to forbid the use. */ + SpinLock(const SpinLock&); + /** Dummy Operator to forbid the use. */ + SpinLock& operator =(const SpinLock&); + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Scoped spin lock device. + */ +class ScopedSpinLock { + public: + /** + * Constructor. + * @param spinlock a spin lock to lock the block. + */ + explicit ScopedSpinLock(SpinLock* spinlock) : spinlock_(spinlock) { + _assert_(spinlock); + spinlock_->lock(); + } + /** + * Destructor. + */ + ~ScopedSpinLock() { + _assert_(true); + spinlock_->unlock(); + } + private: + /** Dummy constructor to forbid the use. */ + ScopedSpinLock(const ScopedSpinLock&); + /** Dummy Operator to forbid the use. */ + ScopedSpinLock& operator =(const ScopedSpinLock&); + /** The inner device. */ + SpinLock* spinlock_; +}; + + +/** + * Slotted spin lock devices. + */ +class SlottedSpinLock { + public: + /** + * Constructor. + * @param slotnum the number of slots. + */ + explicit SlottedSpinLock(size_t slotnum); + /** + * Destructor. + */ + ~SlottedSpinLock(); + /** + * Get the lock of a slot. + * @param idx the index of a slot. + */ + void lock(size_t idx); + /** + * Release the lock of a slot. + * @param idx the index of a slot. + */ + void unlock(size_t idx); + /** + * Get the locks of all slots. + */ + void lock_all(); + /** + * Release the locks of all slots. + */ + void unlock_all(); + private: + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Reader-writer locking device. + */ +class RWLock { + public: + /** + * Default constructor. + */ + explicit RWLock(); + /** + * Destructor. + */ + ~RWLock(); + /** + * Get the writer lock. + */ + void lock_writer(); + /** + * Try to get the writer lock. + * @return true on success, or false on failure. + */ + bool lock_writer_try(); + /** + * Get a reader lock. + */ + void lock_reader(); + /** + * Try to get a reader lock. + * @return true on success, or false on failure. + */ + bool lock_reader_try(); + /** + * Release the lock. + */ + void unlock(); + private: + /** Dummy constructor to forbid the use. */ + RWLock(const RWLock&); + /** Dummy Operator to forbid the use. */ + RWLock& operator =(const RWLock&); + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Scoped reader-writer locking device. + */ +class ScopedRWLock { + public: + /** + * Constructor. + * @param rwlock a rwlock to lock the block. + * @param writer true for writer lock, or false for reader lock. + */ + explicit ScopedRWLock(RWLock* rwlock, bool writer) : rwlock_(rwlock) { + _assert_(rwlock); + if (writer) { + rwlock_->lock_writer(); + } else { + rwlock_->lock_reader(); + } + } + /** + * Destructor. + */ + ~ScopedRWLock() { + _assert_(true); + rwlock_->unlock(); + } + private: + /** Dummy constructor to forbid the use. */ + ScopedRWLock(const ScopedRWLock&); + /** Dummy Operator to forbid the use. */ + ScopedRWLock& operator =(const ScopedRWLock&); + /** The inner device. */ + RWLock* rwlock_; +}; + + +/** + * Slotted reader-writer lock devices. + */ +class SlottedRWLock { + public: + /** + * Constructor. + * @param slotnum the number of slots. + */ + explicit SlottedRWLock(size_t slotnum); + /** + * Destructor. + */ + ~SlottedRWLock(); + /** + * Get the writer lock of a slot. + * @param idx the index of a slot. + */ + void lock_writer(size_t idx); + /** + * Get the reader lock of a slot. + * @param idx the index of a slot. + */ + void lock_reader(size_t idx); + /** + * Release the lock of a slot. + * @param idx the index of a slot. + */ + void unlock(size_t idx); + /** + * Get the writer locks of all slots. + */ + void lock_writer_all(); + /** + * Get the reader locks of all slots. + */ + void lock_reader_all(); + /** + * Release the locks of all slots. + */ + void unlock_all(); + private: + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Lightweight reader-writer locking device. + */ +class SpinRWLock { + public: + /** + * Default constructor. + */ + explicit SpinRWLock(); + /** + * Destructor. + */ + ~SpinRWLock(); + /** + * Get the writer lock. + */ + void lock_writer(); + /** + * Try to get the writer lock. + * @return true on success, or false on failure. + */ + bool lock_writer_try(); + /** + * Get a reader lock. + */ + void lock_reader(); + /** + * Try to get a reader lock. + * @return true on success, or false on failure. + */ + bool lock_reader_try(); + /** + * Release the lock. + */ + void unlock(); + /** + * Promote a reader lock to the writer lock. + * @return true on success, or false on failure. + */ + bool promote(); + /** + * Demote the writer lock to a reader lock. + */ + void demote(); + private: + /** Dummy constructor to forbid the use. */ + SpinRWLock(const SpinRWLock&); + /** Dummy Operator to forbid the use. */ + SpinRWLock& operator =(const SpinRWLock&); + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Scoped reader-writer locking device. + */ +class ScopedSpinRWLock { + public: + /** + * Constructor. + * @param srwlock a spin rwlock to lock the block. + * @param writer true for writer lock, or false for reader lock. + */ + explicit ScopedSpinRWLock(SpinRWLock* srwlock, bool writer) : srwlock_(srwlock) { + _assert_(srwlock); + if (writer) { + srwlock_->lock_writer(); + } else { + srwlock_->lock_reader(); + } + } + /** + * Destructor. + */ + ~ScopedSpinRWLock() { + _assert_(true); + srwlock_->unlock(); + } + private: + /** Dummy constructor to forbid the use. */ + ScopedSpinRWLock(const ScopedSpinRWLock&); + /** Dummy Operator to forbid the use. */ + ScopedSpinRWLock& operator =(const ScopedSpinRWLock&); + /** The inner device. */ + SpinRWLock* srwlock_; +}; + + +/** + * Slotted lightweight reader-writer lock devices. + */ +class SlottedSpinRWLock { + public: + /** + * Constructor. + * @param slotnum the number of slots. + */ + explicit SlottedSpinRWLock(size_t slotnum); + /** + * Destructor. + */ + ~SlottedSpinRWLock(); + /** + * Get the writer lock of a slot. + * @param idx the index of a slot. + */ + void lock_writer(size_t idx); + /** + * Get the reader lock of a slot. + * @param idx the index of a slot. + */ + void lock_reader(size_t idx); + /** + * Release the lock of a slot. + * @param idx the index of a slot. + */ + void unlock(size_t idx); + /** + * Get the writer locks of all slots. + */ + void lock_writer_all(); + /** + * Get the reader locks of all slots. + */ + void lock_reader_all(); + /** + * Release the locks of all slots. + */ + void unlock_all(); + private: + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Condition variable. + */ +class CondVar { + public: + /** + * Default constructor. + */ + explicit CondVar(); + /** + * Destructor. + */ + ~CondVar(); + /** + * Wait for the signal. + * @param mutex a locked mutex. + */ + void wait(Mutex* mutex); + /** + * Wait for the signal. + * @param mutex a locked mutex. + * @param sec the interval of the suspension in seconds. + * @return true on catched signal, or false on timeout. + */ + bool wait(Mutex* mutex, double sec); + /** + * Send the wake-up signal to another waiting thread. + * @note The mutex used for the wait method should be locked by the caller. + */ + void signal(); + /** + * Send the wake-up signals to all waiting threads. + * @note The mutex used for the wait method should be locked by the caller. + */ + void broadcast(); + private: + /** Dummy constructor to forbid the use. */ + CondVar(const CondVar&); + /** Dummy Operator to forbid the use. */ + CondVar& operator =(const CondVar&); + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Assosiative condition variable. + */ +class CondMap { + private: + struct Count; + struct Slot; + /** An alias of set of counters. */ + typedef std::map<std::string, Count> CountMap; + /** The number of slots. */ + static const size_t SLOTNUM = 64; + public: + /** + * Default constructor. + */ + explicit CondMap() : slots_() { + _assert_(true); + } + /** + * Destructor. + */ + ~CondMap() { + _assert_(true); + } + /** + * Wait for a signal. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @param sec the interval of the suspension in seconds. If it is negative, no timeout is + * specified. + * @return true on catched signal, or false on timeout. + */ + bool wait(const char* kbuf, size_t ksiz, double sec = -1) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + std::string key(kbuf, ksiz); + return wait(key, sec); + } + /** + * Wait for a signal by a key. + * @param key the key. + * @param sec the interval of the suspension in seconds. If it is negative, no timeout is + * specified. + * @return true on catched signal, or false on timeout. + */ + bool wait(const std::string& key, double sec = -1) { + _assert_(true); + double invtime = sec < 0 ? 1.0 : sec; + double curtime = time(); + double endtime = curtime + (sec < 0 ? UINT32MAX : sec); + Slot* slot = get_slot(key); + while (curtime < endtime) { + ScopedMutex lock(&slot->mutex); + CountMap::iterator cit = slot->counter.find(key); + if (cit == slot->counter.end()) { + Count cnt = { 1, false }; + slot->counter[key] = cnt; + } else { + cit->second.num++; + } + slot->cond.wait(&slot->mutex, invtime); + cit = slot->counter.find(key); + cit->second.num--; + if (cit->second.wake > 0) { + cit->second.wake--; + if (cit->second.num < 1) slot->counter.erase(cit); + return true; + } + if (cit->second.num < 1) slot->counter.erase(cit); + curtime = time(); + } + return false; + } + /** + * Send a wake-up signal to another thread waiting by a key. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the number of threads waiting for the signal. + */ + size_t signal(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + std::string key(kbuf, ksiz); + return signal(key); + } + /** + * Send a wake-up signal to another thread waiting by a key. + * @param key the key. + * @return the number of threads waiting for the signal. + */ + size_t signal(const std::string& key) { + _assert_(true); + Slot* slot = get_slot(key); + ScopedMutex lock(&slot->mutex); + CountMap::iterator cit = slot->counter.find(key); + if (cit == slot->counter.end() || cit->second.num < 1) return 0; + if (cit->second.wake < cit->second.num) cit->second.wake++; + slot->cond.broadcast(); + return cit->second.num; + } + /** + * Send wake-up signals to all threads waiting by a key. + * @param kbuf the pointer to the key region. + * @param ksiz the size of the key region. + * @return the number of threads waiting for the signal. + */ + size_t broadcast(const char* kbuf, size_t ksiz) { + _assert_(kbuf && ksiz <= MEMMAXSIZ); + std::string key(kbuf, ksiz); + return broadcast(key); + } + /** + * Send wake-up signals to all threads waiting by a key. + * @param key the key. + * @return the number of threads waiting for the signal. + */ + size_t broadcast(const std::string& key) { + _assert_(true); + Slot* slot = get_slot(key); + ScopedMutex lock(&slot->mutex); + CountMap::iterator cit = slot->counter.find(key); + if (cit == slot->counter.end() || cit->second.num < 1) return 0; + cit->second.wake = cit->second.num; + slot->cond.broadcast(); + return cit->second.num; + } + /** + * Send wake-up signals to all threads waiting by each key. + * @return the number of threads waiting for the signal. + */ + size_t broadcast_all() { + _assert_(true); + size_t sum = 0; + for (size_t i = 0; i < SLOTNUM; i++) { + Slot* slot = slots_ + i; + ScopedMutex lock(&slot->mutex); + CountMap::iterator cit = slot->counter.begin(); + CountMap::iterator citend = slot->counter.end(); + while (cit != citend) { + if (cit->second.num > 0) { + cit->second.wake = cit->second.num; + sum += cit->second.num; + } + slot->cond.broadcast(); + ++cit; + } + } + return sum; + } + /** + * Get the total number of threads waiting for signals. + * @return the total number of threads waiting for signals. + */ + size_t count() { + _assert_(true); + size_t sum = 0; + for (size_t i = 0; i < SLOTNUM; i++) { + Slot* slot = slots_ + i; + ScopedMutex lock(&slot->mutex); + CountMap::iterator cit = slot->counter.begin(); + CountMap::iterator citend = slot->counter.end(); + while (cit != citend) { + sum += cit->second.num; + ++cit; + } + } + return sum; + } + private: + /** + * Counter for waiting threads. + */ + struct Count { + size_t num; ///< waiting threads + size_t wake; ///< waking threads + }; + /** + * Slot of a key space. + */ + struct Slot { + CondVar cond; ///< condition variable + Mutex mutex; ///< mutex + CountMap counter; ///< counter + }; + /** + * Get the slot corresponding a key. + * @param key the key. + * @return the slot corresponding the key. + */ + Slot* get_slot(const std::string& key) { + return slots_ + hashmurmur(key.data(), key.size()) % SLOTNUM; + } + /** The slot array. */ + Slot slots_[SLOTNUM]; +}; + + +/** + * Key of thread specific data. + */ +class TSDKey { + public: + /** + * Default constructor. + */ + explicit TSDKey(); + /** + * Constructor. + * @param dstr the destructor for the value. + */ + explicit TSDKey(void (*dstr)(void*)); + /** + * Destructor. + */ + ~TSDKey(); + /** + * Set the value. + * @param ptr an arbitrary pointer. + */ + void set(void* ptr); + /** + * Get the value. + * @return the value. + */ + void* get() const ; + private: + /** Opaque pointer. */ + void* opq_; +}; + + +/** + * Smart pointer to thread specific data. + */ +template <class TYPE> +class TSD { + public: + /** + * Default constructor. + */ + explicit TSD() : key_(delete_value) { + _assert_(true); + } + /** + * Destructor. + */ + ~TSD() { + _assert_(true); + TYPE* obj = (TYPE*)key_.get(); + if (obj) { + delete obj; + key_.set(NULL); + } + } + /** + * Dereference operator. + * @return the reference to the inner object. + */ + TYPE& operator *() { + _assert_(true); + TYPE* obj = (TYPE*)key_.get(); + if (!obj) { + obj = new TYPE; + key_.set(obj); + } + return *obj; + } + /** + * Member reference operator. + * @return the pointer to the inner object. + */ + TYPE* operator ->() { + _assert_(true); + TYPE* obj = (TYPE*)key_.get(); + if (!obj) { + obj = new TYPE; + key_.set(obj); + } + return obj; + } + /** + * Cast operator to the original type. + * @return the copy of the inner object. + */ + operator TYPE() const { + _assert_(true); + TYPE* obj = (TYPE*)key_.get(); + if (!obj) return TYPE(); + return *obj; + } + private: + /** + * Delete the inner object. + * @param obj the inner object. + */ + static void delete_value(void* obj) { + _assert_(true); + delete (TYPE*)obj; + } + /** Dummy constructor to forbid the use. */ + TSD(const TSD&); + /** Dummy Operator to forbid the use. */ + TSD& operator =(const TSD&); + /** Key of thread specific data. */ + TSDKey key_; +}; + + +/** + * Integer with atomic operations. + */ +class AtomicInt64 { + public: + /** + * Default constructor. + */ + explicit AtomicInt64() : value_(0), lock_() { + _assert_(true); + } + /** + * Copy constructor. + * @param src the source object. + */ + AtomicInt64(const AtomicInt64& src) : value_(src.get()), lock_() { + _assert_(true); + }; + /** + * Constructor. + * @param num the initial value. + */ + AtomicInt64(int64_t num) : value_(num), lock_() { + _assert_(true); + } + /** + * Destructor. + */ + ~AtomicInt64() { + _assert_(true); + } + /** + * Set the new value. + * @param val the new value. + * @return the old value. + */ + int64_t set(int64_t val); + /** + * Add a value. + * @param val the additional value. + * @return the old value. + */ + int64_t add(int64_t val); + /** + * Perform compare-and-swap. + * @param oval the old value. + * @param nval the new value. + * @return true on success, or false on failure. + */ + bool cas(int64_t oval, int64_t nval); + /** + * Get the current value. + * @return the current value. + */ + int64_t get() const; + /** + * Assignment operator from the self type. + * @param right the right operand. + * @return the reference to itself. + */ + AtomicInt64& operator =(const AtomicInt64& right) { + _assert_(true); + if (&right == this) return *this; + set(right.get()); + return *this; + } + /** + * Assignment operator from integer. + * @param right the right operand. + * @return the reference to itself. + */ + AtomicInt64& operator =(const int64_t& right) { + _assert_(true); + set(right); + return *this; + } + /** + * Cast operator to integer. + * @return the current value. + */ + operator int64_t() const { + _assert_(true); + return get(); + } + /** + * Summation assignment operator by integer. + * @param right the right operand. + * @return the reference to itself. + */ + AtomicInt64& operator +=(int64_t right) { + _assert_(true); + add(right); + return *this; + } + /** + * Subtraction assignment operator by integer. + * @param right the right operand. + * @return the reference to itself. + */ + AtomicInt64& operator -=(int64_t right) { + _assert_(true); + add(-right); + return *this; + } + /** + * Secure the least value + * @param val the least value + * @return the current value. + */ + int64_t secure_least(int64_t val) { + _assert_(true); + while (true) { + int64_t cur = get(); + if (cur >= val) return cur; + if (cas(cur, val)) break; + } + return val; + } + private: + /** The value. */ + volatile int64_t value_; + /** The alternative lock. */ + mutable SpinLock lock_; +}; + + +/** + * Task queue device. + */ +class TaskQueue { + public: + class Task; + private: + class WorkerThread; + /** An alias of list of tasks. */ + typedef std::list<Task*> TaskList; + public: + /** + * Interface of a task. + */ + class Task { + friend class TaskQueue; + public: + /** + * Default constructor. + */ + explicit Task() : id_(0), thid_(0), aborted_(false) { + _assert_(true); + } + /** + * Destructor. + */ + virtual ~Task() { + _assert_(true); + } + /** + * Get the ID number of the task. + * @return the ID number of the task, which is incremented from 1. + */ + uint64_t id() const { + _assert_(true); + return id_; + } + /** + * Get the ID number of the worker thread. + * @return the ID number of the worker thread. It is from 0 to less than the number of + * worker threads. + */ + uint32_t thread_id() const { + _assert_(true); + return thid_; + } + /** + * Check whether the thread is to be aborted. + * @return true if the thread is to be aborted, or false if not. + */ + bool aborted() const { + _assert_(true); + return aborted_; + } + private: + /** The task ID number. */ + uint64_t id_; + /** The thread ID number. */ + uint64_t thid_; + /** The flag to be aborted. */ + bool aborted_; + }; + /** + * Default Constructor. + */ + TaskQueue() : thary_(NULL), thnum_(0), tasks_(), count_(0), mutex_(), cond_(), seed_(0) { + _assert_(true); + } + /** + * Destructor. + */ + virtual ~TaskQueue() { + _assert_(true); + } + /** + * Process a task. + * @param task a task object. + */ + virtual void do_task(Task* task) = 0; + /** + * Process the starting event. + * @param task a task object. + * @note This is called for each thread on starting. + */ + virtual void do_start(const Task* task) { + _assert_(true); + } + /** + * Process the finishing event. + * @param task a task object. + * @note This is called for each thread on finishing. + */ + virtual void do_finish(const Task* task) { + _assert_(true); + } + /** + * Start the task queue. + * @param thnum the number of worker threads. + */ + void start(size_t thnum) { + _assert_(thnum > 0 && thnum <= MEMMAXSIZ); + thary_ = new WorkerThread[thnum]; + for (size_t i = 0; i < thnum; i++) { + thary_[i].id_ = i; + thary_[i].queue_ = this; + thary_[i].start(); + } + thnum_ = thnum; + } + /** + * Finish the task queue. + * @note This function blocks until all tasks in the queue are popped. + */ + void finish() { + _assert_(true); + mutex_.lock(); + TaskList::iterator it = tasks_.begin(); + TaskList::iterator itend = tasks_.end(); + while (it != itend) { + Task* task = *it; + task->aborted_ = true; + ++it; + } + cond_.broadcast(); + mutex_.unlock(); + Thread::yield(); + for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) { + mutex_.lock(); + if (tasks_.empty()) { + mutex_.unlock(); + break; + } + mutex_.unlock(); + if (wsec > 1.0) wsec = 1.0; + Thread::sleep(wsec); + } + mutex_.lock(); + for (size_t i = 0; i < thnum_; i++) { + thary_[i].aborted_ = true; + } + cond_.broadcast(); + mutex_.unlock(); + for (size_t i = 0; i < thnum_; i++) { + thary_[i].join(); + } + delete[] thary_; + } + /** + * Add a task. + * @param task a task object. + * @return the number of tasks in the queue. + */ + int64_t add_task(Task* task) { + _assert_(task); + mutex_.lock(); + task->id_ = ++seed_; + tasks_.push_back(task); + int64_t count = ++count_; + cond_.signal(); + mutex_.unlock(); + return count; + } + /** + * Get the number of tasks in the queue. + * @return the number of tasks in the queue. + */ + int64_t count() { + _assert_(true); + mutex_.lock(); + int64_t count = count_; + mutex_.unlock(); + return count; + } + private: + /** + * Implementation of the worker thread. + */ + class WorkerThread : public Thread { + friend class TaskQueue; + public: + explicit WorkerThread() : id_(0), queue_(NULL), aborted_(false) { + _assert_(true); + } + private: + void run() { + _assert_(true); + Task* stask = new Task; + stask->thid_ = id_; + queue_->do_start(stask); + delete stask; + bool empty = false; + while (true) { + queue_->mutex_.lock(); + if (aborted_) { + queue_->mutex_.unlock(); + break; + } + if (empty) queue_->cond_.wait(&queue_->mutex_, 1.0); + Task * task = NULL; + if (queue_->tasks_.empty()) { + empty = true; + } else { + task = queue_->tasks_.front(); + task->thid_ = id_; + queue_->tasks_.pop_front(); + queue_->count_--; + empty = false; + } + queue_->mutex_.unlock(); + if (task) queue_->do_task(task); + } + Task* ftask = new Task; + ftask->thid_ = id_; + ftask->aborted_ = true; + queue_->do_finish(ftask); + delete ftask; + } + uint32_t id_; + TaskQueue* queue_; + Task* task_; + bool aborted_; + }; + /** Dummy constructor to forbid the use. */ + TaskQueue(const TaskQueue&); + /** Dummy Operator to forbid the use. */ + TaskQueue& operator =(const TaskQueue&); + /** The array of worker threads. */ + WorkerThread* thary_; + /** The number of worker threads. */ + size_t thnum_; + /** The list of tasks. */ + TaskList tasks_; + /** The number of the tasks. */ + int64_t count_; + /** The mutex for the task list. */ + Mutex mutex_; + /** The condition variable for the task list. */ + CondVar cond_; + /** The seed of ID numbers. */ + uint64_t seed_; +}; + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kctreemgr.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kctreemgr.cc new file mode 100644 index 0000000000..1278367360 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kctreemgr.cc @@ -0,0 +1,1589 @@ +/************************************************************************************************* + * The command line utility of the file tree database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kchashdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, const char* info); +static int32_t runcreate(int argc, char** argv); +static int32_t runinform(int argc, char** argv); +static int32_t runset(int argc, char** argv); +static int32_t runremove(int argc, char** argv); +static int32_t runget(int argc, char** argv); +static int32_t runlist(int argc, char** argv); +static int32_t runclear(int argc, char** argv); +static int32_t runimport(int argc, char** argv); +static int32_t runcopy(int argc, char** argv); +static int32_t rundump(int argc, char** argv); +static int32_t runload(int argc, char** argv); +static int32_t rundefrag(int argc, char** argv); +static int32_t runsetbulk(int argc, char** argv); +static int32_t runremovebulk(int argc, char** argv); +static int32_t rungetbulk(int argc, char** argv); +static int32_t runcheck(int argc, char** argv); +static int32_t proccreate(const char* path, int32_t oflags, int32_t apow, int32_t fpow, + int32_t opts, int64_t bnum, int32_t psiz, kc::Comparator* rcomp); +static int32_t procinform(const char* path, int32_t oflags, bool st); +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode); +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags); +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz); +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + bool des, int64_t max, bool rm, bool pv, bool px); +static int32_t procclear(const char* path, int32_t oflags); +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx); +static int32_t proccopy(const char* path, const char* file, int32_t oflags); +static int32_t procdump(const char* path, const char* file, int32_t oflags); +static int32_t procload(const char* path, const char* file, int32_t oflags); +static int32_t procdefrag(const char* path, int32_t oflags); +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs); +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys); +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px); +static int32_t proccheck(const char* path, int32_t oflags); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "create")) { + rv = runcreate(argc, argv); + } else if (!std::strcmp(argv[1], "inform")) { + rv = runinform(argc, argv); + } else if (!std::strcmp(argv[1], "set")) { + rv = runset(argc, argv); + } else if (!std::strcmp(argv[1], "remove")) { + rv = runremove(argc, argv); + } else if (!std::strcmp(argv[1], "get")) { + rv = runget(argc, argv); + } else if (!std::strcmp(argv[1], "list")) { + rv = runlist(argc, argv); + } else if (!std::strcmp(argv[1], "clear")) { + rv = runclear(argc, argv); + } else if (!std::strcmp(argv[1], "import")) { + rv = runimport(argc, argv); + } else if (!std::strcmp(argv[1], "copy")) { + rv = runcopy(argc, argv); + } else if (!std::strcmp(argv[1], "dump")) { + rv = rundump(argc, argv); + } else if (!std::strcmp(argv[1], "load")) { + rv = runload(argc, argv); + } else if (!std::strcmp(argv[1], "defrag")) { + rv = rundefrag(argc, argv); + } else if (!std::strcmp(argv[1], "setbulk")) { + rv = runsetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "removebulk")) { + rv = runremovebulk(argc, argv); + } else if (!std::strcmp(argv[1], "getbulk")) { + rv = rungetbulk(argc, argv); + } else if (!std::strcmp(argv[1], "check")) { + rv = runcheck(argc, argv); + } else if (!std::strcmp(argv[1], "version") || !std::strcmp(argv[1], "--version")) { + printversion(); + } else { + usage(); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: the command line utility of the file tree database of Kyoto Cabinet\n", + g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s create [-otr] [-onl|-otl|-onr] [-apow num] [-fpow num] [-ts] [-tl] [-tc]" + " [-bnum num] [-psiz num] [-rcd|-rcld|-rcdd] path\n", g_progname); + eprintf(" %s inform [-onl|-otl|-onr] [-st] path\n", g_progname); + eprintf(" %s set [-onl|-otl|-onr] [-add|-rep|-app|-inci|-incd] [-sx] path key value\n", + g_progname); + eprintf(" %s remove [-onl|-otl|-onr] [-sx] path key\n", g_progname); + eprintf(" %s get [-onl|-otl|-onr] [-rm] [-sx] [-px] [-pz] path key\n", g_progname); + eprintf(" %s list [-onl|-otl|-onr] [-des] [-max num] [-rm] [-sx] [-pv] [-px] path [key]\n", + g_progname); + eprintf(" %s clear [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s import [-onl|-otl|-onr] [-sx] path [file]\n", g_progname); + eprintf(" %s copy [-onl|-otl|-onr] path file\n", g_progname); + eprintf(" %s dump [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s load [-otr] [-onl|-otl|-onr] path [file]\n", g_progname); + eprintf(" %s defrag [-onl|-otl|-onr] path\n", g_progname); + eprintf(" %s setbulk [-onl|-otl|-onr] [-sx] path key value ...\n", g_progname); + eprintf(" %s removebulk [-onl|-otl|-onr] [-sx] path key ...\n", g_progname); + eprintf(" %s getbulk [-onl|-otl|-onr] [-sx] [-px] path key ...\n", g_progname); + eprintf(" %s check [-onl|-otl|-onr] path\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print error message of database +static void dberrprint(kc::BasicDB* db, const char* info) { + const kc::BasicDB::Error& err = db->error(); + eprintf("%s: %s: %s: %d: %s: %s\n", + g_progname, info, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// parse arguments of create command +static int32_t runcreate(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int32_t psiz = -1; + kc::Comparator* rcomp = NULL; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::TreeDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::TreeDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::TreeDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::TreeDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccreate(path, oflags, apow, fpow, opts, bnum, psiz, rcomp); + return rv; +} + + +// parse arguments of inform command +static int32_t runinform(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + bool st = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-st")) { + st = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procinform(path, oflags, st); + return rv; +} + + +// parse arguments of set command +static int32_t runset(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + const char* vstr = NULL; + int32_t oflags = 0; + int32_t mode = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-add")) { + mode = 'a'; + } else if (!std::strcmp(argv[i], "-rep")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-app")) { + mode = 'c'; + } else if (!std::strcmp(argv[i], "-inci")) { + mode = 'i'; + } else if (!std::strcmp(argv[i], "-incd")) { + mode = 'd'; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else if (!vstr) { + vstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr || !vstr) usage(); + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + int32_t rv = procset(path, kstr, ksiz, vstr, vsiz, oflags, mode); + delete[] kbuf; + delete[] vbuf; + return rv; +} + + +// parse arguments of remove command +static int32_t runremove(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procremove(path, kstr, ksiz, oflags); + delete[] kbuf; + return rv; +} + + +// parse arguments of get command +static int32_t runget(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool rm = false; + bool sx = false; + bool px = false; + bool pz = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else if (!std::strcmp(argv[i], "-pz")) { + pz = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path || !kstr) usage(); + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + int32_t rv = procget(path, kstr, ksiz, oflags, rm, px, pz); + delete[] kbuf; + return rv; +} + + +// parse arguments of list command +static int32_t runlist(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* kstr = NULL; + int32_t oflags = 0; + bool des = false; + int64_t max = -1; + bool rm = false; + bool sx = false; + bool pv = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-des")) { + des = true; + } else if (!std::strcmp(argv[i], "-max")) { + if (++i >= argc) usage(); + max = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rm")) { + rm = true; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-pv")) { + pv = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!kstr) { + kstr = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + char* kbuf = NULL; + size_t ksiz = 0; + if (kstr) { + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = new char[ksiz+1]; + std::memcpy(kbuf, kstr, ksiz); + kbuf[ksiz] = '\0'; + } + } + int32_t rv = proclist(path, kbuf, ksiz, oflags, des, max, rm, pv, px); + delete[] kbuf; + return rv; +} + + +// parse arguments of clear command +static int32_t runclear(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procclear(path, oflags); + return rv; +} + + +// parse arguments of import command +static int32_t runimport(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procimport(path, file, oflags, sx); + return rv; +} + + +// parse arguments of copy command +static int32_t runcopy(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path || !file) usage(); + int32_t rv = proccopy(path, file, oflags); + return rv; +} + + +// parse arguments of dump command +static int32_t rundump(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procdump(path, file, oflags); + return rv; +} + + +// parse arguments of load command +static int32_t runload(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* file = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-otr")) { + oflags |= kc::TreeDB::OTRUNCATE; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procload(path, file, oflags); + return rv; +} + + +// parse arguments of defrag command +static int32_t rundefrag(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = procdefrag(path, oflags); + return rv; +} + + +// parse arguments of setbulk command +static int32_t runsetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::map<std::string, std::string> recs; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + if (++i >= argc) usage(); + const char* vstr = argv[i]; + char* kbuf; + size_t ksiz; + char* vbuf; + size_t vsiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + vbuf = kc::hexdecode(vstr, &vsiz); + vstr = vbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + vsiz = std::strlen(vstr); + vbuf = NULL; + } + std::string key(kstr, ksiz); + std::string value(vstr, vsiz); + recs[key] = value; + delete[] kbuf; + delete[] vbuf; + } + } + if (!path) usage(); + int32_t rv = procsetbulk(path, oflags, recs); + return rv; +} + + +// parse arguments of removebulk command +static int32_t runremovebulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procremovebulk(path, oflags, keys); + return rv; +} + + +// parse arguments of getbulk command +static int32_t rungetbulk(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + std::vector<std::string> keys; + int32_t oflags = 0; + bool sx = false; + bool px = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-sx")) { + sx = true; + } else if (!std::strcmp(argv[i], "-px")) { + px = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + const char* kstr = argv[i]; + char* kbuf; + size_t ksiz; + if (sx) { + kbuf = kc::hexdecode(kstr, &ksiz); + kstr = kbuf; + } else { + ksiz = std::strlen(kstr); + kbuf = NULL; + } + std::string key(kstr, ksiz); + keys.push_back(key); + delete[] kbuf; + } + } + if (!path) usage(); + int32_t rv = procgetbulk(path, oflags, keys, px); + return rv; +} + + +// parse arguments of check command +static int32_t runcheck(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + int32_t oflags = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else { + usage(); + } + } + if (!path) usage(); + int32_t rv = proccheck(path, oflags); + return rv; +} + + +// perform create command +static int32_t proccreate(const char* path, int32_t oflags, int32_t apow, int32_t fpow, + int32_t opts, int64_t bnum, int32_t psiz, kc::Comparator* rcomp) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (rcomp) db.tune_comparator(rcomp); + if (!db.open(path, kc::TreeDB::OWRITER | kc::TreeDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform inform command +static int32_t procinform(const char* path, int32_t oflags, bool st) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (st) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["fbpnum_used"] = ""; + status["bnum_used"] = ""; + status["cusage_lcnt"] = ""; + status["cusage_lsiz"] = ""; + status["cusage_icnt"] = ""; + status["cusage_isiz"] = ""; + status["tree_level"] = ""; + if (db.status(&status)) { + uint32_t type = kc::atoi(status["realtype"].c_str()); + oprintf("type: %s (type=0x%02X) (%s)\n", + status["type"].c_str(), type, kc::BasicDB::typestring(type)); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::TreeDB::FOPEN) oprintf(" open"); + if (flags & kc::TreeDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + if (kc::atoi(status["trimmed"].c_str()) > 0) oprintf(" (trimmed)"); + oprintf("\n", flags); + int32_t apow = kc::atoi(status["apow"].c_str()); + oprintf("alignment: %d (apow=%d)\n", 1 << apow, apow); + int32_t fpow = kc::atoi(status["fpow"].c_str()); + int32_t fbpnum = fpow > 0 ? 1 << fpow : 0; + int32_t fbpused = kc::atoi(status["fbpnum_used"].c_str()); + int64_t frgcnt = kc::atoi(status["frgcnt"].c_str()); + oprintf("free block pool: %d (fpow=%d) (used=%d) (frg=%lld)\n", + fbpnum, fpow, fbpused, (long long)frgcnt); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::TreeDB::TSMALL) oprintf(" small"); + if (opts & kc::TreeDB::TLINEAR) oprintf(" linear"); + if (opts & kc::TreeDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + oprintf("comparator: %s\n", status["rcomp"].c_str()); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t bnumused = kc::atoi(status["bnum_used"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + int64_t pnum = kc::atoi(status["pnum"].c_str()); + int64_t lcnt = kc::atoi(status["lcnt"].c_str()); + int64_t icnt = kc::atoi(status["icnt"].c_str()); + int32_t tlevel = kc::atoi(status["tree_level"].c_str()); + int32_t psiz = kc::atoi(status["psiz"].c_str()); + double load = 0; + if (pnum > 0 && bnumused > 0) { + load = (double)pnum / bnumused; + if (!(opts & kc::TreeDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (used=%lld) (load=%.2f)\n", + (long long)bnum, (long long)bnumused, load); + oprintf("pages: %lld (leaf=%lld) (inner=%lld) (level=%d) (psiz=%d)\n", + (long long)pnum, (long long)lcnt, (long long)icnt, tlevel, psiz); + int64_t pccap = kc::atoi(status["pccap"].c_str()); + int64_t cusage = kc::atoi(status["cusage"].c_str()); + int64_t culcnt = kc::atoi(status["cusage_lcnt"].c_str()); + int64_t culsiz = kc::atoi(status["cusage_lsiz"].c_str()); + int64_t cuicnt = kc::atoi(status["cusage_icnt"].c_str()); + int64_t cuisiz = kc::atoi(status["cusage_isiz"].c_str()); + oprintf("cache: %lld (cap=%lld) (ratio=%.2f) (leaf=%lld:%lld) (inner=%lld:%lld)\n", + (long long)cusage, (long long)pccap, (double)cusage / pccap, + (long long)culsiz, (long long)culcnt, (long long)cuisiz, (long long)cuicnt); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + int64_t msiz = kc::atoi(status["msiz"].c_str()); + int64_t realsize = kc::atoi(status["realsize"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s) (map=%lld)", size, sizestr.c_str(), (long long)msiz); + if (size != realsize) oprintf(" (gap=%lld)", (long long)(realsize - size)); + oprintf("\n"); + } else { + dberrprint(&db, "DB::status failed"); + err = true; + } + } else { + uint8_t flags = db.flags(); + if (flags != 0) { + oprintf("status:"); + if (flags & kc::TreeDB::FOPEN) oprintf(" open"); + if (flags & kc::TreeDB::FFATAL) oprintf(" fatal"); + oprintf("\n"); + } + oprintf("count: %lld\n", (long long)db.count()); + oprintf("size: %lld\n", (long long)db.size()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform set command +static int32_t procset(const char* path, const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, int32_t oflags, int32_t mode) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + switch (mode) { + default: { + if (!db.set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 'a': { + if (!db.add(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::add failed"); + err = true; + } + break; + } + case 'r': { + if (!db.replace(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::replace failed"); + err = true; + } + break; + } + case 'c': { + if (!db.append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(&db, "DB::append failed"); + err = true; + } + break; + } + case 'i': { + int64_t onum = db.increment(kbuf, ksiz, kc::atoi(vbuf)); + if (onum == kc::INT64MIN) { + dberrprint(&db, "DB::increment failed"); + err = true; + } else { + oprintf("%lld\n", (long long)onum); + } + break; + } + case 'd': { + double onum = db.increment_double(kbuf, ksiz, kc::atof(vbuf)); + if (kc::chknan(onum)) { + dberrprint(&db, "DB::increment_double failed"); + err = true; + } else { + oprintf("%f\n", onum); + } + break; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform remove command +static int32_t procremove(const char* path, const char* kbuf, size_t ksiz, int32_t oflags) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.remove(kbuf, ksiz)) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform get command +static int32_t procget(const char* path, const char* kbuf, size_t ksiz, + int32_t oflags, bool rm, bool px, bool pz) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::TreeDB::OWRITER : kc::TreeDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + char* vbuf; + size_t vsiz; + if (rm) { + vbuf = db.seize(kbuf, ksiz, &vsiz); + } else { + vbuf = db.get(kbuf, ksiz, &vsiz); + } + if (vbuf) { + printdata(vbuf, vsiz, px); + if (!pz) oprintf("\n"); + delete[] vbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform list command +static int32_t proclist(const char* path, const char*kbuf, size_t ksiz, int32_t oflags, + bool des, int64_t max, bool rm, bool pv, bool px) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + uint32_t omode = rm ? kc::TreeDB::OWRITER : kc::TreeDB::OREADER; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(bool rm, bool pv, bool px) : rm_(rm), pv_(pv), px_(px) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + printdata(kbuf, ksiz, px_); + if (pv_) { + oprintf("\t"); + printdata(vbuf, vsiz, px_); + } + oprintf("\n"); + return rm_ ? REMOVE : NOP; + } + bool rm_; + bool pv_; + bool px_; + } visitor(rm, pv, px); + if (kbuf || des || max >= 0) { + if (max < 0) max = kc::INT64MAX; + kc::TreeDB::Cursor cur(&db); + if (des) { + if (kbuf) { + if (!cur.jump_back(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump_back() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } else { + if (kbuf) { + if (!cur.jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } else { + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::jump failed"); + err = true; + } + } + while (!err && max > 0) { + if (!cur.accept(&visitor, rm, true)) { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::accept failed"); + err = true; + } + break; + } + max--; + } + } + } else { + if (!db.iterate(&visitor, rm)) { + dberrprint(&db, "DB::iterate failed"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform clear command +static int32_t procclear(const char* path, int32_t oflags) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.clear()) { + dberrprint(&db, "DB::clear failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform import command +static int32_t procimport(const char* path, const char* file, int32_t oflags, bool sx) { + std::istream *is = &std::cin; + std::ifstream ifs; + if (file) { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | kc::TreeDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + int64_t cnt = 0; + std::string line; + std::vector<std::string> fields; + while (!err && mygetline(is, &line)) { + cnt++; + kc::strsplit(line, '\t', &fields); + if (sx) { + std::vector<std::string>::iterator it = fields.begin(); + std::vector<std::string>::iterator itend = fields.end(); + while (it != itend) { + size_t esiz; + char* ebuf = kc::hexdecode(it->c_str(), &esiz); + it->clear(); + it->append(ebuf, esiz); + delete[] ebuf; + ++it; + } + } + switch (fields.size()) { + case 2: { + if (!db.set(fields[0], fields[1])) { + dberrprint(&db, "DB::set failed"); + err = true; + } + break; + } + case 1: { + if (!db.remove(fields[0]) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::remove failed"); + err = true; + } + break; + } + } + oputchar('.'); + if (cnt % 50 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + if (cnt % 50 > 0) oprintf(" (%lld)\n", (long long)cnt); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform copy command +static int32_t proccopy(const char* path, const char* file, int32_t oflags) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + DotChecker checker(&std::cout, -100); + if (!db.copy(file, &checker)) { + dberrprint(&db, "DB::copy failed"); + err = true; + } + oprintf(" (end)\n"); + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld blocks were copied successfully\n", (long long)checker.count()); + return err ? 1 : 0; +} + + +// perform dump command +static int32_t procdump(const char* path, const char* file, int32_t oflags) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, 1000); + if (!db.dump_snapshot(file)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were dumped successfully\n", (long long)checker.count()); + } else { + if (!db.dump_snapshot(&std::cout)) { + dberrprint(&db, "DB::dump_snapshot"); + err = true; + } + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform load command +static int32_t procload(const char* path, const char* file, int32_t oflags) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | kc::TreeDB::OCREATE | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (file) { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(file)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } else { + DotChecker checker(&std::cout, -1000); + if (!db.load_snapshot(&std::cin)) { + dberrprint(&db, "DB::load_snapshot"); + err = true; + } + oprintf(" (end)\n"); + if (!err) oprintf("%lld records were loaded successfully\n", (long long)checker.count()); + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform defrag command +static int32_t procdefrag(const char* path, int32_t oflags) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (!db.defrag(0)) { + dberrprint(&db, "DB::defrag failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform setbulk command +static int32_t procsetbulk(const char* path, int32_t oflags, + const std::map<std::string, std::string>& recs) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.set_bulk(recs) != (int64_t)recs.size()) { + dberrprint(&db, "DB::set_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform removebulk command +static int32_t procremovebulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OWRITER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + if (db.remove_bulk(keys) < 0) { + dberrprint(&db, "DB::remove_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform getbulk command +static int32_t procgetbulk(const char* path, int32_t oflags, + const std::vector<std::string>& keys, bool px) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + std::map<std::string, std::string> recs; + if (db.get_bulk(keys, &recs) >= 0) { + std::map<std::string, std::string>::iterator it = recs.begin(); + std::map<std::string, std::string>::iterator itend = recs.end(); + while (it != itend) { + printdata(it->first.data(), it->first.size(), px); + oprintf("\t"); + printdata(it->second.data(), it->second.size(), px); + oprintf("\n"); + ++it; + } + } else { + dberrprint(&db, "DB::get_bulk failed"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + return err ? 1 : 0; +} + + +// perform check command +static int32_t proccheck(const char* path, int32_t oflags) { + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cerr)); + if (!db.open(path, kc::TreeDB::OREADER | oflags)) { + dberrprint(&db, "DB::open failed"); + return 1; + } + bool err = false; + kc::TreeDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "DB::jump failed"); + err = true; + } + int64_t cnt = 0; + while (!err) { + size_t ksiz; + const char* vbuf; + size_t vsiz; + char* kbuf = cur.get(&ksiz, &vbuf, &vsiz); + if (kbuf) { + cnt++; + size_t rsiz; + char* rbuf = db.get(kbuf, ksiz, &rsiz); + if (rbuf) { + if (rsiz != vsiz || std::memcmp(rbuf, vbuf, rsiz)) { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] rbuf; + } else { + dberrprint(&db, "DB::get failed"); + err = true; + } + delete[] kbuf; + if (cnt % 1000 == 0) { + oputchar('.'); + if (cnt % 50000 == 0) oprintf(" (%lld)\n", (long long)cnt); + } + } else { + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::get failed"); + err = true; + } + break; + } + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, "Cursor::step failed"); + err = true; + } + } + oprintf(" (end)\n"); + if (db.count() != cnt) { + dberrprint(&db, "DB::count failed"); + err = true; + } + kc::File::Status sbuf; + if (kc::File::status(path, &sbuf)) { + if (db.size() != sbuf.size && sbuf.size % (1 << 20) != 0) { + dberrprint(&db, "DB::size failed"); + err = true; + } + } else { + dberrprint(&db, "File::status failed"); + err = true; + } + if (db.flags() & kc::TreeDB::FFATAL) { + dberrprint(&db, "DB::flags indicated fatal error"); + err = true; + } + if (!db.close()) { + dberrprint(&db, "DB::close failed"); + err = true; + } + if (!err) oprintf("%lld records were checked successfully\n", (long long)cnt); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kctreetest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kctreetest.cc new file mode 100644 index 0000000000..3a26095027 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kctreetest.cc @@ -0,0 +1,2534 @@ +/************************************************************************************************* + * The test cases of the file tree database + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include <kchashdb.h> +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func); +static void dbmetaprint(kc::BasicDB* db, bool verbose); +static int32_t runorder(int argc, char** argv); +static int32_t runqueue(int argc, char** argv); +static int32_t runwicked(int argc, char** argv); +static int32_t runtran(int argc, char** argv); +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t apow, int32_t fpow, + int32_t opts, int64_t bnum, int32_t psiz, int64_t msiz, + int64_t dfunit, int64_t pccap, kc::Comparator* rcomp, bool lv); +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, int64_t bnum, + int32_t psiz, int64_t msiz, int64_t dfunit, int64_t pccap, + kc::Comparator* rcomp, bool lv); +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, int64_t bnum, + int32_t psiz, int64_t msiz, int64_t dfunit, int64_t pccap, + kc::Comparator* rcomp, bool lv); +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, int64_t bnum, + int32_t psiz, int64_t msiz, int64_t dfunit, int64_t pccap, + kc::Comparator* rcomp, bool lv); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "order")) { + rv = runorder(argc, argv); + } else if (!std::strcmp(argv[1], "queue")) { + rv = runqueue(argc, argv); + } else if (!std::strcmp(argv[1], "wicked")) { + rv = runwicked(argc, argv); + } else if (!std::strcmp(argv[1], "tran")) { + rv = runtran(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the file tree database of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s order [-th num] [-rnd] [-set|-get|-getw|-rem|-etc] [-tran]" + " [-oat|-oas|-onl|-otl|-onr] [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num]" + " [-psiz num] [-msiz num] [-dfunit num] [-pccap num] [-rcd|-rcld|-rcdd] [-lv]" + " path rnum\n", g_progname); + eprintf(" %s queue [-th num] [-it num] [-rnd] [-oat|-oas|-onl|-otl|-onr]" + " [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num] [-psiz num] [-msiz num]" + " [-dfunit num] [-pccap num] [-rcd|-rcld|-rcdd] [-lv] path rnum\n", g_progname); + eprintf(" %s wicked [-th num] [-it num] [-oat|-oas|-onl|-otl|-onr]" + " [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num] [-psiz num] [-msiz num]" + " [-dfunit num] [-pccap num] [-rcd|-rcld|-rcdd] [-lv] path rnum\n", g_progname); + eprintf(" %s tran [-th num] [-it num] [-hard] [-oat|-oas|-onl|-otl|-onr]" + " [-apow num] [-fpow num] [-ts] [-tl] [-tc] [-bnum num] [-psiz num] [-msiz num]" + " [-dfunit num] [-pccap num] [-rcd|-rcld|-rcdd] [-lv] path rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print the error message of a database +static void dberrprint(kc::BasicDB* db, int32_t line, const char* func) { + const kc::BasicDB::Error& err = db->error(); + oprintf("%s: %d: %s: %s: %d: %s: %s\n", + g_progname, line, func, db->path().c_str(), err.code(), err.name(), err.message()); +} + + +// print members of a database +static void dbmetaprint(kc::BasicDB* db, bool verbose) { + if (verbose) { + std::map<std::string, std::string> status; + status["opaque"] = ""; + status["fbpnum_used"] = ""; + status["bnum_used"] = ""; + status["cusage_lcnt"] = ""; + status["cusage_lsiz"] = ""; + status["cusage_icnt"] = ""; + status["cusage_isiz"] = ""; + status["tree_level"] = ""; + if (db->status(&status)) { + uint32_t type = kc::atoi(status["type"].c_str()); + oprintf("type: %s (%s) (type=0x%02X)\n", + kc::BasicDB::typecname(type), kc::BasicDB::typestring(type), type); + uint32_t chksum = kc::atoi(status["chksum"].c_str()); + oprintf("format version: %s (libver=%s.%s) (chksum=0x%02X)\n", status["fmtver"].c_str(), + status["libver"].c_str(), status["librev"].c_str(), chksum); + oprintf("path: %s\n", status["path"].c_str()); + int32_t flags = kc::atoi(status["flags"].c_str()); + oprintf("status flags:"); + if (flags & kc::TreeDB::FOPEN) oprintf(" open"); + if (flags & kc::TreeDB::FFATAL) oprintf(" fatal"); + oprintf(" (flags=%d)", flags); + if (kc::atoi(status["recovered"].c_str()) > 0) oprintf(" (recovered)"); + if (kc::atoi(status["reorganized"].c_str()) > 0) oprintf(" (reorganized)"); + if (kc::atoi(status["trimmed"].c_str()) > 0) oprintf(" (trimmed)"); + oprintf("\n", flags); + int32_t apow = kc::atoi(status["apow"].c_str()); + oprintf("alignment: %d (apow=%d)\n", 1 << apow, apow); + int32_t fpow = kc::atoi(status["fpow"].c_str()); + int32_t fbpnum = fpow > 0 ? 1 << fpow : 0; + int32_t fbpused = kc::atoi(status["fbpnum_used"].c_str()); + int64_t frgcnt = kc::atoi(status["frgcnt"].c_str()); + oprintf("free block pool: %d (fpow=%d) (used=%d) (frg=%lld)\n", + fbpnum, fpow, fbpused, (long long)frgcnt); + int32_t opts = kc::atoi(status["opts"].c_str()); + oprintf("options:"); + if (opts & kc::TreeDB::TSMALL) oprintf(" small"); + if (opts & kc::TreeDB::TLINEAR) oprintf(" linear"); + if (opts & kc::TreeDB::TCOMPRESS) oprintf(" compress"); + oprintf(" (opts=%d)\n", opts); + oprintf("comparator: %s\n", status["rcomp"].c_str()); + if (status["opaque"].size() >= 16) { + const char* opaque = status["opaque"].c_str(); + oprintf("opaque:"); + if (std::count(opaque, opaque + 16, 0) != 16) { + for (int32_t i = 0; i < 16; i++) { + oprintf(" %02X", ((unsigned char*)opaque)[i]); + } + } else { + oprintf(" 0"); + } + oprintf("\n"); + } + int64_t bnum = kc::atoi(status["bnum"].c_str()); + int64_t bnumused = kc::atoi(status["bnum_used"].c_str()); + int64_t count = kc::atoi(status["count"].c_str()); + int64_t pnum = kc::atoi(status["pnum"].c_str()); + int64_t lcnt = kc::atoi(status["lcnt"].c_str()); + int64_t icnt = kc::atoi(status["icnt"].c_str()); + int32_t tlevel = kc::atoi(status["tree_level"].c_str()); + int32_t psiz = kc::atoi(status["psiz"].c_str()); + double load = 0; + if (pnum > 0 && bnumused > 0) { + load = (double)pnum / bnumused; + if (!(opts & kc::TreeDB::TLINEAR)) load = std::log(load + 1) / std::log(2.0); + } + oprintf("buckets: %lld (used=%lld) (load=%.2f)\n", + (long long)bnum, (long long)bnumused, load); + oprintf("pages: %lld (leaf=%lld) (inner=%lld) (level=%d) (psiz=%d)\n", + (long long)pnum, (long long)lcnt, (long long)icnt, tlevel, psiz); + int64_t pccap = kc::atoi(status["pccap"].c_str()); + int64_t cusage = kc::atoi(status["cusage"].c_str()); + int64_t culcnt = kc::atoi(status["cusage_lcnt"].c_str()); + int64_t culsiz = kc::atoi(status["cusage_lsiz"].c_str()); + int64_t cuicnt = kc::atoi(status["cusage_icnt"].c_str()); + int64_t cuisiz = kc::atoi(status["cusage_isiz"].c_str()); + oprintf("cache: %lld (cap=%lld) (ratio=%.2f) (leaf=%lld:%lld) (inner=%lld:%lld)\n", + (long long)cusage, (long long)pccap, (double)cusage / pccap, + (long long)culsiz, (long long)culcnt, (long long)cuisiz, (long long)cuicnt); + std::string cntstr = unitnumstr(count); + oprintf("count: %lld (%s)\n", count, cntstr.c_str()); + int64_t size = kc::atoi(status["size"].c_str()); + int64_t msiz = kc::atoi(status["msiz"].c_str()); + int64_t realsize = kc::atoi(status["realsize"].c_str()); + std::string sizestr = unitnumstrbyte(size); + oprintf("size: %lld (%s) (map=%lld)", size, sizestr.c_str(), (long long)msiz); + if (size != realsize) oprintf(" (gap=%lld)", (long long)(realsize - size)); + oprintf("\n"); + } + } else { + oprintf("count: %lld\n", (long long)db->count()); + oprintf("size: %lld\n", (long long)db->size()); + } + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); +} + + +// parse arguments of order command +static int32_t runorder(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + int32_t mode = 0; + bool tran = false; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-set")) { + mode = 's'; + } else if (!std::strcmp(argv[i], "-get")) { + mode = 'g'; + } else if (!std::strcmp(argv[i], "-getw")) { + mode = 'w'; + } else if (!std::strcmp(argv[i], "-rem")) { + mode = 'r'; + } else if (!std::strcmp(argv[i], "-etc")) { + mode = 'e'; + } else if (!std::strcmp(argv[i], "-tran")) { + tran = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::TreeDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::TreeDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::TreeDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::TreeDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::TreeDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procorder(path, rnum, thnum, rnd, mode, tran, oflags, + apow, fpow, opts, bnum, psiz, msiz, dfunit, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of queue command +static int32_t runqueue(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool rnd = false; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::TreeDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::TreeDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::TreeDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::TreeDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::TreeDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procqueue(path, rnum, thnum, itnum, rnd, oflags, + apow, fpow, opts, bnum, psiz, msiz, dfunit, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of wicked command +static int32_t runwicked(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::TreeDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::TreeDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::TreeDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::TreeDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::TreeDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procwicked(path, rnum, thnum, itnum, oflags, + apow, fpow, opts, bnum, psiz, msiz, dfunit, pccap, rcomp, lv); + return rv; +} + + +// parse arguments of tran command +static int32_t runtran(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + int32_t itnum = 1; + bool hard = false; + int32_t oflags = 0; + int32_t apow = -1; + int32_t fpow = -1; + int32_t opts = 0; + int64_t bnum = -1; + int64_t psiz = -1; + int64_t msiz = -1; + int64_t dfunit = -1; + int64_t pccap = 0; + kc::Comparator* rcomp = NULL; + bool lv = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-it")) { + if (++i >= argc) usage(); + itnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-hard")) { + hard = true; + } else if (!std::strcmp(argv[i], "-oat")) { + oflags |= kc::TreeDB::OAUTOTRAN; + } else if (!std::strcmp(argv[i], "-oas")) { + oflags |= kc::TreeDB::OAUTOSYNC; + } else if (!std::strcmp(argv[i], "-onl")) { + oflags |= kc::TreeDB::ONOLOCK; + } else if (!std::strcmp(argv[i], "-otl")) { + oflags |= kc::TreeDB::OTRYLOCK; + } else if (!std::strcmp(argv[i], "-onr")) { + oflags |= kc::TreeDB::ONOREPAIR; + } else if (!std::strcmp(argv[i], "-apow")) { + if (++i >= argc) usage(); + apow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-fpow")) { + if (++i >= argc) usage(); + fpow = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-ts")) { + opts |= kc::TreeDB::TSMALL; + } else if (!std::strcmp(argv[i], "-tl")) { + opts |= kc::TreeDB::TLINEAR; + } else if (!std::strcmp(argv[i], "-tc")) { + opts |= kc::TreeDB::TCOMPRESS; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-psiz")) { + if (++i >= argc) usage(); + psiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-dfunit")) { + if (++i >= argc) usage(); + dfunit = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-pccap")) { + if (++i >= argc) usage(); + pccap = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rcd")) { + rcomp = kc::DECIMALCOMP; + } else if (!std::strcmp(argv[i], "-rcld")) { + rcomp = kc::LEXICALDESCCOMP; + } else if (!std::strcmp(argv[i], "-rcdd")) { + rcomp = kc::DECIMALDESCCOMP; + } else if (!std::strcmp(argv[i], "-lv")) { + lv = true; + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || itnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proctran(path, rnum, thnum, itnum, hard, oflags, + apow, fpow, opts, bnum, psiz, msiz, dfunit, pccap, rcomp, lv); + return rv; +} + + +// perform order command +static int32_t procorder(const char* path, int64_t rnum, int32_t thnum, bool rnd, int32_t mode, + bool tran, int32_t oflags, int32_t apow, int32_t fpow, + int32_t opts, int64_t bnum, int32_t psiz, int64_t msiz, + int64_t dfunit, int64_t pccap, kc::Comparator* rcomp, bool lv) { + oprintf("<In-order Test>\n seed=%u path=%s rnum=%lld thnum=%d rnd=%d mode=%d tran=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld psiz=%d msiz=%lld" + " dfunit=%lld pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, rnd, mode, tran, oflags, apow, fpow, opts, + (long long)bnum, psiz, (long long)msiz, (long long)dfunit, (long long)pccap, + rcomp, lv); + bool err = false; + kc::TreeDB db; + oprintf("opening the database:\n"); + double stime = kc::time(); + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + uint32_t omode = kc::TreeDB::OWRITER | kc::TreeDB::OCREATE | kc::TreeDB::OTRUNCATE; + if (mode == 'r') { + omode = kc::TreeDB::OWRITER | kc::TreeDB::OCREATE; + } else if (mode == 'g' || mode == 'w') { + omode = kc::TreeDB::OREADER; + } + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + double etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + if (mode == 0 || mode == 's' || mode == 'e') { + oprintf("setting records:\n"); + stime = kc::time(); + class ThreadSet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadSet threadsets[THREADMAX]; + if (thnum < 2) { + threadsets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadsets[0].run(); + if (threadsets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadsets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsets[i].join(); + if (threadsets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 's'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("adding records:\n"); + stime = kc::time(); + class ThreadAdd : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->add(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAdd threadadds[THREADMAX]; + if (thnum < 2) { + threadadds[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadadds[0].run(); + if (threadadds[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadadds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadadds[i].join(); + if (threadadds[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("appending records:\n"); + stime = kc::time(); + class ThreadAppend : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadAppend threadappends[THREADMAX]; + if (thnum < 2) { + threadappends[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadappends[0].run(); + if (threadappends[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadappends[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadappends[i].join(); + if (threadappends[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + char* opaque = db.opaque(); + if (opaque) { + std::memcpy(opaque, "1234567890123456", 16); + if (!db.synchronize_opaque()) { + dberrprint(&db, __LINE__, "DB::synchronize_opaque"); + err = true; + } + } else { + dberrprint(&db, __LINE__, "DB::opaque"); + err = true; + } + } + if (mode == 0 || mode == 'g' || mode == 'e') { + oprintf("getting records:\n"); + stime = kc::time(); + class ThreadGet : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + if (vsiz < ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + delete[] vbuf; + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOPERM && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGet threadgets[THREADMAX]; + if (thnum < 2) { + threadgets[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgets[0].run(); + if (threadgets[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgets[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgets[i].join(); + if (threadgets[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'g'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'w' || mode == 'e') { + oprintf("getting records with a buffer:\n"); + stime = kc::time(); + class ThreadGetBuffer : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + char vbuf[RECBUFSIZ]; + int32_t vsiz = db_->get(kbuf, ksiz, vbuf, sizeof(vbuf)); + if (vsiz >= 0) { + if (vsiz < (int32_t)ksiz || std::memcmp(vbuf, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + } else if (!rnd_ || db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + bool tran_; + }; + ThreadGetBuffer threadgetbuffers[THREADMAX]; + if (thnum < 2) { + threadgetbuffers[0].setparams(0, &db, rnum, thnum, rnd, tran); + threadgetbuffers[0].run(); + if (threadgetbuffers[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].setparams(i, &db, rnum, thnum, rnd, tran); + threadgetbuffers[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadgetbuffers[i].join(); + if (threadgetbuffers[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'w'); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the inner iterator:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorIterator : public kc::DB::Visitor { + public: + explicit VisitorIterator(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '+', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitoriterator(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + if (!db.iterate(&visitoriterator, true)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + if (rnd) oprintf(" (end)\n"); + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (visitoriterator.cnt() != cnt) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("traversing the database by the outer cursor:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + class VisitorCursor : public kc::DB::Visitor { + public: + explicit VisitorCursor(int64_t rnum, bool rnd) : + rnum_(rnum), rnd_(rnd), cnt_(0), rbuf_() { + std::memset(rbuf_, '-', sizeof(rbuf_)); + } + int64_t cnt() { + return cnt_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + const char* rv = NOP; + switch (rnd_ ? myrand(7) : cnt_ % 7) { + case 0: { + rv = rbuf_; + *sp = rnd_ ? myrand(sizeof(rbuf_)) : sizeof(rbuf_) / (cnt_ % 5 + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return rv; + } + int64_t rnum_; + bool rnd_; + int64_t cnt_; + char rbuf_[RECBUFSIZ]; + } visitorcursor(rnum, rnd); + if (tran && !db.begin_transaction(false)) { + dberrprint(&db, __LINE__, "DB::begin_transaction"); + err = true; + } + kc::TreeDB::Cursor cur(&db); + if (!cur.jump() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + kc::DB::Cursor* paracur = db.cursor(); + int64_t range = rnum * thnum; + while (!err && cur.accept(&visitorcursor, true, !rnd)) { + if (rnd) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)myrand(range)); + switch (myrand(3)) { + case 0: { + if (!db.remove(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "DB::remove"); + err = true; + } + break; + } + case 1: { + if (!paracur->jump(kbuf, ksiz) && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + break; + } + default: { + if (!cur.step() && db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::step"); + err = true; + } + break; + } + } + } + } + if (db.error() != kc::BasicDB::Error::NOREC) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + oprintf(" (end)\n"); + delete paracur; + if (tran && !db.end_transaction(true)) { + dberrprint(&db, __LINE__, "DB::end_transaction"); + err = true; + } + if (!rnd && visitorcursor.cnt() != cnt) { + dberrprint(&db, __LINE__, "Cursor::accept"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e') { + oprintf("synchronizing the database:\n"); + stime = kc::time(); + if (!db.synchronize(false, NULL)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + class SyncProcessor : public kc::BasicDB::FileProcessor { + public: + explicit SyncProcessor(int64_t rnum, bool rnd, int64_t size, int64_t msiz) : + rnum_(rnum), rnd_(rnd), size_(size), msiz_(msiz) {} + private: + bool process(const std::string& path, int64_t count, int64_t size) { + kc::File::Status sbuf; + if (!kc::File::status(path, &sbuf)) return false; + if (sbuf.size != size_ && sbuf.size != msiz_ && + sbuf.size % (1 << 20) != 0) return false; + if (size != size_) return false; + return true; + } + int64_t rnum_; + bool rnd_; + int64_t size_; + int64_t msiz_; + } syncprocessor(rnum, rnd, db.size(), msiz); + if (!db.synchronize(false, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::synchronize"); + err = true; + } + if (!db.occupy(rnd ? myrand(2) == 0 : true, &syncprocessor)) { + dberrprint(&db, __LINE__, "DB::occupy"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 'e' && db.size() < (256LL << 20)) { + oprintf("dumping records into snapshot:\n"); + stime = kc::time(); + std::ostringstream ostrm; + if (!db.dump_snapshot(&ostrm)) { + dberrprint(&db, __LINE__, "DB::dump_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + oprintf("loading records from snapshot:\n"); + stime = kc::time(); + int64_t cnt = db.count(); + if (rnd && myrand(2) == 0 && !db.clear()) { + dberrprint(&db, __LINE__, "DB::clear"); + err = true; + } + const std::string& str = ostrm.str(); + std::istringstream istrm(str); + if (!db.load_snapshot(&istrm) || db.count() != cnt) { + dberrprint(&db, __LINE__, "DB::load_snapshot"); + err = true; + } + etime = kc::time(); + dbmetaprint(&db, false); + oprintf("time: %.3f\n", etime - stime); + } + if (mode == 0 || mode == 'r' || mode == 'e') { + oprintf("removing records:\n"); + stime = kc::time(); + class ThreadRemove : public kc::Thread { + public: + void setparams(int32_t id, kc::BasicDB* db, int64_t rnum, int32_t thnum, + bool rnd, int32_t mode, bool tran) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + err_ = false; + rnd_ = rnd; + mode_ = mode; + tran_ = tran; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (tran_ && !db_->begin_transaction(false)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + err_ = true; + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", + (long long)(rnd_ ? myrand(range) + 1 : base + i)); + if (!db_->remove(kbuf, ksiz) && + ((!rnd_ && mode_ != 'e') || db_->error() != kc::BasicDB::Error::NOREC)) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (rnd_ && i % 8 == 0) { + switch (myrand(8)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 3: { + kc::DB::Cursor* cur = db_->cursor(); + if (cur->jump(kbuf, ksiz)) { + switch (myrand(8)) { + default: { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + case 1: { + size_t rsiz; + char* rbuf = cur->get_value(&rsiz, myrand(10) == 0); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + break; + } + case 2: { + size_t rksiz; + const char* rvbuf; + size_t rvsiz; + char* rkbuf = cur->get(&rksiz, &rvbuf, &rvsiz, myrand(10) == 0); + if (rkbuf) { + delete[] rkbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 3: { + std::string key, value; + if (!cur->get(&key, &value, myrand(10) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get"); + err_ = true; + } + break; + } + case 4: { + if (myrand(8) == 0 && !cur->remove() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + break; + } + } + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + delete cur; + break; + } + default: { + size_t vsiz; + char* vbuf = db_->get(kbuf, ksiz, &vsiz); + if (vbuf) { + delete[] vbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } + if (tran_ && !db_->end_transaction(true)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::BasicDB* db_; + int64_t rnum_; + int32_t thnum_; + bool err_; + bool rnd_; + int32_t mode_; + bool tran_; + }; + ThreadRemove threadremoves[THREADMAX]; + if (thnum < 2) { + threadremoves[0].setparams(0, &db, rnum, thnum, rnd, mode, tran); + threadremoves[0].run(); + if (threadremoves[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].setparams(i, &db, rnum, thnum, rnd, mode, tran); + threadremoves[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadremoves[i].join(); + if (threadremoves[i].error()) err = true; + } + } + etime = kc::time(); + dbmetaprint(&db, mode == 'r' || mode == 'e'); + oprintf("time: %.3f\n", etime - stime); + } + oprintf("closing the database:\n"); + stime = kc::time(); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform queue command +static int32_t procqueue(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool rnd, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, int64_t bnum, + int32_t psiz, int64_t msiz, int64_t dfunit, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Queue Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d rnd=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld psiz=%d msiz=%lld" + " dfunit=%lld pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, rnd, oflags, apow, fpow, opts, + (long long)bnum, psiz, (long long)msiz, (long long)dfunit, (long long)pccap, + rcomp, lv); + bool err = false; + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::TreeDB::OWRITER | kc::TreeDB::OCREATE; + if (itcnt == 1) omode |= kc::TreeDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadQueue : public kc::Thread { + public: + void setparams(int32_t id, kc::TreeDB* db, int64_t rnum, int32_t thnum, bool rnd, + int64_t width) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + width_ = width; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%010lld", (long long)(base + i)); + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + if (rnd_) { + if (myrand(width_ / 2) == 0) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + ksiz = std::sprintf(kbuf, "%010lld", (long long)myrand(range) + 1); + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->append(kbuf, ksiz, kbuf, ksiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 2: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + } + int64_t dnum = myrand(width_) + 2; + for (int64_t j = 0; j < dnum; j++) { + if (myrand(2) == 0) { + size_t rsiz; + char* rbuf = cur->get_key(&rsiz); + if (rbuf) { + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + if (myrand(2) == 0 && !cur->jump(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (myrand(10) == 0 && !db_->remove(rbuf, rsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + } else { + if (i > width_) { + if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + if (!cur->remove() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::remove"); + err_ = true; + } + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::TreeDB* db_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t width_; + bool err_; + }; + int64_t width = rnum / 10; + ThreadQueue threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, rnd, width); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, rnd, width); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + int64_t count = db.count(); + if (!rnd && itcnt == 1 && count != width * thnum) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + if ((rnd ? (myrand(2) == 0) : itcnt == itnum) && count > 0) { + kc::DB::Cursor* cur = db.cursor(); + if (!cur->jump()) { + dberrprint(&db, __LINE__, "Cursor::jump"); + err = true; + } + for (int64_t i = 1; i <= count; i++) { + if (!cur->remove()) { + dberrprint(&db, __LINE__, "Cursor::remove"); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (rnd) oprintf(" (end)\n"); + delete cur; + if (db.count() != 0) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform wicked command +static int32_t procwicked(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, int64_t bnum, + int32_t psiz, int64_t msiz, int64_t dfunit, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Wicked Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld psiz=%d msiz=%lld" + " dfunit=%lld pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, oflags, apow, fpow, opts, + (long long)bnum, psiz, (long long)msiz, (long long)dfunit, (long long)pccap, + rcomp, lv); + bool err = false; + kc::TreeDB db; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + if (itnum > 1) oprintf("iteration %d:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::TreeDB::OWRITER | kc::TreeDB::OCREATE; + if (itcnt == 1) omode |= kc::TreeDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + class ThreadWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::TreeDB* db, int64_t rnum, int32_t thnum, + const char* lbuf) { + id_ = id; + db_ = db; + rnum_ = rnum; + thnum_ = thnum; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_ / 2; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + bool tran = myrand(100) == 0; + if (tran) { + if (myrand(2) == 0) { + if (!db_->begin_transaction(myrand(rnum_) == 0)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + if (!db_->begin_transaction_try(myrand(rnum_) == 0)) { + if (db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::begin_transaction_try"); + err_ = true; + } + tran = false; + } + } + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (myrand(1000) == 0) { + ksiz = myrand(RECBUFSIZ) + 1; + if (myrand(2) == 0) { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = j; + } + } else { + for (size_t j = 0; j < ksiz; j++) { + kbuf[j] = myrand(256); + } + } + } + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + do { + switch (myrand(10)) { + case 0: { + if (!db_->set(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::set"); + err_ = true; + } + break; + } + case 1: { + if (!db_->add(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::DUPREC) { + dberrprint(db_, __LINE__, "DB::add"); + err_ = true; + } + break; + } + case 2: { + if (!db_->replace(kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::replace"); + err_ = true; + } + break; + } + case 3: { + if (!db_->append(kbuf, ksiz, vbuf, vsiz)) { + dberrprint(db_, __LINE__, "DB::append"); + err_ = true; + } + break; + } + case 4: { + if (myrand(2) == 0) { + int64_t num = myrand(rnum_); + int64_t orig = myrand(10) == 0 ? kc::INT64MIN : myrand(rnum_); + if (myrand(10) == 0) orig = orig == kc::INT64MIN ? kc::INT64MAX : -orig; + if (db_->increment(kbuf, ksiz, num, orig) == kc::INT64MIN && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment"); + err_ = true; + } + } else { + double num = myrand(rnum_ * 10) / (myrand(rnum_) + 1.0); + double orig = myrand(10) == 0 ? -kc::inf() : myrand(rnum_); + if (myrand(10) == 0) orig = -orig; + if (kc::chknan(db_->increment_double(kbuf, ksiz, num, orig)) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::increment_double"); + err_ = true; + } + } + break; + } + case 5: { + if (!db_->cas(kbuf, ksiz, kbuf, ksiz, vbuf, vsiz) && + db_->error() != kc::BasicDB::Error::LOGIC) { + dberrprint(db_, __LINE__, "DB::cas"); + err_ = true; + } + break; + } + case 6: { + if (!db_->remove(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::remove"); + err_ = true; + } + break; + } + case 7: { + if (myrand(2) == 0) { + if (db_->check(kbuf, ksiz) < 0 && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::check"); + err_ = true; + } + } else { + size_t rsiz; + char* rbuf = db_->seize(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::seize"); + err_ = true; + } + } + break; + } + case 8: { + if (myrand(10) == 0) { + if (myrand(4) == 0) { + if (!cur->jump_back(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump_back"); + err_ = true; + } + } else { + if (!cur->jump(kbuf, ksiz) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + } else { + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* lbuf) : lbuf_(lbuf) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = lbuf_; + *sp = myrand(RECBUFSIZL) / (myrand(5) + 1); + break; + } + case 1: { + rv = REMOVE; + break; + } + } + return rv; + } + const char* lbuf_; + } visitor(lbuf_); + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + if (myrand(3) == 0 && !cur->step() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + if (myrand(3) == 0 && !cur->step_back() && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::step"); + err_ = true; + } + } + break; + } + default: { + size_t rsiz; + char* rbuf = db_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "DB::get"); + err_ = true; + } + break; + } + } + } while (myrand(100) == 0); + if (myrand(100) == 0) { + int32_t jnum = myrand(10); + switch (myrand(4)) { + case 0: { + std::map<std::string, std::string> recs; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + recs[std::string(jbuf, jsiz)] = std::string(kbuf, ksiz); + } + if (db_->set_bulk(recs, myrand(4)) != (int64_t)recs.size()) { + dberrprint(db_, __LINE__, "DB::set_bulk"); + err_ = true; + } + break; + } + case 1: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + if (db_->remove_bulk(keys, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::remove_bulk"); + err_ = true; + } + break; + } + default: { + std::vector<std::string> keys; + for (int32_t j = 0; j < jnum; j++) { + char jbuf[RECBUFSIZ]; + size_t jsiz = std::sprintf(jbuf, "%lld", (long long)(myrand(range) + 1)); + keys.push_back(std::string(jbuf, jsiz)); + } + std::map<std::string, std::string> recs; + if (db_->get_bulk(keys, &recs, myrand(4)) < 0) { + dberrprint(db_, __LINE__, "DB::get_bulk"); + err_ = true; + } + break; + } + } + } + if (i == rnum_ / 2) { + if (myrand(thnum_ * 4) == 0) { + if (myrand(2) == 0) { + if (!db_->defrag(0)) { + dberrprint(db_, __LINE__, "DB::defrag"); + err_ = true; + } + } else { + if (!db_->clear()) { + dberrprint(db_, __LINE__, "DB::clear"); + err_ = true; + } + } + } else { + class SyncProcessor : public kc::BasicDB::FileProcessor { + private: + bool process(const std::string& path, int64_t count, int64_t size) { + yield(); + return true; + } + } syncprocessor; + if (!db_->synchronize(false, &syncprocessor)) { + dberrprint(db_, __LINE__, "DB::synchronize"); + err_ = true; + } + } + } + if (tran) { + yield(); + if (!db_->end_transaction(myrand(10) > 0)) { + dberrprint(db_, __LINE__, "DB::end_transactin"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + delete cur; + } + private: + int32_t id_; + kc::TreeDB* db_; + int64_t rnum_; + int32_t thnum_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadWicked threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, rnum, thnum, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, rnum, thnum, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform tran command +static int32_t proctran(const char* path, int64_t rnum, int32_t thnum, int32_t itnum, bool hard, + int32_t oflags, int32_t apow, int32_t fpow, int32_t opts, int64_t bnum, + int32_t psiz, int64_t msiz, int64_t dfunit, int64_t pccap, + kc::Comparator* rcomp, bool lv) { + oprintf("<Transaction Test>\n seed=%u path=%s rnum=%lld thnum=%d itnum=%d hard=%d" + " oflags=%d apow=%d fpow=%d opts=%d bnum=%lld psiz=%d msiz=%lld" + " dfunit=%lld pccap=%lld rcomp=%p lv=%d\n\n", + g_randseed, path, (long long)rnum, thnum, itnum, hard, oflags, apow, fpow, opts, + (long long)bnum, psiz, (long long)msiz, (long long)dfunit, (long long)pccap, + rcomp, lv); + bool err = false; + kc::TreeDB db; + kc::TreeDB paradb; + db.tune_logger(stdlogger(g_progname, &std::cout), + lv ? kc::UINT32MAX : kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + paradb.tune_logger(stdlogger(g_progname, &std::cout), lv ? kc::UINT32MAX : + kc::BasicDB::Logger::WARN | kc::BasicDB::Logger::ERROR); + if (apow >= 0) db.tune_alignment(apow); + if (fpow >= 0) db.tune_fbp(fpow); + if (opts > 0) db.tune_options(opts); + if (bnum > 0) db.tune_buckets(bnum); + if (psiz > 0) db.tune_page(psiz); + if (msiz >= 0) db.tune_map(msiz); + if (dfunit > 0) db.tune_defrag(dfunit); + if (pccap > 0) db.tune_page_cache(pccap); + if (rcomp) db.tune_comparator(rcomp); + for (int32_t itcnt = 1; itcnt <= itnum; itcnt++) { + oprintf("iteration %d updating:\n", itcnt); + double stime = kc::time(); + uint32_t omode = kc::TreeDB::OWRITER | kc::TreeDB::OCREATE; + if (itcnt == 1) omode |= kc::TreeDB::OTRUNCATE; + if (!db.open(path, omode | oflags)) { + dberrprint(&db, __LINE__, "DB::open"); + err = true; + } + std::string parapath = db.path() + "-para"; + if (!paradb.open(parapath, omode)) { + dberrprint(¶db, __LINE__, "DB::open"); + err = true; + } + class ThreadTran : public kc::Thread { + public: + void setparams(int32_t id, kc::TreeDB* db, kc::TreeDB* paradb, int64_t rnum, + int32_t thnum, bool hard, const char* lbuf) { + id_ = id; + db_ = db; + paradb_ = paradb; + rnum_ = rnum; + thnum_ = thnum; + hard_ = hard; + lbuf_ = lbuf; + err_ = false; + } + bool error() { + return err_; + } + void run() { + kc::DB::Cursor* cur = db_->cursor(); + int64_t range = rnum_ * thnum_; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz) && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + bool tran = true; + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + bool commit = myrand(10) > 0; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + const char* vbuf = kbuf; + size_t vsiz = ksiz; + if (myrand(10) == 0) { + vbuf = lbuf_; + vsiz = myrand(RECBUFSIZL) / (myrand(5) + 1); + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(const char* vbuf, size_t vsiz, kc::BasicDB* paradb) : + vbuf_(vbuf), vsiz_(vsiz), paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + return visit_empty(kbuf, ksiz, sp); + } + const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { + const char* rv = NOP; + switch (myrand(3)) { + case 0: { + rv = vbuf_; + *sp = vsiz_; + if (paradb_) paradb_->set(kbuf, ksiz, vbuf_, vsiz_); + break; + } + case 1: { + rv = REMOVE; + if (paradb_) paradb_->remove(kbuf, ksiz); + break; + } + } + return rv; + } + const char* vbuf_; + size_t vsiz_; + kc::BasicDB* paradb_; + } visitor(vbuf, vsiz, !tran || commit ? paradb_ : NULL); + if (myrand(4) == 0) { + if (!cur->accept(&visitor, true, myrand(2) == 0) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(kbuf, ksiz, &visitor, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + if (myrand(1000) == 0) { + ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(range) + 1)); + if (!cur->jump(kbuf, ksiz)) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } else if (!cur->jump() && db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + } + std::vector<std::string> keys; + keys.reserve(100); + while (myrand(50) != 0) { + std::string key; + if (cur->get_key(&key)) { + keys.push_back(key); + if (!cur->get_value(&key) && kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_value"); + err_ = true; + } + } else { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::get_key"); + err_ = true; + } + break; + } + if (!cur->step()) { + if (db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::jump"); + err_ = true; + } + break; + } + } + class Remover : public kc::DB::Visitor { + public: + explicit Remover(kc::BasicDB* paradb) : paradb_(paradb) {} + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + if (myrand(200) == 0) return NOP; + if (paradb_) paradb_->remove(kbuf, ksiz); + return REMOVE; + } + kc::BasicDB* paradb_; + } remover(!tran || commit ? paradb_ : NULL); + std::vector<std::string>::iterator it = keys.begin(); + std::vector<std::string>::iterator end = keys.end(); + while (it != end) { + if (myrand(50) == 0) { + if (!cur->accept(&remover, true, false) && + db_->error() != kc::BasicDB::Error::NOREC) { + dberrprint(db_, __LINE__, "Cursor::accept"); + err_ = true; + } + } else { + if (!db_->accept(it->c_str(), it->size(), &remover, true)) { + dberrprint(db_, __LINE__, "DB::accept"); + err_ = true; + } + } + ++it; + } + } + if (tran && myrand(100) == 0) { + if (db_->end_transaction(commit)) { + yield(); + if (!db_->begin_transaction(hard_)) { + dberrprint(db_, __LINE__, "DB::begin_transaction"); + tran = false; + err_ = true; + } + } else { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + if (tran && !db_->end_transaction(commit)) { + dberrprint(db_, __LINE__, "DB::end_transaction"); + err_ = true; + } + delete cur; + } + private: + int32_t id_; + kc::TreeDB* db_; + kc::TreeDB* paradb_; + int64_t rnum_; + int32_t thnum_; + bool hard_; + const char* lbuf_; + bool err_; + }; + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + ThreadTran threads[THREADMAX]; + if (thnum < 2) { + threads[0].setparams(0, &db, ¶db, rnum, thnum, hard, lbuf); + threads[0].run(); + if (threads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threads[i].setparams(i, &db, ¶db, rnum, thnum, hard, lbuf); + threads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threads[i].join(); + if (threads[i].error()) err = true; + } + } + oprintf("iteration %d checking:\n", itcnt); + if (db.count() != paradb.count()) { + dberrprint(&db, __LINE__, "DB::count"); + err = true; + } + class VisitorImpl : public kc::DB::Visitor { + public: + explicit VisitorImpl(int64_t rnum, kc::BasicDB* paradb) : + rnum_(rnum), paradb_(paradb), err_(false), cnt_(0) {} + bool error() { + return err_; + } + private: + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t* sp) { + cnt_++; + size_t rsiz; + char* rbuf = paradb_->get(kbuf, ksiz, &rsiz); + if (rbuf) { + delete[] rbuf; + } else { + dberrprint(paradb_, __LINE__, "DB::get"); + err_ = true; + } + if (rnum_ > 250 && cnt_ % (rnum_ / 250) == 0) { + oputchar('.'); + if (cnt_ == rnum_ || cnt_ % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt_); + } + return NOP; + } + int64_t rnum_; + kc::BasicDB* paradb_; + bool err_; + int64_t cnt_; + } visitor(rnum, ¶db), paravisitor(rnum, &db); + if (!db.iterate(&visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (visitor.error()) err = true; + if (!paradb.iterate(¶visitor, false)) { + dberrprint(&db, __LINE__, "DB::iterate"); + err = true; + } + oprintf(" (end)\n"); + if (paravisitor.error()) err = true; + if (!paradb.close()) { + dberrprint(¶db, __LINE__, "DB::close"); + err = true; + } + dbmetaprint(&db, itcnt == itnum); + if (!db.close()) { + dberrprint(&db, __LINE__, "DB::close"); + err = true; + } + oprintf("time: %.3f\n", kc::time() - stime); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcutil.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcutil.cc new file mode 100644 index 0000000000..f46c6f5ce7 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcutil.cc @@ -0,0 +1,389 @@ +/************************************************************************************************* + * Utility functions + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "kcutil.h" +#include "myconf.h" + +namespace kyotocabinet { // common namespace + + +/** The package version. */ +const char* const VERSION = _KC_VERSION; + + +/** The library version. */ +const int32_t LIBVER = _KC_LIBVER; + + +/** The library revision. */ +const int32_t LIBREV = _KC_LIBREV; + + +/** The database format version. */ +const int32_t FMTVER = _KC_FMTVER; + + +/** The system name. */ +const char* const OSNAME = _KC_OSNAME; + + +/** The flag for big endian environments. */ +const bool BIGEND = _KC_BIGEND ? true : false; + + +/** The clock tick of interruption. */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +const int32_t CLOCKTICK = 100; +#else +const int32_t CLOCKTICK = sysconf(_SC_CLK_TCK); +#endif + + +/** The size of a page. */ +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) +static int32_t win_getpagesize() { + ::SYSTEM_INFO ibuf; + ::GetSystemInfo(&ibuf); + return ibuf.dwPageSize; +} +const int32_t PAGESIZ = win_getpagesize(); +#else +const int32_t PAGESIZ = sysconf(_SC_PAGESIZE); +#endif + + +/** The extra feature list. */ +const char* const FEATURES = "" +#if _KC_GCCATOMIC + "(atomic)" +#endif +#if _KC_ZLIB + "(zlib)" +#endif +#if _KC_LZO + "(lzo)" +#endif +#if _KC_LZMA + "(lzma)" +#endif + ; + + +// get the levenshtein distance of two arrays +template<class CHARTYPE, class CNTTYPE> +static size_t levdist(const CHARTYPE* abuf, size_t asiz, const CHARTYPE* bbuf, size_t bsiz) { + size_t dsiz = bsiz + 1; + size_t tsiz = (asiz + 1) * dsiz; + CNTTYPE tblstack[2048/sizeof(CNTTYPE)]; + CNTTYPE* tbl = tsiz > sizeof(tblstack) / sizeof(*tblstack) ? new CNTTYPE[tsiz] : tblstack; + tbl[0] = 0; + for (size_t i = 1; i <= asiz; i++) { + tbl[i*dsiz] = i; + } + for (size_t i = 1; i <= bsiz; i++) { + tbl[i] = i; + } + abuf--; + bbuf--; + for (size_t i = 1; i <= asiz; i++) { + for (size_t j = 1; j <= bsiz; j++) { + uint32_t ac = tbl[(i-1)*dsiz+j] + 1; + uint32_t bc = tbl[i*dsiz+j-1] + 1; + uint32_t cc = tbl[(i-1)*dsiz+j-1] + (abuf[i] != bbuf[j]); + ac = ac < bc ? ac : bc; + tbl[i*dsiz+j] = ac < cc ? ac : cc; + } + } + size_t ed = tbl[asiz*dsiz+bsiz]; + if (tbl != tblstack) delete[] tbl; + return ed; +} + + +/** + * Calculate the levenshtein distance of two regions in bytes. + */ +size_t memdist(const void* abuf, size_t asiz, const void* bbuf, size_t bsiz) { + _assert_(abuf && asiz <= MEMMAXSIZ && bbuf && bsiz <= MEMMAXSIZ); + return asiz > UINT8MAX || bsiz > UINT8MAX ? + levdist<const char, uint32_t>((const char*)abuf, asiz, (const char*)bbuf, bsiz) : + levdist<const char, uint8_t>((const char*)abuf, asiz, (const char*)bbuf, bsiz); +} + + +/** + * Calculate the levenshtein distance of two UTF-8 strings. + */ +size_t strutfdist(const char* astr, const char* bstr) { + _assert_(astr && bstr); + size_t anum = strutflen(astr); + uint32_t astack[128]; + uint32_t* aary = anum > sizeof(astack) / sizeof(*astack) ? new uint32_t[anum] : astack; + strutftoucs(astr, aary, &anum); + size_t bnum = strutflen(bstr); + uint32_t bstack[128]; + uint32_t* bary = bnum > sizeof(bstack) / sizeof(*bstack) ? new uint32_t[bnum] : bstack; + strutftoucs(bstr, bary, &bnum); + size_t dist = strucsdist(aary, anum, bary, bnum); + if (bary != bstack) delete[] bary; + if (aary != astack) delete[] aary; + return dist; +} + + +/** + * Calculate the levenshtein distance of two UCS-4 arrays. + */ +size_t strucsdist(const uint32_t* aary, size_t anum, const uint32_t* bary, size_t bnum) { + _assert_(aary && anum <= MEMMAXSIZ && bary && bnum <= MEMMAXSIZ); + return anum > UINT8MAX || bnum > UINT8MAX ? + levdist<const uint32_t, uint32_t>(aary, anum, bary, bnum) : + levdist<const uint32_t, uint8_t>(aary, anum, bary, bnum); +} + + +/** + * Allocate a nullified region on memory. + */ +void* mapalloc(size_t size) { +#if defined(_SYS_LINUX_) + _assert_(size > 0 && size <= MEMMAXSIZ); + void* ptr = ::mmap(0, sizeof(size) + size, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) throw std::bad_alloc(); + *(size_t*)ptr = size; + return (char*)ptr + sizeof(size); +#else + _assert_(size > 0 && size <= MEMMAXSIZ); + void* ptr = std::calloc(size, 1); + if (!ptr) throw std::bad_alloc(); + return ptr; +#endif +} + + +/** + * Free a region on memory. + */ +void mapfree(void* ptr) { +#if defined(_SYS_LINUX_) + _assert_(ptr); + size_t size = *((size_t*)ptr - 1); + ::munmap((char*)ptr - sizeof(size), sizeof(size) + size); +#else + _assert_(ptr); + std::free(ptr); +#endif +} + + +/** + * Get the time of day in seconds. + * @return the time of day in seconds. The accuracy is in microseconds. + */ +double time() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + ::FILETIME ft; + ::GetSystemTimeAsFileTime(&ft); + ::LARGE_INTEGER li; + li.LowPart = ft.dwLowDateTime; + li.HighPart = ft.dwHighDateTime; + return li.QuadPart / 10000000.0; +#else + _assert_(true); + struct ::timeval tv; + if (::gettimeofday(&tv, NULL) != 0) return 0.0; + return tv.tv_sec + tv.tv_usec / 1000000.0; +#endif +} + + +/** + * Get the process ID. + */ +int64_t getpid() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + return ::GetCurrentProcessId(); +#else + _assert_(true); + return ::getpid(); +#endif +} + + +/** + * Get the value of an environment variable. + */ +const char* getenv(const char* name) { + _assert_(name); + return ::getenv(name); +} + + +/** + * Get system information of the environment. + */ +void getsysinfo(std::map<std::string, std::string>* strmap) { +#if defined(_SYS_LINUX_) + _assert_(strmap); + struct ::rusage rbuf; + std::memset(&rbuf, 0, sizeof(rbuf)); + if (::getrusage(RUSAGE_SELF, &rbuf) == 0) { + (*strmap)["ru_utime"] = strprintf("%0.6f", + rbuf.ru_utime.tv_sec + rbuf.ru_utime.tv_usec / 1000000.0); + (*strmap)["ru_stime"] = strprintf("%0.6f", + rbuf.ru_stime.tv_sec + rbuf.ru_stime.tv_usec / 1000000.0); + if (rbuf.ru_maxrss > 0) { + int64_t size = rbuf.ru_maxrss * 1024LL; + (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); + (*strmap)["mem_size"] = strprintf("%lld", (long long)size); + (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); + } + } + std::ifstream ifs; + ifs.open("/proc/self/status", std::ios_base::in | std::ios_base::binary); + if (ifs) { + std::string line; + while (getline(ifs, line)) { + size_t idx = line.find(':'); + if (idx != std::string::npos) { + const std::string& name = line.substr(0, idx); + idx++; + while (idx < line.size() && line[idx] >= '\0' && line[idx] <= ' ') { + idx++; + } + const std::string& value = line.substr(idx); + if (name == "VmPeak") { + int64_t size = atoix(value.c_str()); + if (size > 0) (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); + } else if (name == "VmSize") { + int64_t size = atoix(value.c_str()); + if (size > 0) (*strmap)["mem_size"] = strprintf("%lld", (long long)size); + } else if (name == "VmRSS") { + int64_t size = atoix(value.c_str()); + if (size > 0) (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); + } + } + } + ifs.close(); + } + ifs.open("/proc/meminfo", std::ios_base::in | std::ios_base::binary); + if (ifs) { + std::string line; + while (getline(ifs, line)) { + size_t idx = line.find(':'); + if (idx != std::string::npos) { + const std::string& name = line.substr(0, idx); + idx++; + while (idx < line.size() && line[idx] >= '\0' && line[idx] <= ' ') { + idx++; + } + const std::string& value = line.substr(idx); + if (name == "MemTotal") { + int64_t size = atoix(value.c_str()); + if (size > 0) (*strmap)["mem_total"] = strprintf("%lld", (long long)size); + } else if (name == "MemFree") { + int64_t size = atoix(value.c_str()); + if (size > 0) (*strmap)["mem_free"] = strprintf("%lld", (long long)size); + } else if (name == "Cached") { + int64_t size = atoix(value.c_str()); + if (size > 0) (*strmap)["mem_cached"] = strprintf("%lld", (long long)size); + } + } + } + ifs.close(); + } +#elif defined(_SYS_MACOSX_) + _assert_(strmap); + struct ::rusage rbuf; + std::memset(&rbuf, 0, sizeof(rbuf)); + if (::getrusage(RUSAGE_SELF, &rbuf) == 0) { + (*strmap)["ru_utime"] = strprintf("%0.6f", + rbuf.ru_utime.tv_sec + rbuf.ru_utime.tv_usec / 1000000.0); + (*strmap)["ru_stime"] = strprintf("%0.6f", + rbuf.ru_stime.tv_sec + rbuf.ru_stime.tv_usec / 1000000.0); + if (rbuf.ru_maxrss > 0) { + int64_t size = rbuf.ru_maxrss; + (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); + (*strmap)["mem_size"] = strprintf("%lld", (long long)size); + (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); + } + } +#elif defined(_SYS_FREEBSD_) || defined(_SYS_SUNOS_) + _assert_(strmap); + struct ::rusage rbuf; + std::memset(&rbuf, 0, sizeof(rbuf)); + if (::getrusage(RUSAGE_SELF, &rbuf) == 0) { + (*strmap)["ru_utime"] = strprintf("%0.6f", + rbuf.ru_utime.tv_sec + rbuf.ru_utime.tv_usec / 1000000.0); + (*strmap)["ru_stime"] = strprintf("%0.6f", + rbuf.ru_stime.tv_sec + rbuf.ru_stime.tv_usec / 1000000.0); + if (rbuf.ru_maxrss > 0) { + int64_t size = rbuf.ru_maxrss * 1024LL; + (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); + (*strmap)["mem_size"] = strprintf("%lld", (long long)size); + (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); + } + } +#elif defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(strmap); + ::DWORD pid = ::GetCurrentProcessId(); + ::HANDLE ph = ::OpenProcess(PROCESS_QUERY_INFORMATION, false, pid); + if (ph) { + ::FILETIME ct, et, kt, ut; + if (::GetProcessTimes(ph, &ct, &et, &kt, &ut)) { + ::LARGE_INTEGER li; + li.LowPart = ut.dwLowDateTime; + li.HighPart = ut.dwHighDateTime; + (*strmap)["ru_utime"] = strprintf("%0.6f", li.QuadPart / 10000000.0); + li.LowPart = kt.dwLowDateTime; + li.HighPart = kt.dwHighDateTime; + (*strmap)["ru_stime"] = strprintf("%0.6f", li.QuadPart / 10000000.0); + } + ::CloseHandle(ph); + } + ::MEMORYSTATUSEX msbuf; + msbuf.dwLength = sizeof(msbuf); + ::GlobalMemoryStatusEx(&msbuf); + (*strmap)["mem_total"] = strprintf("%lld", (long long)msbuf.ullTotalPhys); + (*strmap)["mem_free"] = strprintf("%lld", (long long)msbuf.ullAvailPhys); + int64_t cached = msbuf.ullTotalPhys - msbuf.ullAvailPhys; + (*strmap)["mem_cached"] = strprintf("%lld", (long long)cached); +#else + _assert_(strmap); +#endif +} + + +/** + * Set the standard streams into the binary mode. + */ +void setstdiobin() { +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + _assert_(true); + _setmode(_fileno(stdin), O_BINARY); + _setmode(_fileno(stdout), O_BINARY); + _setmode(_fileno(stderr), O_BINARY); +#else + _assert_(true); +#endif +} + + +} // common namespace + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcutil.h b/plugins/Dbx_kyoto/src/kyotocabinet/kcutil.h new file mode 100644 index 0000000000..8e3f74786a --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcutil.h @@ -0,0 +1,2924 @@ +/************************************************************************************************* + * Utility functions + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _KCUTIL_H // duplication check +#define _KCUTIL_H + +#include <kccommon.h> + +namespace kyotocabinet { // common namespace + + +/** The maximum value of int8_t. */ +const int8_t INT8MAX = (std::numeric_limits<int8_t>::max)(); + + +/** The maximum value of int16_t. */ +const int16_t INT16MAX = (std::numeric_limits<int16_t>::max)(); + + +/** The maximum value of int32_t. */ +const int32_t INT32MAX = (std::numeric_limits<int32_t>::max)(); + + +/** The maximum value of int64_t. */ +const int64_t INT64MAX = (std::numeric_limits<int64_t>::max)(); + + +/** The minimum value of int8_t. */ +const int8_t INT8MIN = (std::numeric_limits<int8_t>::min)(); + + +/** The minimum value of int16_t. */ +const int16_t INT16MIN = (std::numeric_limits<int16_t>::min)(); + + +/** The minimum value of int32_t. */ +const int32_t INT32MIN = (std::numeric_limits<int32_t>::min)(); + + +/** The minimum value of int64_t. */ +const int64_t INT64MIN = (std::numeric_limits<int64_t>::min)(); + + +/** The maximum value of uint8_t. */ +const uint8_t UINT8MAX = (std::numeric_limits<uint8_t>::max)(); + + +/** The maximum value of uint16_t. */ +const uint16_t UINT16MAX = (std::numeric_limits<uint16_t>::max)(); + + +/** The maximum value of uint32_t. */ +const uint32_t UINT32MAX = (std::numeric_limits<uint32_t>::max)(); + + +/** The maximum value of uint64_t. */ +const uint64_t UINT64MAX = (std::numeric_limits<uint64_t>::max)(); + + +/** The maximum value of size_t. */ +const size_t SIZEMAX = (std::numeric_limits<size_t>::max)(); + + +/** The maximum value of float. */ +const float FLTMAX = (std::numeric_limits<float>::max)(); + + +/** The maximum value of double. */ +const double DBLMAX = (std::numeric_limits<double>::max)(); + + +/** An alias of hash map of strings. */ +typedef std::unordered_map<std::string, std::string> StringHashMap; + + +/** An alias of tree map of strings. */ +typedef std::map<std::string, std::string> StringTreeMap; + + +/** The package version. */ +extern const char* const VERSION; + + +/** The library version. */ +extern const int32_t LIBVER; + + +/** The library revision. */ +extern const int32_t LIBREV; + + +/** The database format version. */ +extern const int32_t FMTVER; + + +/** The system name. */ +extern const char* const OSNAME; + + +/** The flag for big endian environments. */ +extern const bool BIGEND; + + +/** The clock tick of interruption. */ +extern const int32_t CLOCKTICK; + + +/** The size of a page. */ +extern const int32_t PAGESIZ; + + +/** The extra feature list. */ +extern const char* const FEATURES; + + +/** The buffer size for numeric data. */ +const size_t NUMBUFSIZ = 32; + + +/** The maximum memory size for debugging. */ +const size_t MEMMAXSIZ = INT32MAX / 2; + + +/** + * Convert a decimal string to an integer. + * @param str the decimal string. + * @return the integer. If the string does not contain numeric expression, 0 is returned. + */ +int64_t atoi(const char* str); + + +/** + * Convert a decimal string with a metric prefix to an integer. + * @param str the decimal string, which can be trailed by a binary metric prefix. "K", "M", "G", + * "T", "P", and "E" are supported. They are case-insensitive. + * @return the integer. If the string does not contain numeric expression, 0 is returned. If + * the integer overflows the domain, kyotocabinet::INT64MAX or kyotocabinet::INT64_MIN is + * returned according to the sign. + */ +int64_t atoix(const char* str); + + +/** + * Convert a hexadecimal string to an integer. + * @param str the hexadecimal string. + * @return the integer. If the string does not contain numeric expression, 0 is returned. + */ +int64_t atoih(const char* str); + + +/** + * Convert a decimal byte array to an integer. + * @param ptr the decimal byte array. + * @param size the size of the decimal byte array. + * @return the integer. If the string does not contain numeric expression, 0 is returned. + */ +int64_t atoin(const char* ptr, size_t size); + + +/** + * Convert a decimal string to a real number. + * @param str the decimal string. + * @return the real number. If the string does not contain numeric expression, 0.0 is returned. + */ +double atof(const char* str); + + +/** + * Convert a decimal byte array to a real number. + * @param ptr the decimal byte array. + * @param size the size of the decimal byte array. + * @return the real number. If the string does not contain numeric expression, 0.0 is returned. + */ +double atofn(const char* ptr, size_t size); + + +/** + * Normalize a 16-bit number in the native order into the network byte order. + * @param num the 16-bit number in the native order. + * @return the number in the network byte order. + */ +uint16_t hton16(uint16_t num); + + +/** + * Normalize a 32-bit number in the native order into the network byte order. + * @param num the 32-bit number in the native order. + * @return the number in the network byte order. + */ +uint32_t hton32(uint32_t num); + + +/** + * Normalize a 64-bit number in the native order into the network byte order. + * @param num the 64-bit number in the native order. + * @return the number in the network byte order. + */ +uint64_t hton64(uint64_t num); + + +/** + * Denormalize a 16-bit number in the network byte order into the native order. + * @param num the 16-bit number in the network byte order. + * @return the converted number in the native order. + */ +uint16_t ntoh16(uint16_t num); + + +/** + * Denormalize a 32-bit number in the network byte order into the native order. + * @param num the 32-bit number in the network byte order. + * @return the converted number in the native order. + */ +uint32_t ntoh32(uint32_t num); + + +/** + * Denormalize a 64-bit number in the network byte order into the native order. + * @param num the 64-bit number in the network byte order. + * @return the converted number in the native order. + */ +uint64_t ntoh64(uint64_t num); + + +/** + * Write a number in fixed length format into a buffer. + * @param buf the desitination buffer. + * @param num the number. + * @param width the width. + */ +void writefixnum(void* buf, uint64_t num, size_t width); + + +/** + * Read a number in fixed length format from a buffer. + * @param buf the source buffer. + * @param width the width. + * @return the read number. + */ +uint64_t readfixnum(const void* buf, size_t width); + + +/** + * Write a number in variable length format into a buffer. + * @param buf the desitination buffer. + * @param num the number. + * @return the length of the written region. + */ +size_t writevarnum(void* buf, uint64_t num); + + +/** + * Read a number in variable length format from a buffer. + * @param buf the source buffer. + * @param size the size of the source buffer. + * @param np the pointer to the variable into which the read number is assigned. + * @return the length of the read region, or 0 on failure. + */ +size_t readvarnum(const void* buf, size_t size, uint64_t* np); + + +/** + * Check the size of variable length format of a number. + * @return the size of variable length format. + */ +size_t sizevarnum(uint64_t num); + + +/** + * Get the hash value by MurMur hashing. + * @param buf the source buffer. + * @param size the size of the source buffer. + * @return the hash value. + */ +uint64_t hashmurmur(const void* buf, size_t size); + + +/** + * Get the hash value by FNV hashing. + * @param buf the source buffer. + * @param size the size of the source buffer. + * @return the hash value. + */ +uint64_t hashfnv(const void* buf, size_t size); + + +/** + * Get the hash value suitable for a file name. + * @param buf the source buffer. + * @param size the size of the source buffer. + * @param obuf the buffer into which the result hash string is written. It must be more than + * NUMBUFSIZ. + * @return the auxiliary hash value. + */ +uint32_t hashpath(const void* buf, size_t size, char* obuf); + + +/** + * Get a prime number nearby a number. + * @param num a natural number. + * @return the result number. + */ +uint64_t nearbyprime(uint64_t num); + + +/** + * Get the quiet Not-a-Number value. + * @return the quiet Not-a-Number value. + */ +double nan(); + + +/** + * Get the positive infinity value. + * @return the positive infinity value. + */ +double inf(); + + +/** + * Check a number is a Not-a-Number value. + * @return true for the number is a Not-a-Number value, or false if not. + */ +bool chknan(double num); + + +/** + * Check a number is an infinity value. + * @return true for the number is an infinity value, or false if not. + */ +bool chkinf(double num); + + +/** + * Append a formatted string at the end of a string. + * @param dest the destination string. + * @param format the printf-like format string. The conversion character `%' can be used with + * such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f', `g', `G', and `%'. + * @param ap used according to the format string. + */ +void vstrprintf(std::string* dest, const char* format, va_list ap); + + +/** + * Append a formatted string at the end of a string. + * @param dest the destination string. + * @param format the printf-like format string. The conversion character `%' can be used with + * such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f', `g', `G', and `%'. + * @param ... used according to the format string. + */ +void strprintf(std::string* dest, const char* format, ...); + + +/** + * Generate a formatted string. + * @param format the printf-like format string. The conversion character `%' can be used with + * such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f', `g', `G', and `%'. + * @param ... used according to the format string. + * @return the result string. + */ +std::string strprintf(const char* format, ...); + + +/** + * Split a string with a delimiter. + * @param str the string. + * @param delim the delimiter. + * @param elems a vector object into which the result elements are pushed. + * @return the number of result elements. + */ +size_t strsplit(const std::string& str, char delim, std::vector<std::string>* elems); + + +/** + * Split a string with delimiters. + * @param str the string. + * @param delims the delimiters. + * @param elems a vector object into which the result elements are pushed. + * @return the number of result elements. + */ +size_t strsplit(const std::string& str, const std::string& delims, + std::vector<std::string>* elems); + + +/** + * Convert the letters of a string into upper case. + * @param str the string to convert. + * @return the string itself. + */ +std::string* strtoupper(std::string* str); + + +/** + * Convert the letters of a string into lower case. + * @param str the string to convert. + * @return the string itself. + */ +std::string* strtolower(std::string* str); + + +/** + * Check whether a string begins with a key. + * @param str the string. + * @param key the forward matching key string. + * @return true if the target string begins with the key, else, it is false. + */ +bool strfwm(const std::string& str, const std::string& key); + + +/** + * Check whether a string ends with a key. + * @param str the string. + * @param key the backward matching key string. + * @return true if the target string ends with the key, else, it is false. + */ +bool strbwm(const std::string& str, const std::string& key); + + +/** + * Cut space characters at head or tail of a string. + * @param str the string to convert. + * @return the string itself. + */ +std::string* strtrim(std::string* str); + + +/** + * Convert a UTF-8 string into a UCS-4 array. + * @param src the source object. + * @param dest the destination object. + */ +void strutftoucs(const std::string& src, std::vector<uint32_t>* dest); + + +/** + * Convert a UCS-4 array into a UTF-8 string. + * @param src the source object. + * @param dest the destination object. + */ +void strucstoutf(const std::vector<uint32_t>& src, std::string* dest); + + +/** + * Serialize a string vector object into a string object. + * @param src the source object. + * @param dest the destination object. + */ +void strvecdump(const std::vector<std::string>& src, std::string* dest); + + +/** + * Deserialize a string object into a string vector object. + * @param src the source object. + * @param dest the destination object. + */ +void strvecload(const std::string& src, std::vector<std::string>* dest); + + +/** + * Serialize a string vector object into a string object. + * @param src the source object. + * @param dest the destination object. + */ +void strmapdump(const std::map<std::string, std::string>& src, std::string* dest); + + +/** + * Deserialize a string object into a string map object. + * @param src the source object. + * @param dest the destination object. + */ +void strmapload(const std::string& src, std::map<std::string, std::string>* dest); + + +/** + * Encode a serial object by hexadecimal encoding. + * @param buf the pointer to the region. + * @param size the size of the region. + * @return the result string. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ +char* hexencode(const void* buf, size_t size); + + +/** + * Decode a string encoded by hexadecimal encoding. + * @param str specifies the encoded string. + * @param sp the pointer to the variable into which the size of the region of the return value + * is assigned. + * @return the pointer to the region of the result. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a character string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the delete[] + * operator when it is no longer in use. + */ +char* hexdecode(const char* str, size_t* sp); + + +/** + * Encode a serial object by URL encoding. + * @param buf the pointer to the region. + * @param size the size of the region. + * @return the result string. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ +char* urlencode(const void* buf, size_t size); + + +/** + * Decode a string encoded by URL encoding. + * @param str specifies the encoded string. + * @param sp the pointer to the variable into which the size of the region of the return value + * is assigned. + * @return the pointer to the region of the result. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a character string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the delete[] + * operator when it is no longer in use. + */ +char* urldecode(const char* str, size_t* sp); + + +/** + * Encode a serial object by Quoted-printable encoding. + * @param buf the pointer to the region. + * @param size the size of the region. + * @return the result string. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ +char* quoteencode(const void* buf, size_t size); + + +/** + * Decode a string encoded by Quoted-printable encoding. + * @param str specifies the encoded string. + * @param sp the pointer to the variable into which the size of the region of the return value + * is assigned. + * @return the pointer to the region of the result. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a character string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the delete[] + * operator when it is no longer in use. + */ +char* quotedecode(const char* str, size_t* sp); + + +/** + * Encode a serial object by Base64 encoding. + * @param buf the pointer to the region. + * @param size the size of the region. + * @return the result string. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ +char* baseencode(const void* buf, size_t size); + + +/** + * Decode a string encoded by Base64 encoding. + * @param str specifies the encoded string. + * @param sp the pointer to the variable into which the size of the region of the return value + * is assigned. + * @return the pointer to the region of the result. + * @note Because an additional zero code is appended at the end of the region of the return + * value, the return value can be treated as a character string. Because the region of the + * return value is allocated with the the new[] operator, it should be released with the delete[] + * operator when it is no longer in use. + */ +char* basedecode(const char* str, size_t* sp); + + +/** + * Cipher or decipher a serial object with the Arcfour stream cipher. + * @param ptr the pointer to the region. + * @param size the size of the region. + * @param kbuf the pointer to the region of the cipher key. + * @param ksiz the size of the region of the cipher key. + * @param obuf the pointer to the region into which the result data is written. The size of the + * buffer should be equal to or more than the input region. The region can be the same as the + * source region. + */ +void arccipher(const void* ptr, size_t size, const void* kbuf, size_t ksiz, void* obuf); + + +/** + * Duplicate a region on memory. + * @param ptr the source buffer. + * @param size the size of the source buffer. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ +char* memdup(const char* ptr, size_t size); + + +/** + * Compare two regions by case insensitive evaluation. + * @param abuf a buffer. + * @param bbuf the other buffer. + * @param size the size of each buffer. + * @return positive if the former is big, negative if the latter is big, 0 if both are + * equivalent. + */ +int32_t memicmp(const void* abuf, const void* bbuf, size_t size); + + +/** + * Find the first occurrence of a sub pattern. + * @param hbuf the target pattern buffer. + * @param hsiz the size of the target pattern buffer. + * @param nbuf the sub pattern buffer. + * @param nsiz the size of the sub pattern buffer. + * @return the pointer to the beginning of the sub pattern in the target pattern buffer, or NULL + * if the sub pattern is not found. + */ +void* memmem(const void* hbuf, size_t hsiz, const void* nbuf, size_t nsiz); + + +/** + * Find the first occurrence of a sub pattern by case insensitive evaluation. + * @param hbuf the target pattern buffer. + * @param hsiz the size of the target pattern buffer. + * @param nbuf the sub pattern buffer. + * @param nsiz the size of the sub pattern buffer. + * @return the pointer to the beginning of the sub pattern in the target pattern buffer, or NULL + * if the sub pattern is not found. + */ +void* memimem(const void* hbuf, size_t hsiz, const void* nbuf, size_t nsiz); + + +/** + * Calculate the levenshtein distance of two regions in bytes. + * @param abuf the pointer to the region of one buffer. + * @param asiz the size of the region of one buffer. + * @param bbuf the pointer to the region of the other buffer. + * @param bsiz the size of the region of the other buffer. + * @return the levenshtein distance of two regions. + */ +size_t memdist(const void* abuf, size_t asiz, const void* bbuf, size_t bsiz); + + +/** + * Duplicate a string on memory. + * @param str the source string. + * @note Because the region of the return value is allocated with the the new[] operator, it + * should be released with the delete[] operator when it is no longer in use. + */ +char* strdup(const char* str); + + +/** + * Convert the letters of a string into upper case. + * @param str the string to convert. + * @return the string itself. + */ +char* strtoupper(char* str); + + +/** + * Convert the letters of a string into lower case. + * @param str the string to convert. + * @return the string itself. + */ +char* strtolower(char* str); + + +/** + * Cut space characters at head or tail of a string. + * @param str the string to convert. + * @return the string itself. + */ +char* strtrim(char* str); + + +/** + * Squeeze space characters in a string and trim it. + * @param str the string to convert. + * @return the string itself. + */ +char* strsqzspc(char* str); + + +/** + * Normalize space characters in a string and trim it. + * @param str the string to convert. + * @return the string itself. + */ +char* strnrmspc(char* str); + + +/** + * Compare two strings by case insensitive evaluation. + * @param astr a string. + * @param bstr the other string. + * @return positive if the former is big, negative if the latter is big, 0 if both are + * equivalent. + */ +int32_t stricmp(const char* astr, const char* bstr); + + +/** + * Find the first occurrence of a substring by case insensitive evaluation. + * @param hstr the target string. + * @param nstr the substring. + * @return the pointer to the beginning of the substring in the target string, or NULL if the + * substring is not found. + */ +char* stristr(const char* hstr, const char* nstr); + + +/** + * Check whether a string begins with a key. + * @param str the string. + * @param key the forward matching key string. + * @return true if the target string begins with the key, else, it is false. + */ +bool strfwm(const char* str, const char* key); + + +/** + * Check whether a string begins with a key by case insensitive evaluation. + * @param str the string. + * @param key the forward matching key string. + * @return true if the target string begins with the key, else, it is false. + */ +bool strifwm(const char* str, const char* key); + + +/** + * Check whether a string ends with a key. + * @param str the string. + * @param key the backward matching key string. + * @return true if the target string ends with the key, else, it is false. + */ +bool strbwm(const char* str, const char* key); + + +/** + * Check whether a string ends with a key by case insensitive evaluation. + * @param str the string. + * @param key the backward matching key string. + * @return true if the target string ends with the key, else, it is false. + */ +bool stribwm(const char* str, const char* key); + + +/** + * Get the number of characters in a UTF-8 string. + * @param str the UTF-8 string. + * @return the number of characters in the string. + */ +size_t strutflen(const char* str); + + +/** + * Convert a UTF-8 string into a UCS-4 array. + * @param src the source object. + * @param dest the destination object. It must have enough size. + * @param np the pointer to the variable into which the number of elements in the destination + * object is assgined. + */ +void strutftoucs(const char* src, uint32_t* dest, size_t* np); + + +/** + * Convert a UTF-8 string into a UCS-4 array. + * @param src the source object which does not have to be trailed by zero code. + * @param slen the length of the source object. + * @param dest the destination object. It must have enough size. + * @param np the pointer to the variable into which the number of elements in the destination + * object is assgined. + */ +void strutftoucs(const char* src, size_t slen, uint32_t* dest, size_t* np); + + +/** + * Convert a UCS-4 array into a UTF-8 string. + * @param src the source object. + * @param snum the number of elements in the source object. + * @param dest the destination object. It must have enough size. + * @return the size of the result string. + */ +size_t strucstoutf(const uint32_t* src, size_t snum, char* dest); + + +/** + * Calculate the levenshtein distance of two UTF-8 strings. + * @param astr one UTF-8 string. + * @param bstr the other UTF-8 string. + * @return the levenshtein distance of two arrays. + */ +size_t strutfdist(const char* astr, const char* bstr); + + +/** + * Calculate the levenshtein distance of two UCS-4 arrays. + * @param aary one UCS-4 array. + * @param anum the number of elements of one array. + * @param bary the other UCS-4 array. + * @param bnum the number of elements of the other array. + * @return the levenshtein distance of two arrays. + */ +size_t strucsdist(const uint32_t* aary, size_t anum, const uint32_t* bary, size_t bnum); + + +/** + * Allocate a region on memory. + * @param size the size of the region. + * @return the pointer to the allocated region. + */ +void* xmalloc(size_t size); + + +/** + * Allocate a nullified region on memory. + * @param nmemb the number of elements. + * @param size the size of each element. + * @return the pointer to the allocated region. + */ +void* xcalloc(size_t nmemb, size_t size); + + +/** + * Re-allocate a region on memory. + * @param ptr the pointer to the region. + * @param size the size of the region. + * @return the pointer to the re-allocated region. + */ +void* xrealloc(void* ptr, size_t size); + + +/** + * Free a region on memory. + * @param ptr the pointer to the region. + */ +void xfree(void* ptr); + + +/** + * Allocate a nullified region on mapped memory. + * @param size the size of the region. + * @return the pointer to the allocated region. It should be released with the memfree call. + */ +void* mapalloc(size_t size); + + +/** + * Free a region on mapped memory. + * @param ptr the pointer to the allocated region. + */ +void mapfree(void* ptr); + + +/** + * Get the time of day in seconds. + * @return the time of day in seconds. The accuracy is in microseconds. + */ +double time(); + + +/** + * Get the process ID. + * @return the process ID. + */ +int64_t getpid(); + + +/** + * Get the value of an environment variable. + * @return the value of the environment variable, or NULL on failure. + */ +const char* getenv(const char* name); + + +/** + * Get system information of the environment. + * @param strmap a string map to contain the result. + */ +void getsysinfo(std::map<std::string, std::string>* strmap); + + +/** + * Set the standard streams into the binary mode. + */ +void setstdiobin(); + + +/** + * Dummy test driver. + * @return always true. + */ +bool _dummytest(); + + +/** + * Convert a decimal string to an integer. + */ +inline int64_t atoi(const char* str) { + _assert_(str); + while (*str > '\0' && *str <= ' ') { + str++; + } + int32_t sign = 1; + int64_t num = 0; + if (*str == '-') { + str++; + sign = -1; + } else if (*str == '+') { + str++; + } + while (*str != '\0') { + if (*str < '0' || *str > '9') break; + num = num * 10 + *str - '0'; + str++; + } + return num * sign; +} + + +/** + * Convert a decimal string with a metric prefix to an integer. + */ +inline int64_t atoix(const char* str) { + _assert_(str); + while (*str > '\0' && *str <= ' ') { + str++; + } + int32_t sign = 1; + if (*str == '-') { + str++; + sign = -1; + } else if (*str == '+') { + str++; + } + long double num = 0; + while (*str != '\0') { + if (*str < '0' || *str > '9') break; + num = num * 10 + *str - '0'; + str++; + } + if (*str == '.') { + str++; + long double base = 10; + while (*str != '\0') { + if (*str < '0' || *str > '9') break; + num += (*str - '0') / base; + str++; + base *= 10; + } + } + num *= sign; + while (*str > '\0' && *str <= ' ') { + str++; + } + if (*str == 'k' || *str == 'K') { + num *= 1LL << 10; + } else if (*str == 'm' || *str == 'M') { + num *= 1LL << 20; + } else if (*str == 'g' || *str == 'G') { + num *= 1LL << 30; + } else if (*str == 't' || *str == 'T') { + num *= 1LL << 40; + } else if (*str == 'p' || *str == 'P') { + num *= 1LL << 50; + } else if (*str == 'e' || *str == 'E') { + num *= 1LL << 60; + } + if (num > INT64MAX) return INT64MAX; + if (num < INT64MIN) return INT64MIN; + return (int64_t)num; +} + + +/** + * Convert a hexadecimal string to an integer. + */ +inline int64_t atoih(const char* str) { + _assert_(str); + while (*str > '\0' && *str <= ' ') { + str++; + } + if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) { + str += 2; + } + int64_t num = 0; + while (true) { + if (*str >= '0' && *str <= '9') { + num = num * 0x10 + *str - '0'; + } else if (*str >= 'a' && *str <= 'f') { + num = num * 0x10 + *str - 'a' + 10; + } else if (*str >= 'A' && *str <= 'F') { + num = num * 0x10 + *str - 'A' + 10; + } else { + break; + } + str++; + } + return num; +} + + +/** + * Convert a decimal byte array to an integer. + */ +inline int64_t atoin(const char* ptr, size_t size) { + _assert_(ptr && size <= MEMMAXSIZ); + while (size > 0 && *ptr >= '\0' && *ptr <= ' ') { + ptr++; + size--; + } + int32_t sign = 1; + int64_t num = 0; + if (size > 0) { + if (*ptr == '-') { + ptr++; + size--; + sign = -1; + } else if (*ptr == '+') { + ptr++; + size--; + } + } + while (size > 0) { + if (*ptr < '0' || *ptr > '9') break; + num = num * 10 + *ptr - '0'; + ptr++; + size--; + } + return num * sign; +} + + +/** + * Convert a decimal string to a real number. + */ +inline double atof(const char* str) { + _assert_(str); + while (*str > '\0' && *str <= ' ') { + str++; + } + int32_t sign = 1; + if (*str == '-') { + str++; + sign = -1; + } else if (*str == '+') { + str++; + } + if ((str[0] == 'i' || str[0] == 'I') && (str[1] == 'n' || str[1] == 'N') && + (str[2] == 'f' || str[2] == 'F')) return HUGE_VAL * sign; + if ((str[0] == 'n' || str[0] == 'N') && (str[1] == 'a' || str[1] == 'A') && + (str[2] == 'n' || str[2] == 'N')) return nan(); + long double num = 0; + int32_t col = 0; + while (*str != '\0') { + if (*str < '0' || *str > '9') break; + num = num * 10 + *str - '0'; + str++; + if (num > 0) col++; + } + if (*str == '.') { + str++; + long double fract = 0.0; + long double base = 10; + while (col < 16 && *str != '\0') { + if (*str < '0' || *str > '9') break; + fract += (*str - '0') / base; + str++; + col++; + base *= 10; + } + num += fract; + } + if (*str == 'e' || *str == 'E') { + str++; + num *= std::pow((long double)10, (long double)atoi(str)); + } + return num * sign; +} + + +/** + * Convert a decimal byte array to a real number. + */ +inline double atofn(const char* ptr, size_t size) { + _assert_(ptr && size <= MEMMAXSIZ); + while (size > 0 && *ptr >= '\0' && *ptr <= ' ') { + ptr++; + size--; + } + int32_t sign = 1; + if (size > 0) { + if (*ptr == '-') { + ptr++; + size--; + sign = -1; + } else if (*ptr == '+') { + ptr++; + size--; + } + } + if (size > 2) { + if ((ptr[0] == 'i' || ptr[0] == 'I') && (ptr[1] == 'n' || ptr[1] == 'N') && + (ptr[2] == 'f' || ptr[2] == 'F')) return HUGE_VAL * sign; + if ((ptr[0] == 'n' || ptr[0] == 'N') && (ptr[1] == 'a' || ptr[1] == 'A') && + (ptr[2] == 'n' || ptr[2] == 'N')) return nan(); + } + long double num = 0; + int32_t col = 0; + while (size > 0) { + if (*ptr < '0' || *ptr > '9') break; + num = num * 10 + *ptr - '0'; + ptr++; + size--; + if (num > 0) col++; + } + if (size > 0 && *ptr == '.') { + ptr++; + size--; + long double fract = 0.0; + long double base = 10; + while (col < 16 && size > 0) { + if (*ptr < '0' || *ptr > '9') break; + fract += (*ptr - '0') / base; + ptr++; + size--; + col++; + base *= 10; + } + num += fract; + } + if (size > 0 && (*ptr == 'e' || *ptr == 'E')) { + ptr++; + size--; + num *= std::pow((long double)10, (long double)atoin(ptr, size)); + } + return num * sign; +} + + + +/** + * Normalize a 16-bit number in the native order into the network byte order. + */ +inline uint16_t hton16(uint16_t num) { + _assert_(true); + if (BIGEND) return num; + return ((num & 0x00ffU) << 8) | ((num & 0xff00U) >> 8); +} + + +/** + * Normalize a 32-bit number in the native order into the network byte order. + */ +inline uint32_t hton32(uint32_t num) { + _assert_(true); + if (BIGEND) return num; + return ((num & 0x000000ffUL) << 24) | ((num & 0x0000ff00UL) << 8) | \ + ((num & 0x00ff0000UL) >> 8) | ((num & 0xff000000UL) >> 24); +} + + +/** + * Normalize a 64-bit number in the native order into the network byte order. + */ +inline uint64_t hton64(uint64_t num) { + _assert_(true); + if (BIGEND) return num; + return ((num & 0x00000000000000ffULL) << 56) | ((num & 0x000000000000ff00ULL) << 40) | + ((num & 0x0000000000ff0000ULL) << 24) | ((num & 0x00000000ff000000ULL) << 8) | + ((num & 0x000000ff00000000ULL) >> 8) | ((num & 0x0000ff0000000000ULL) >> 24) | + ((num & 0x00ff000000000000ULL) >> 40) | ((num & 0xff00000000000000ULL) >> 56); +} + + +/** + * Denormalize a 16-bit number in the network byte order into the native order. + */ +inline uint16_t ntoh16(uint16_t num) { + _assert_(true); + return hton16(num); +} + + +/** + * Denormalize a 32-bit number in the network byte order into the native order. + */ +inline uint32_t ntoh32(uint32_t num) { + _assert_(true); + return hton32(num); +} + + +/** + * Denormalize a 64-bit number in the network byte order into the native order. + */ +inline uint64_t ntoh64(uint64_t num) { + _assert_(true); + return hton64(num); +} + + +/** + * Write a number in fixed length format into a buffer. + */ +inline void writefixnum(void* buf, uint64_t num, size_t width) { + _assert_(buf && width <= sizeof(int64_t)); + num = hton64(num); + std::memcpy(buf, (const char*)&num + sizeof(num) - width, width); +} + + +/** + * Read a number in fixed length format from a buffer. + */ +inline uint64_t readfixnum(const void* buf, size_t width) { + _assert_(buf && width <= sizeof(int64_t)); + uint64_t num = 0; + std::memcpy(&num, buf, width); + return ntoh64(num) >> ((sizeof(num) - width) * 8); +} + + +/** + * Write a number in variable length format into a buffer. + */ +inline size_t writevarnum(void* buf, uint64_t num) { + _assert_(buf); + unsigned char* wp = (unsigned char*)buf; + if (num < (1ULL << 7)) { + *(wp++) = num; + } else if (num < (1ULL << 14)) { + *(wp++) = (num >> 7) | 0x80; + *(wp++) = num & 0x7f; + } else if (num < (1ULL << 21)) { + *(wp++) = (num >> 14) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } else if (num < (1ULL << 28)) { + *(wp++) = (num >> 21) | 0x80; + *(wp++) = ((num >> 14) & 0x7f) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } else if (num < (1ULL << 35)) { + *(wp++) = (num >> 28) | 0x80; + *(wp++) = ((num >> 21) & 0x7f) | 0x80; + *(wp++) = ((num >> 14) & 0x7f) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } else if (num < (1ULL << 42)) { + *(wp++) = (num >> 35) | 0x80; + *(wp++) = ((num >> 28) & 0x7f) | 0x80; + *(wp++) = ((num >> 21) & 0x7f) | 0x80; + *(wp++) = ((num >> 14) & 0x7f) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } else if (num < (1ULL << 49)) { + *(wp++) = (num >> 42) | 0x80; + *(wp++) = ((num >> 35) & 0x7f) | 0x80; + *(wp++) = ((num >> 28) & 0x7f) | 0x80; + *(wp++) = ((num >> 21) & 0x7f) | 0x80; + *(wp++) = ((num >> 14) & 0x7f) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } else if (num < (1ULL << 56)) { + *(wp++) = (num >> 49) | 0x80; + *(wp++) = ((num >> 42) & 0x7f) | 0x80; + *(wp++) = ((num >> 35) & 0x7f) | 0x80; + *(wp++) = ((num >> 28) & 0x7f) | 0x80; + *(wp++) = ((num >> 21) & 0x7f) | 0x80; + *(wp++) = ((num >> 14) & 0x7f) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } else if (num < (1ULL << 63)) { + *(wp++) = (num >> 56) | 0x80; + *(wp++) = ((num >> 49) & 0x7f) | 0x80; + *(wp++) = ((num >> 42) & 0x7f) | 0x80; + *(wp++) = ((num >> 35) & 0x7f) | 0x80; + *(wp++) = ((num >> 28) & 0x7f) | 0x80; + *(wp++) = ((num >> 21) & 0x7f) | 0x80; + *(wp++) = ((num >> 14) & 0x7f) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } else { + *(wp++) = (num >> 63) | 0x80; + *(wp++) = ((num >> 56) & 0x7f) | 0x80; + *(wp++) = ((num >> 49) & 0x7f) | 0x80; + *(wp++) = ((num >> 42) & 0x7f) | 0x80; + *(wp++) = ((num >> 35) & 0x7f) | 0x80; + *(wp++) = ((num >> 28) & 0x7f) | 0x80; + *(wp++) = ((num >> 21) & 0x7f) | 0x80; + *(wp++) = ((num >> 14) & 0x7f) | 0x80; + *(wp++) = ((num >> 7) & 0x7f) | 0x80; + *(wp++) = num & 0x7f; + } + return wp - (unsigned char*)buf; +} + + +/** + * Read a number in variable length format from a buffer. + */ +inline size_t readvarnum(const void* buf, size_t size, uint64_t* np) { + _assert_(buf && size <= MEMMAXSIZ && np); + const unsigned char* rp = (const unsigned char*)buf; + const unsigned char* ep = rp + size; + uint64_t num = 0; + uint32_t c; + do { + if (rp >= ep) { + *np = 0; + return 0; + } + c = *rp; + num = (num << 7) + (c & 0x7f); + rp++; + } while (c >= 0x80); + *np = num; + return rp - (const unsigned char*)buf; +} + + +/** + * Check the size of variable length format of a number. + */ +inline size_t sizevarnum(uint64_t num) { + _assert_(true); + if (num < (1ULL << 7)) return 1; + if (num < (1ULL << 14)) return 2; + if (num < (1ULL << 21)) return 3; + if (num < (1ULL << 28)) return 4; + if (num < (1ULL << 35)) return 5; + if (num < (1ULL << 42)) return 6; + if (num < (1ULL << 49)) return 7; + if (num < (1ULL << 56)) return 8; + if (num < (1ULL << 63)) return 9; + return 10; +} + + +/** + * Get the hash value by MurMur hashing. + */ +inline uint64_t hashmurmur(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + const uint64_t mul = 0xc6a4a7935bd1e995ULL; + const int32_t rtt = 47; + uint64_t hash = 19780211ULL ^ (size * mul); + const unsigned char* rp = (const unsigned char*)buf; + while (size >= sizeof(uint64_t)) { + uint64_t num = ((uint64_t)rp[0] << 0) | ((uint64_t)rp[1] << 8) | + ((uint64_t)rp[2] << 16) | ((uint64_t)rp[3] << 24) | + ((uint64_t)rp[4] << 32) | ((uint64_t)rp[5] << 40) | + ((uint64_t)rp[6] << 48) | ((uint64_t)rp[7] << 56); + num *= mul; + num ^= num >> rtt; + num *= mul; + hash *= mul; + hash ^= num; + rp += sizeof(uint64_t); + size -= sizeof(uint64_t); + } + switch (size) { + case 7: hash ^= (uint64_t)rp[6] << 48; + case 6: hash ^= (uint64_t)rp[5] << 40; + case 5: hash ^= (uint64_t)rp[4] << 32; + case 4: hash ^= (uint64_t)rp[3] << 24; + case 3: hash ^= (uint64_t)rp[2] << 16; + case 2: hash ^= (uint64_t)rp[1] << 8; + case 1: hash ^= (uint64_t)rp[0]; + hash *= mul; + }; + hash ^= hash >> rtt; + hash *= mul; + hash ^= hash >> rtt; + return hash; +} + + +/** + * Get the hash value by FNV hashing. + */ +inline uint64_t hashfnv(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + uint64_t hash = 14695981039346656037ULL; + const unsigned char* rp = (unsigned char*)buf; + const unsigned char* ep = rp + size; + while (rp < ep) { + hash = (hash ^ *(rp++)) * 109951162811ULL; + } + return hash; +} + + +/** + * Get the hash value suitable for a file name. + */ +inline uint32_t hashpath(const void* buf, size_t size, char* obuf) { + _assert_(buf && size <= MEMMAXSIZ && obuf); + const unsigned char* rp = (const unsigned char*)buf; + uint32_t rv; + char* wp = obuf; + if (size <= 10) { + if (size > 0) { + const unsigned char* ep = rp + size; + while (rp < ep) { + int32_t num = *rp >> 4; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + num = *rp & 0x0f; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + rp++; + } + } else { + *(wp++) = '0'; + } + uint64_t hash = hashmurmur(buf, size); + rv = (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff00000000ULL) >> 16)) ^ + (((hash & 0x000000000000ffffULL) << 16) | ((hash & 0x00000000ffff0000ULL) >> 16)); + } else { + *(wp++) = 'f' + 1 + (size & 0x0f); + for (int32_t i = 0; i <= 6; i += 3) { + uint32_t num = (rp[i] ^ rp[i+1] ^ rp[i+2] ^ + rp[size-i-1] ^ rp[size-i-2] ^ rp[size-i-3]) % 36; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + } + uint64_t hash = hashmurmur(buf, size); + rv = (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff00000000ULL) >> 16)) ^ + (((hash & 0x000000000000ffffULL) << 16) | ((hash & 0x00000000ffff0000ULL) >> 16)); + uint64_t inc = hashfnv(buf, size); + inc = (((inc & 0xffff000000000000ULL) >> 48) | ((inc & 0x0000ffff00000000ULL) >> 16)) ^ + (((inc & 0x000000000000ffffULL) << 16) | ((inc & 0x00000000ffff0000ULL) >> 16)); + for (size_t i = 0; i < sizeof(hash); i++) { + uint32_t least = hash >> ((sizeof(hash) - 1) * 8); + uint64_t num = least >> 4; + if (inc & 0x01) num += 0x10; + inc = inc >> 1; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + num = least & 0x0f; + if (inc & 0x01) num += 0x10; + inc = inc >> 1; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + hash = hash << 8; + } + } + *wp = '\0'; + return rv; +} + + +/** + * Get a prime number nearby a number. + */ +inline uint64_t nearbyprime(uint64_t num) { + _assert_(true); + static uint64_t table[] = { + 2ULL, 3ULL, 5ULL, 7ULL, 11ULL, 13ULL, 17ULL, 19ULL, 23ULL, 29ULL, 31ULL, 37ULL, 41ULL, + 43ULL, 47ULL, 53ULL, 59ULL, 61ULL, 67ULL, 71ULL, 79ULL, 97ULL, 107ULL, 131ULL, 157ULL, + 181ULL, 223ULL, 257ULL, 307ULL, 367ULL, 431ULL, 521ULL, 613ULL, 727ULL, 863ULL, 1031ULL, + 1217ULL, 1451ULL, 1723ULL, 2053ULL, 2437ULL, 2897ULL, 3449ULL, 4099ULL, 4871ULL, 5801ULL, + 6899ULL, 8209ULL, 9743ULL, 11587ULL, 13781ULL, 16411ULL, 19483ULL, 23173ULL, 27581ULL, + 32771ULL, 38971ULL, 46349ULL, 55109ULL, 65537ULL, 77951ULL, 92681ULL, 110221ULL, 131101ULL, + 155887ULL, 185363ULL, 220447ULL, 262147ULL, 311743ULL, 370759ULL, 440893ULL, 524309ULL, + 623521ULL, 741457ULL, 881743ULL, 1048583ULL, 1246997ULL, 1482919ULL, 1763491ULL, + 2097169ULL, 2493949ULL, 2965847ULL, 3526987ULL, 4194319ULL, 4987901ULL, 5931641ULL, + 7053971ULL, 8388617ULL, 9975803ULL, 11863289ULL, 14107921ULL, 16777259ULL, 19951597ULL, + 23726569ULL, 28215809ULL, 33554467ULL, 39903197ULL, 47453149ULL, 56431657ULL, + 67108879ULL, 79806341ULL, 94906297ULL, 112863217ULL, 134217757ULL, 159612679ULL, + 189812533ULL, 225726419ULL, 268435459ULL, 319225391ULL, 379625083ULL, 451452839ULL, + 536870923ULL, 638450719ULL, 759250133ULL, 902905657ULL, 1073741827ULL, 1276901429ULL, + 1518500279ULL, 1805811341ULL, 2147483659ULL, 2553802871ULL, 3037000507ULL, 3611622607ULL, + 4294967311ULL, 5107605691ULL, 6074001001ULL, 7223245229ULL, 8589934609ULL, 10215211387ULL, + 12148002047ULL, 14446490449ULL, 17179869209ULL, 20430422699ULL, 24296004011ULL, + 28892980877ULL, 34359738421ULL, 40860845437ULL, 48592008053ULL, 57785961671ULL, + 68719476767ULL, 81721690807ULL, 97184016049ULL, 115571923303ULL, 137438953481ULL, + 163443381347ULL, 194368032011ULL, 231143846587ULL, 274877906951ULL, 326886762733ULL, + 388736063999ULL, 462287693167ULL, 549755813911ULL, 653773525393ULL, 777472128049ULL, + 924575386373ULL, 1099511627791ULL, 1307547050819ULL, 1554944255989ULL, 1849150772699ULL, + 2199023255579ULL, 2615094101561ULL, 3109888512037ULL, 3698301545321ULL, + 4398046511119ULL, 5230188203153ULL, 6219777023959ULL, 7396603090651ULL, + 8796093022237ULL, 10460376406273ULL, 12439554047911ULL, 14793206181251ULL, + 17592186044423ULL, 20920752812471ULL, 24879108095833ULL, 29586412362491ULL, + 35184372088891ULL, 41841505624973ULL, 49758216191633ULL, 59172824724919ULL, + 70368744177679ULL, 83683011249917ULL, 99516432383281ULL, 118345649449813ULL, + 140737488355333ULL, 167366022499847ULL, 199032864766447ULL, 236691298899683ULL, + 281474976710677ULL, 334732044999557ULL, 398065729532981ULL, 473382597799229ULL, + 562949953421381ULL, 669464089999087ULL, 796131459065743ULL, 946765195598473ULL, + 1125899906842679ULL, 1338928179998197ULL, 1592262918131449ULL, 1893530391196921ULL, + 2251799813685269ULL, 2677856359996339ULL, 3184525836262943ULL, 3787060782393821ULL, + 4503599627370517ULL, 5355712719992603ULL, 6369051672525833ULL, 7574121564787633ULL + }; + static const size_t tnum = sizeof(table) / sizeof(table[0]); + uint64_t* ub = std::lower_bound(table, table + tnum, num); + return ub == (uint64_t*)table + tnum ? num : *ub; +} + + +/** + * Get the quiet Not-a-Number value. + */ +inline double nan() { + _assert_(true); + return std::numeric_limits<double>::quiet_NaN(); +} + + +/** + * Get the positive infinity value. + */ +inline double inf() { + _assert_(true); + return std::numeric_limits<double>::infinity(); +} + + +/** + * Check a number is a Not-a-Number value. + */ +inline bool chknan(double num) { + _assert_(true); + return num != num; +} + + +/** + * Check a number is an infinity value. + */ +inline bool chkinf(double num) { + _assert_(true); + return num == inf() || num == -inf(); +} + + +/** + * Append a formatted string at the end of a string. + */ +inline void vstrprintf(std::string* dest, const char* format, va_list ap) { + _assert_(dest && format); + while (*format != '\0') { + if (*format == '%') { + char cbuf[NUMBUFSIZ]; + cbuf[0] = '%'; + size_t cbsiz = 1; + int32_t lnum = 0; + format++; + while (std::strchr("0123456789 .+-hlLz", *format) && *format != '\0' && + cbsiz < NUMBUFSIZ - 1) { + if (*format == 'l' || *format == 'L') lnum++; + cbuf[cbsiz++] = *(format++); + } + cbuf[cbsiz++] = *format; + cbuf[cbsiz] = '\0'; + switch (*format) { + case 's': { + const char* tmp = va_arg(ap, const char*); + if (tmp) { + dest->append(tmp); + } else { + dest->append("(null)"); + } + break; + } + case 'd': { + char tbuf[NUMBUFSIZ*4]; + size_t tsiz; + if (lnum >= 2) { + tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, long long)); + } else if (lnum >= 1) { + tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, long)); + } else { + tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, int)); + } + dest->append(tbuf, tsiz); + break; + } + case 'o': case 'u': case 'x': case 'X': case 'c': { + char tbuf[NUMBUFSIZ*4]; + size_t tsiz; + if (lnum >= 2) { + tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, unsigned long long)); + } else if (lnum >= 1) { + tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, unsigned long)); + } else { + tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, unsigned int)); + } + dest->append(tbuf, tsiz); + break; + } + case 'e': case 'E': case 'f': case 'g': case 'G': { + char tbuf[NUMBUFSIZ*4]; + size_t tsiz; + if (lnum >= 1) { + tsiz = std::snprintf(tbuf, sizeof(tbuf), cbuf, va_arg(ap, long double)); + } else { + tsiz = std::snprintf(tbuf, sizeof(tbuf), cbuf, va_arg(ap, double)); + } + if (tsiz > sizeof(tbuf)) { + tbuf[sizeof(tbuf)-1] = '*'; + tsiz = sizeof(tbuf); + } + dest->append(tbuf, tsiz); + break; + } + case 'p': { + char tbuf[NUMBUFSIZ*4]; + size_t tsiz = std::sprintf(tbuf, "%p", va_arg(ap, void*)); + dest->append(tbuf, tsiz); + break; + } + case '%': { + dest->append("%", 1); + break; + } + } + } else { + dest->append(format, 1); + } + format++; + } +} + + +/** + * Append a formatted string at the end of a string. + */ +inline void strprintf(std::string* dest, const char* format, ...) { + _assert_(dest && format); + va_list ap; + va_start(ap, format); + vstrprintf(dest, format, ap); + va_end(ap); +} + + +/** + * Generate a formatted string on memory. + */ +inline std::string strprintf(const char* format, ...) { + _assert_(format); + std::string str; + va_list ap; + va_start(ap, format); + vstrprintf(&str, format, ap); + va_end(ap); + return str; +} + + +/** + * Split a string with a delimiter + */ +inline size_t strsplit(const std::string& str, char delim, std::vector<std::string>* elems) { + _assert_(elems); + elems->clear(); + std::string::const_iterator it = str.begin(); + std::string::const_iterator pv = it; + while (it != str.end()) { + if (*it == delim) { + std::string col(pv, it); + elems->push_back(col); + pv = it + 1; + } + ++it; + } + std::string col(pv, it); + elems->push_back(col); + return elems->size(); +} + + +/** + * Split a string with delimiters. + */ +inline size_t strsplit(const std::string& str, const std::string& delims, + std::vector<std::string>* elems) { + _assert_(elems); + elems->clear(); + std::string::const_iterator it = str.begin(); + std::string::const_iterator pv = it; + while (it != str.end()) { + while (delims.find(*it, 0) != std::string::npos) { + std::string col(pv, it); + elems->push_back(col); + pv = it + 1; + break; + } + ++it; + } + std::string col(pv, it); + elems->push_back(col); + return elems->size(); +} + + +/** + * Convert the letters of a string into upper case. + */ +inline std::string* strtoupper(std::string* str) { + _assert_(str); + size_t size = str->size(); + for (size_t i = 0; i < size; i++) { + int32_t c = (unsigned char)(*str)[i]; + if (c >= 'a' && c <= 'z') (*str)[i] = c - ('a' - 'A'); + } + return str; +} + + +/** + * Convert the letters of a string into lower case. + */ +inline std::string* strtolower(std::string* str) { + _assert_(str); + size_t size = str->size(); + for (size_t i = 0; i < size; i++) { + int32_t c = (unsigned char)(*str)[i]; + if (c >= 'A' && c <= 'Z') (*str)[i] = c + ('a' - 'A'); + } + return str; +} + + +/** + * Check whether a string begins with a key. + */ +inline bool strfwm(const std::string& str, const std::string& key) { + _assert_(true); + size_t ksiz = key.size(); + if (ksiz > str.size()) return false; + return !std::memcmp(str.data(), key.data(), ksiz); +} + + +/** + * Check whether a string ends with a key. + */ +inline bool strbwm(const std::string& str, const std::string& key) { + _assert_(true); + size_t ksiz = key.size(); + if (ksiz > str.size()) return false; + return !std::memcmp(str.data() + str.size() - ksiz, key.data(), ksiz); +} + + +/** + * Cut space characters at head or tail of a string. + */ +inline std::string* strtrim(std::string* str) { + _assert_(str); + size_t size = str->size(); + size_t wi = 0; + size_t li = 0; + for (size_t i = 0; i < size; i++) { + int32_t c = (unsigned char)(*str)[i]; + if (c >= '\0' && c <= ' ') { + if (wi > 0) (*str)[wi++] = c; + } else { + (*str)[wi++] = c; + li = wi; + } + } + str->resize(li); + return str; +} + + +/** + * Convert a UTF-8 string into a UCS-4 array. + */ +inline void strutftoucs(const std::string& src, std::vector<uint32_t>* dest) { + _assert_(dest); + dest->reserve(dest->size() + src.size()); + size_t size = src.size(); + size_t ri = 0; + while (ri < size) { + uint32_t c = (unsigned char)src[ri]; + if (c < 0x80) { + dest->push_back(c); + } else if (c < 0xe0) { + if (c >= 0xc0 && ri + 1 < size) { + c = ((c & 0x1f) << 6) | (src[ri+1] & 0x3f); + if (c >= 0x80) dest->push_back(c); + ri++; + } + } else if (c < 0xf0) { + if (ri + 2 < size) { + c = ((c & 0x0f) << 12) | ((src[ri+1] & 0x3f) << 6) | (src[ri+2] & 0x3f); + if (c >= 0x800) dest->push_back(c); + ri += 2; + } + } else if (c < 0xf8) { + if (ri + 3 < size) { + c = ((c & 0x07) << 18) | ((src[ri+1] & 0x3f) << 12) | ((src[ri+2] & 0x3f) << 6) | + (src[ri+3] & 0x3f); + if (c >= 0x10000) dest->push_back(c); + ri += 3; + } + } else if (c < 0xfc) { + if (ri + 4 < size) { + c = ((c & 0x03) << 24) | ((src[ri+1] & 0x3f) << 18) | ((src[ri+2] & 0x3f) << 12) | + ((src[ri+3] & 0x3f) << 6) | (src[ri+4] & 0x3f); + if (c >= 0x200000) dest->push_back(c); + ri += 4; + } + } else if (c < 0xfe) { + if (ri + 5 < size) { + c = ((c & 0x01) << 30) | ((src[ri+1] & 0x3f) << 24) | ((src[ri+2] & 0x3f) << 18) | + ((src[ri+3] & 0x3f) << 12) | ((src[ri+4] & 0x3f) << 6) | (src[ri+5] & 0x3f); + if (c >= 0x4000000) dest->push_back(c); + ri += 5; + } + } + ri++; + } +} + + +/** + * Convert a UCS-4 array into a UTF-8 string. + */ +inline void strucstoutf(const std::vector<uint32_t>& src, std::string* dest) { + _assert_(dest); + dest->reserve(dest->size() + src.size() * 3); + std::vector<uint32_t>::const_iterator it = src.begin(); + std::vector<uint32_t>::const_iterator itend = src.end(); + while (it != itend) { + uint32_t c = *it; + if (c < 0x80) { + dest->append(1, c); + } else if (c < 0x800) { + dest->append(1, 0xc0 | (c >> 6)); + dest->append(1, 0x80 | (c & 0x3f)); + } else if (c < 0x10000) { + dest->append(1, 0xe0 | (c >> 12)); + dest->append(1, 0x80 | ((c & 0xfff) >> 6)); + dest->append(1, 0x80 | (c & 0x3f)); + } else if (c < 0x200000) { + dest->append(1, 0xf0 | (c >> 18)); + dest->append(1, 0x80 | ((c & 0x3ffff) >> 12)); + dest->append(1, 0x80 | ((c & 0xfff) >> 6)); + dest->append(1, 0x80 | (c & 0x3f)); + } else if (c < 0x4000000) { + dest->append(1, 0xf8 | (c >> 24)); + dest->append(1, 0x80 | ((c & 0xffffff) >> 18)); + dest->append(1, 0x80 | ((c & 0x3ffff) >> 12)); + dest->append(1, 0x80 | ((c & 0xfff) >> 6)); + dest->append(1, 0x80 | (c & 0x3f)); + } else if (c < 0x80000000) { + dest->append(1, 0xfc | (c >> 30)); + dest->append(1, 0x80 | ((c & 0x3fffffff) >> 24)); + dest->append(1, 0x80 | ((c & 0xffffff) >> 18)); + dest->append(1, 0x80 | ((c & 0x3ffff) >> 12)); + dest->append(1, 0x80 | ((c & 0xfff) >> 6)); + dest->append(1, 0x80 | (c & 0x3f)); + } + ++it; + } +} + + +/** + * Serialize a string vector object into a string object. + */ +inline void strvecdump(const std::vector<std::string>& src, std::string* dest) { + _assert_(dest); + std::vector<std::string>::const_iterator it = src.begin(); + std::vector<std::string>::const_iterator itend = src.end(); + size_t dsiz = 1; + while (it != itend) { + dsiz += 2 + it->size(); + ++it; + } + dest->reserve(dest->size() + dsiz); + it = src.begin(); + while (it != itend) { + char nbuf[NUMBUFSIZ]; + size_t nsiz = writevarnum(nbuf, it->size()); + dest->append(nbuf, nsiz); + dest->append(it->data(), it->size()); + ++it; + } +} + + +/** + * Deserialize a string object into a string vector object. + */ +inline void strvecload(const std::string& src, std::vector<std::string>* dest) { + _assert_(dest); + const char* rp = src.data(); + size_t size = src.size(); + while (size > 0) { + uint64_t vsiz; + size_t step = readvarnum(rp, size, &vsiz); + rp += step; + size -= step; + if (vsiz > size) break; + dest->push_back(std::string(rp, vsiz)); + rp += vsiz; + size -= vsiz; + } +} + + +/** + * Serialize a string vector object into a string object. + */ +inline void strmapdump(const std::map<std::string, std::string>& src, std::string* dest) { + _assert_(dest); + std::map<std::string, std::string>::const_iterator it = src.begin(); + std::map<std::string, std::string>::const_iterator itend = src.end(); + size_t dsiz = 1; + while (it != itend) { + dsiz += 4 + it->first.size() + it->second.size(); + ++it; + } + dest->reserve(dest->size() + dsiz); + it = src.begin(); + while (it != itend) { + char nbuf[NUMBUFSIZ*2]; + size_t nsiz = writevarnum(nbuf, it->first.size()); + nsiz += writevarnum(nbuf + nsiz, it->second.size()); + dest->append(nbuf, nsiz); + dest->append(it->first.data(), it->first.size()); + dest->append(it->second.data(), it->second.size()); + ++it; + } +} + + +/** + * Deserialize a string object into a string map object. + */ +inline void strmapload(const std::string& src, std::map<std::string, std::string>* dest) { + _assert_(dest); + const char* rp = src.data(); + int64_t size = src.size(); + while (size > 1) { + uint64_t ksiz; + size_t step = readvarnum(rp, size, &ksiz); + rp += step; + size -= step; + if (size < 1) break; + uint64_t vsiz; + step = readvarnum(rp, size, &vsiz); + rp += step; + size -= step; + int64_t rsiz = ksiz + vsiz; + if (rsiz > size) break; + (*dest)[std::string(rp, ksiz)] = std::string(rp + ksiz, vsiz); + rp += rsiz; + size -= rsiz; + } +} + + +/** + * Encode a serial object by hexadecimal encoding. + */ +inline char* hexencode(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + const unsigned char* rp = (const unsigned char*)buf; + char* zbuf = new char[size*2+1]; + char* wp = zbuf; + for (const unsigned char* ep = rp + size; rp < ep; rp++) { + int32_t num = *rp >> 4; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + num = *rp & 0x0f; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + } + *wp = '\0'; + return zbuf; +} + + +/** + * Decode a string encoded by hexadecimal encoding. + */ +inline char* hexdecode(const char* str, size_t* sp) { + _assert_(str && sp); + char* zbuf = new char[std::strlen(str)+1]; + char* wp = zbuf; + while (true) { + while (*str > '\0' && *str <= ' ') { + str++; + } + int32_t num = 0; + int32_t c = *(str++); + if (c >= '0' && c <= '9') { + num = c - '0'; + } else if (c >= 'a' && c <= 'f') { + num = c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + num = c - 'A' + 10; + } else if (c == '\0') { + break; + } + c = *(str++); + if (c >= '0' && c <= '9') { + num = num * 0x10 + c - '0'; + } else if (c >= 'a' && c <= 'f') { + num = num * 0x10 + c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + num = num * 0x10 + c - 'A' + 10; + } else if (c == '\0') { + *(wp++) = num; + break; + } + *(wp++) = num; + } + *wp = '\0'; + *sp = wp - zbuf; + return zbuf; +} + + +/** + * Encode a serial object by URL encoding. + */ +inline char* urlencode(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + const unsigned char* rp = (const unsigned char*)buf; + char* zbuf = new char[size*3+1]; + char* wp = zbuf; + for (const unsigned char* ep = rp + size; rp < ep; rp++) { + int32_t c = *rp; + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || (c != '\0' && std::strchr("_-.~", c))) { + *(wp++) = c; + } else { + *(wp++) = '%'; + int32_t num = c >> 4; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + num = c & 0x0f; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'a' + num - 10; + } + } + } + *wp = '\0'; + return zbuf; +} + + +/** + * Decode a string encoded by URL encoding. + */ +inline char* urldecode(const char* str, size_t* sp) { + _assert_(str && sp); + size_t zsiz = std::strlen(str); + char* zbuf = new char[zsiz+1]; + char* wp = zbuf; + const char* ep = str + zsiz; + while (str < ep) { + int32_t c = *str; + if (c == '%') { + int32_t num = 0; + if (++str >= ep) break; + c = *str; + if (c >= '0' && c <= '9') { + num = c - '0'; + } else if (c >= 'a' && c <= 'f') { + num = c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + num = c - 'A' + 10; + } + if (++str >= ep) break; + c = *str; + if (c >= '0' && c <= '9') { + num = num * 0x10 + c - '0'; + } else if (c >= 'a' && c <= 'f') { + num = num * 0x10 + c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + num = num * 0x10 + c - 'A' + 10; + } + *(wp++) = num; + str++; + } else if (c == '+') { + *(wp++) = ' '; + str++; + } else if (c <= ' ' || c == 0x7f) { + str++; + } else { + *(wp++) = c; + str++; + } + } + *wp = '\0'; + *sp = wp - zbuf; + return zbuf; +} + + +/** + * Encode a serial object by Quoted-printable encoding. + */ +inline char* quoteencode(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + const unsigned char* rp = (const unsigned char*)buf; + char* zbuf = new char[size*3+1]; + char* wp = zbuf; + for (const unsigned char* ep = rp + size; rp < ep; rp++) { + int32_t c = *rp; + if (c == '=' || c < ' ' || c > 0x7e) { + *(wp++) = '='; + int32_t num = c >> 4; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'A' + num - 10; + } + num = c & 0x0f; + if (num < 10) { + *(wp++) = '0' + num; + } else { + *(wp++) = 'A' + num - 10; + } + } else { + *(wp++) = c; + } + } + *wp = '\0'; + return zbuf; +} + + +/** + * Decode a string encoded by Quoted-printable encoding. + */ +inline char* quotedecode(const char* str, size_t* sp) { + _assert_(str && sp); + size_t zsiz = std::strlen(str); + char* zbuf = new char[zsiz+1]; + char* wp = zbuf; + const char* ep = str + zsiz; + while (str < ep) { + int32_t c = *str; + if (c == '=') { + int32_t num = 0; + if (++str >= ep) break; + c = *str; + if (c == '\r') { + if (++str >= ep) break; + if (*str == '\n') str++; + } else if (c == '\n') { + str++; + } else { + if (c >= '0' && c <= '9') { + num = c - '0'; + } else if (c >= 'a' && c <= 'f') { + num = c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + num = c - 'A' + 10; + } + if (++str >= ep) break; + c = *str; + if (c >= '0' && c <= '9') { + num = num * 0x10 + c - '0'; + } else if (c >= 'a' && c <= 'f') { + num = num * 0x10 + c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + num = num * 0x10 + c - 'A' + 10; + } + *(wp++) = num; + str++; + } + } else if (c < ' ' || c == 0x7f) { + str++; + } else { + *(wp++) = c; + str++; + } + } + *wp = '\0'; + *sp = wp - zbuf; + return zbuf; +} + + +/** + * Encode a serial object by Base64 encoding. + */ +inline char* baseencode(const void* buf, size_t size) { + _assert_(buf && size <= MEMMAXSIZ); + const char* tbl = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const unsigned char* rp = (const unsigned char*)buf; + char* zbuf = new char[size*4/3+5]; + char* wp = zbuf; + for (size_t i = 0; i < size; i += 3) { + switch (size - i) { + case 1: { + *(wp++) = tbl[rp[0] >> 2]; + *(wp++) = tbl[(rp[0] & 3) << 4]; + *(wp++) = '='; + *(wp++) = '='; + break; + } + case 2: { + *(wp++) = tbl[rp[0] >> 2]; + *(wp++) = tbl[((rp[0] & 3) << 4) + (rp[1] >> 4)]; + *(wp++) = tbl[(rp[1] & 0xf) << 2]; + *(wp++) = '='; + break; + } + default: { + *(wp++) = tbl[rp[0] >> 2]; + *(wp++) = tbl[((rp[0] & 3) << 4) + (rp[1] >> 4)]; + *(wp++) = tbl[((rp[1] & 0xf) << 2) + (rp[2] >> 6)]; + *(wp++) = tbl[rp[2] & 0x3f]; + break; + } + } + rp += 3; + } + *wp = '\0'; + return zbuf; +} + + +/** + * Decode a string encoded by Base64 encoding. + */ +inline char* basedecode(const char* str, size_t* sp) { + _assert_(str && sp); + size_t bpos = 0; + size_t eqcnt = 0; + size_t len = std::strlen(str); + unsigned char* zbuf = new unsigned char[len+4]; + unsigned char* wp = zbuf; + size_t zsiz = 0; + while (bpos < len && eqcnt == 0) { + size_t bits = 0; + size_t i; + for (i = 0; bpos < len && i < 4; bpos++) { + if (str[bpos] >= 'A' && str[bpos] <= 'Z') { + bits = (bits << 6) | (str[bpos] - 'A'); + i++; + } else if (str[bpos] >= 'a' && str[bpos] <= 'z') { + bits = (bits << 6) | (str[bpos] - 'a' + 26); + i++; + } else if (str[bpos] >= '0' && str[bpos] <= '9') { + bits = (bits << 6) | (str[bpos] - '0' + 52); + i++; + } else if (str[bpos] == '+') { + bits = (bits << 6) | 62; + i++; + } else if (str[bpos] == '/') { + bits = (bits << 6) | 63; + i++; + } else if (str[bpos] == '=') { + bits <<= 6; + i++; + eqcnt++; + } + } + if (i == 0 && bpos >= len) continue; + switch (eqcnt) { + case 0: { + *wp++ = (bits >> 16) & 0xff; + *wp++ = (bits >> 8) & 0xff; + *wp++ = bits & 0xff; + zsiz += 3; + break; + } + case 1: { + *wp++ = (bits >> 16) & 0xff; + *wp++ = (bits >> 8) & 0xff; + zsiz += 2; + break; + } + case 2: { + *wp++ = (bits >> 16) & 0xff; + zsiz += 1; + break; + } + } + } + zbuf[zsiz] = '\0'; + *sp = zsiz; + return (char*)zbuf; +} + + +/** + * Cipher or decipher a serial object with the Arcfour stream cipher. + */ +inline void arccipher(const void* ptr, size_t size, const void* kbuf, size_t ksiz, void* obuf) { + _assert_(ptr && size <= MEMMAXSIZ && kbuf && ksiz <= MEMMAXSIZ && obuf); + if (ksiz < 1) { + kbuf = ""; + ksiz = 1; + } + uint32_t sbox[0x100], kbox[0x100]; + for (int32_t i = 0; i < 0x100; i++) { + sbox[i] = i; + kbox[i] = ((uint8_t*)kbuf)[i%ksiz]; + } + uint32_t sidx = 0; + for (int32_t i = 0; i < 0x100; i++) { + sidx = (sidx + sbox[i] + kbox[i]) & 0xff; + uint32_t swap = sbox[i]; + sbox[i] = sbox[sidx]; + sbox[sidx] = swap; + } + uint32_t x = 0; + uint32_t y = 0; + for (size_t i = 0; i < size; i++) { + x = (x + 1) & 0xff; + y = (y + sbox[x]) & 0xff; + uint32_t swap = sbox[x]; + sbox[x] = sbox[y]; + sbox[y] = swap; + ((uint8_t*)obuf)[i] = ((uint8_t*)ptr)[i] ^ sbox[(sbox[x]+sbox[y])&0xff]; + } +} + + +/** + * Duplicate a region on memory. + */ +inline char* memdup(const char* ptr, size_t size) { + _assert_(ptr && size <= MEMMAXSIZ); + char* obuf = new char[size+1]; + std::memcpy(obuf, ptr, size); + return obuf; +} + + +/** + * Compare two regions by case insensitive evaluation. + */ +inline int32_t memicmp(const void* abuf, const void* bbuf, size_t size) { + _assert_(abuf && bbuf && size <= MEMMAXSIZ); + const unsigned char* ap = (unsigned char*)abuf; + const unsigned char* bp = (unsigned char*)bbuf; + const unsigned char* ep = ap + size; + while (ap < ep) { + int32_t ac = *ap; + if (ac >= 'A' && ac <= 'Z') ac += 'a' - 'A'; + int32_t bc = *bp; + if (bc >= 'A' && bc <= 'Z') bc += 'a' - 'A'; + if (ac != bc) return ac - bc; + ap++; + bp++; + } + return 0; +} + + +/** + * Find the first occurrence of a sub pattern. + */ +inline void* memmem(const void* hbuf, size_t hsiz, const void* nbuf, size_t nsiz) { + _assert_(hbuf && hsiz <= MEMMAXSIZ && nbuf && nsiz <= MEMMAXSIZ); + if (nsiz < 1) return (void*)hbuf; + if (hsiz < nsiz) return NULL; + int32_t tc = *(unsigned char*)nbuf; + const unsigned char* rp = (unsigned char*)hbuf; + const unsigned char* ep = (unsigned char*)hbuf + hsiz - nsiz; + while (rp <= ep) { + if (*rp == tc) { + bool hit = true; + for (size_t i = 1; i < nsiz; i++) { + if (rp[i] != ((unsigned char*)nbuf)[i]) { + hit = false; + break; + } + } + if (hit) return (void*)rp; + } + rp++; + } + return NULL; +} + + +/** + * Find the first occurrence of a sub pattern by case insensitive evaluation. + */ +inline void* memimem(const void* hbuf, size_t hsiz, const void* nbuf, size_t nsiz) { + _assert_(hbuf && hsiz <= MEMMAXSIZ && nbuf && nsiz <= MEMMAXSIZ); + if (nsiz < 1) return (void*)hbuf; + if (hsiz < nsiz) return NULL; + int32_t tc = *(unsigned char*)nbuf; + if (tc >= 'A' && tc <= 'Z') tc += 'a' - 'A'; + const unsigned char* rp = (unsigned char*)hbuf; + const unsigned char* ep = (unsigned char*)hbuf + hsiz - nsiz; + while (rp <= ep) { + int32_t cc = *rp; + if (cc >= 'A' && cc <= 'Z') cc += 'a' - 'A'; + if (cc == tc) { + bool hit = true; + for (size_t i = 1; i < nsiz; i++) { + int32_t hc = rp[i]; + if (hc >= 'A' && hc <= 'Z') hc += 'a' - 'A'; + int32_t nc = ((unsigned char*)nbuf)[i]; + if (nc >= 'A' && nc <= 'Z') nc += 'a' - 'A'; + if (hc != nc) { + hit = false; + break; + } + } + if (hit) return (void*)rp; + } + rp++; + } + return NULL; +} + + +/** + * Duplicate a string on memory. + */ +inline char* strdup(const char* str) { + _assert_(str); + size_t size = std::strlen(str); + char* obuf = memdup(str, size); + obuf[size] = '\0'; + return obuf; +} + + +/** + * Convert the letters of a string into upper case. + */ +inline char* strtoupper(char* str) { + _assert_(str); + char* wp = str; + while (*wp != '\0') { + if (*wp >= 'a' && *wp <= 'z') *wp -= 'a' - 'A'; + wp++; + } + return str; +} + + +/** + * Convert the letters of a string into lower case. + */ +inline char* strtolower(char* str) { + _assert_(str); + char* wp = str; + while (*wp != '\0') { + if (*wp >= 'A' && *wp <= 'Z') *wp += 'a' - 'A'; + wp++; + } + return str; +} + + +/** + * Cut space characters at head or tail of a string. + */ +inline char* strtrim(char* str) { + _assert_(str); + const char* rp = str; + char* wp = str; + bool head = true; + while (*rp != '\0') { + if (*rp > '\0' && *rp <= ' ') { + if (!head) *(wp++) = *rp; + } else { + *(wp++) = *rp; + head = false; + } + rp++; + } + *wp = '\0'; + while (wp > str && wp[-1] > '\0' && wp[-1] <= ' ') { + *(--wp) = '\0'; + } + return str; +} + + +/** + * Squeeze space characters in a string and trim it. + */ +inline char* strsqzspc(char* str) { + _assert_(str); + const char* rp = str; + char* wp = str; + bool spc = true; + while (*rp != '\0') { + if (*rp > '\0' && *rp <= ' ') { + if (!spc) *(wp++) = *rp; + spc = true; + } else { + *(wp++) = *rp; + spc = false; + } + rp++; + } + *wp = '\0'; + for (wp--; wp >= str; wp--) { + if (*wp > '\0' && *wp <= ' ') { + *wp = '\0'; + } else { + break; + } + } + return str; +} + + +/** + * Normalize space characters in a string and trim it. + */ +inline char* strnrmspc(char* str) { + _assert_(str); + const char* rp = str; + char* wp = str; + bool spc = true; + while (*rp != '\0') { + if ((*rp > '\0' && *rp <= ' ') || *rp == 0x7f) { + if (!spc) *(wp++) = ' '; + spc = true; + } else { + *(wp++) = *rp; + spc = false; + } + rp++; + } + *wp = '\0'; + for (wp--; wp >= str; wp--) { + if (*wp == ' ') { + *wp = '\0'; + } else { + break; + } + } + return str; +} + + + +/** + * Compare two strings by case insensitive evaluation. + */ +inline int32_t stricmp(const char* astr, const char* bstr) { + _assert_(astr && bstr); + while (*astr != '\0') { + if (*bstr == '\0') return 1; + int32_t ac = *(unsigned char*)astr; + if (ac >= 'A' && ac <= 'Z') ac += 'a' - 'A'; + int32_t bc = *(unsigned char*)bstr; + if (bc >= 'A' && bc <= 'Z') bc += 'a' - 'A'; + if (ac != bc) return ac - bc; + astr++; + bstr++; + } + return (*bstr == '\0') ? 0 : -1; +} + + +/** + * Find the first occurrence of a substring by case insensitive evaluation. + */ +inline char* stristr(const char* hstr, const char* nstr) { + _assert_(hstr && nstr); + if (*nstr == '\0') return (char*)hstr; + int32_t tc = *nstr; + if (tc >= 'A' && tc <= 'Z') tc += 'a' - 'A'; + const char* rp = hstr; + while (*rp != '\0') { + int32_t cc = *rp; + if (cc >= 'A' && cc <= 'Z') cc += 'a' - 'A'; + if (cc == tc) { + bool hit = true; + for (size_t i = 1; nstr[i] != '\0'; i++) { + int32_t hc = rp[i]; + if (hc >= 'A' && hc <= 'Z') hc += 'a' - 'A'; + int32_t nc = nstr[i]; + if (nc >= 'A' && nc <= 'Z') nc += 'a' - 'A'; + if (hc != nc) { + hit = false; + break; + } + } + if (hit) return (char*)rp; + } + rp++; + } + return NULL; +} + + +/** + * Check whether a string begins with a key. + */ +inline bool strfwm(const char* str, const char* key) { + _assert_(str && key); + while (*key != '\0') { + if (*str != *key || *str == '\0') return false; + key++; + str++; + } + return true; +} + + +/** + * Check whether a string begins with a key by case insensitive evaluation. + */ +inline bool strifwm(const char* str, const char* key) { + _assert_(str && key); + while (*key != '\0') { + if (*str == '\0') return false; + int32_t sc = *str; + if (sc >= 'A' && sc <= 'Z') sc += 'a' - 'A'; + int32_t kc = *key; + if (kc >= 'A' && kc <= 'Z') kc += 'a' - 'A'; + if (sc != kc) return false; + key++; + str++; + } + return true; +} + + +/** + * Check whether a string ends with a key. + */ +inline bool strbwm(const char* str, const char* key) { + _assert_(str && key); + size_t slen = std::strlen(str); + size_t klen = std::strlen(key); + for (size_t i = 1; i <= klen; i++) { + if (i > slen || str[slen-i] != key[klen-i]) return false; + } + return true; +} + + +/** + * Check whether a string ends with a key by case insensitive evaluation. + */ +inline bool stribwm(const char* str, const char* key) { + _assert_(str && key); + size_t slen = std::strlen(str); + size_t klen = std::strlen(key); + for (size_t i = 1; i <= klen; i++) { + if (i > slen) return false; + int32_t sc = str[slen-i]; + if (sc >= 'A' && sc <= 'Z') sc += 'a' - 'A'; + int32_t kc = key[klen-i]; + if (kc >= 'A' && kc <= 'Z') kc += 'a' - 'A'; + if (sc != kc) return false; + } + return true; +} + + +/** + * Get the number of characters in a UTF-8 string. + */ +inline size_t strutflen(const char* str) { + _assert_(str); + size_t len = 0; + while (*str != '\0') { + len += (*(unsigned char*)str & 0xc0) != 0x80; + str++; + } + return len; +} + + +/** + * Convert a UTF-8 string into a UCS-4 array. + */ +inline void strutftoucs(const char* src, uint32_t* dest, size_t* np) { + _assert_(src && dest && np); + const unsigned char* rp = (unsigned char*)src; + size_t dnum = 0; + while (*rp != '\0') { + uint32_t c = *rp; + if (c < 0x80) { + dest[dnum++] = c; + } else if (c < 0xe0) { + if (rp[1] != '\0') { + c = ((c & 0x1f) << 6) | (rp[1] & 0x3f); + if (c >= 0x80) dest[dnum++] = c; + rp++; + } + } else if (c < 0xf0) { + if (rp[1] != '\0' && rp[2] != '\0') { + c = ((c & 0x0f) << 12) | ((rp[1] & 0x3f) << 6) | (rp[2] & 0x3f); + if (c >= 0x800) dest[dnum++] = c; + rp += 2; + } + } else if (c < 0xf8) { + if (rp[1] != '\0' && rp[2] != '\0' && rp[3] != '\0') { + c = ((c & 0x07) << 18) | ((rp[1] & 0x3f) << 12) | ((rp[2] & 0x3f) << 6) | + (rp[3] & 0x3f); + if (c >= 0x10000) dest[dnum++] = c; + rp += 3; + } + } else if (c < 0xfc) { + if (rp[1] != '\0' && rp[2] != '\0' && rp[3] != '\0' && rp[4] != '\0') { + c = ((c & 0x03) << 24) | ((rp[1] & 0x3f) << 18) | ((rp[2] & 0x3f) << 12) | + ((rp[3] & 0x3f) << 6) | (rp[4] & 0x3f); + if (c >= 0x200000) dest[dnum++] = c; + rp += 4; + } + } else if (c < 0xfe) { + if (rp[1] != '\0' && rp[2] != '\0' && rp[3] != '\0' && rp[4] != '\0' && rp[5] != '\0') { + c = ((c & 0x01) << 30) | ((rp[1] & 0x3f) << 24) | ((rp[2] & 0x3f) << 18) | + ((rp[3] & 0x3f) << 12) | ((rp[4] & 0x3f) << 6) | (rp[5] & 0x3f); + if (c >= 0x4000000) dest[dnum++] = c; + rp += 5; + } + } + rp++; + } + *np = dnum; +} + + +/** + * Convert a UTF-8 string into a UCS-4 array. + */ +inline void strutftoucs(const char* src, size_t slen, uint32_t* dest, size_t* np) { + _assert_(src && slen <= MEMMAXSIZ && dest && np); + const unsigned char* rp = (unsigned char*)src; + const unsigned char* ep = rp + slen; + size_t dnum = 0; + while (rp < ep) { + uint32_t c = *rp; + if (c < 0x80) { + dest[dnum++] = c; + } else if (c < 0xe0) { + if (rp[1] != '\0') { + c = ((c & 0x1f) << 6) | (rp[1] & 0x3f); + if (c >= 0x80) dest[dnum++] = c; + rp++; + } + } else if (c < 0xf0) { + if (rp[1] != '\0' && rp[2] != '\0') { + c = ((c & 0x0f) << 12) | ((rp[1] & 0x3f) << 6) | (rp[2] & 0x3f); + if (c >= 0x800) dest[dnum++] = c; + rp += 2; + } + } else if (c < 0xf8) { + if (rp[1] != '\0' && rp[2] != '\0' && rp[3] != '\0') { + c = ((c & 0x07) << 18) | ((rp[1] & 0x3f) << 12) | ((rp[2] & 0x3f) << 6) | + (rp[3] & 0x3f); + if (c >= 0x10000) dest[dnum++] = c; + rp += 3; + } + } else if (c < 0xfc) { + if (rp[1] != '\0' && rp[2] != '\0' && rp[3] != '\0' && rp[4] != '\0') { + c = ((c & 0x03) << 24) | ((rp[1] & 0x3f) << 18) | ((rp[2] & 0x3f) << 12) | + ((rp[3] & 0x3f) << 6) | (rp[4] & 0x3f); + if (c >= 0x200000) dest[dnum++] = c; + rp += 4; + } + } else if (c < 0xfe) { + if (rp[1] != '\0' && rp[2] != '\0' && rp[3] != '\0' && rp[4] != '\0' && rp[5] != '\0') { + c = ((c & 0x01) << 30) | ((rp[1] & 0x3f) << 24) | ((rp[2] & 0x3f) << 18) | + ((rp[3] & 0x3f) << 12) | ((rp[4] & 0x3f) << 6) | (rp[5] & 0x3f); + if (c >= 0x4000000) dest[dnum++] = c; + rp += 5; + } + } + rp++; + } + *np = dnum; +} + + +/** + * Convert a UCS-4 array into a UTF-8 string. + */ +inline size_t strucstoutf(const uint32_t* src, size_t snum, char* dest) { + _assert_(src && snum <= MEMMAXSIZ && dest); + const uint32_t* ep = src + snum; + unsigned char* wp = (unsigned char*)dest; + while (src < ep) { + uint32_t c = *src; + if (c < 0x80) { + *(wp++) = c; + } else if (c < 0x800) { + *(wp++) = 0xc0 | (c >> 6); + *(wp++) = 0x80 | (c & 0x3f); + } else if (c < 0x10000) { + *(wp++) = 0xe0 | (c >> 12); + *(wp++) = 0x80 | ((c & 0xfff) >> 6); + *(wp++) = 0x80 | (c & 0x3f); + } else if (c < 0x200000) { + *(wp++) = 0xf0 | (c >> 18); + *(wp++) = 0x80 | ((c & 0x3ffff) >> 12); + *(wp++) = 0x80 | ((c & 0xfff) >> 6); + *(wp++) = 0x80 | (c & 0x3f); + } else if (c < 0x4000000) { + *(wp++) = 0xf8 | (c >> 24); + *(wp++) = 0x80 | ((c & 0xffffff) >> 18); + *(wp++) = 0x80 | ((c & 0x3ffff) >> 12); + *(wp++) = 0x80 | ((c & 0xfff) >> 6); + *(wp++) = 0x80 | (c & 0x3f); + } else if (c < 0x80000000) { + *(wp++) = 0xfc | (c >> 30); + *(wp++) = 0x80 | ((c & 0x3fffffff) >> 24); + *(wp++) = 0x80 | ((c & 0xffffff) >> 18); + *(wp++) = 0x80 | ((c & 0x3ffff) >> 12); + *(wp++) = 0x80 | ((c & 0xfff) >> 6); + *(wp++) = 0x80 | (c & 0x3f); + } + src++; + } + *wp = '\0'; + return wp - (unsigned char*)dest; +} + + +/** + * Allocate a region on memory. + */ +inline void* xmalloc(size_t size) { + _assert_(size <= MEMMAXSIZ); + void* ptr = std::malloc(size); + if (!ptr) throw std::bad_alloc(); + return ptr; +} + + +/** + * Allocate a nullified region on memory. + */ +inline void* xcalloc(size_t nmemb, size_t size) { + _assert_(nmemb <= MEMMAXSIZ && size <= MEMMAXSIZ); + void* ptr = std::calloc(nmemb, size); + if (!ptr) throw std::bad_alloc(); + return ptr; +} + + +/** + * Re-allocate a region on memory. + */ +inline void* xrealloc(void* ptr, size_t size) { + _assert_(size <= MEMMAXSIZ); + ptr = std::realloc(ptr, size); + if (!ptr) throw std::bad_alloc(); + return ptr; +} + + +/** + * Free a region on memory. + */ +inline void xfree(void* ptr) { + _assert_(true); + std::free(ptr); +} + + +/** + * Dummy test driver. + */ +inline bool _dummytest() { + _assert_(true); + std::ostringstream oss; + oss << INT8MAX << INT16MAX << INT32MAX << INT64MAX; + oss << INT8MIN << INT16MIN << INT32MIN << INT64MIN; + oss << UINT8MAX << UINT16MAX << UINT32MAX << UINT64MAX; + oss << SIZEMAX << FLTMAX << DBLMAX; + oss << VERSION << LIBVER << LIBREV << FMTVER << OSNAME; + oss << BIGEND << CLOCKTICK << PAGESIZ << FEATURES; + oss << NUMBUFSIZ << MEMMAXSIZ; + return oss.tellp() > 0; +} + + +} // common namespace + +#endif // duplication check + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcutilmgr.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcutilmgr.cc new file mode 100644 index 0000000000..2655b35da4 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcutilmgr.cc @@ -0,0 +1,781 @@ +/************************************************************************************************* + * The command line interface of miscellaneous utilities + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "cmdcommon.h" + + +// global variables +const char* g_progname; // program name + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static int32_t runhex(int argc, char** argv); +static int32_t runenc(int argc, char** argv); +static int32_t runciph(int argc, char** argv); +static int32_t runcomp(int argc, char** argv); +static int32_t runhash(int argc, char** argv); +static int32_t runregex(int argc, char** argv); +static int32_t runconf(int argc, char** argv); +static int32_t prochex(const char* file, bool dec); +static int32_t procenc(const char* file, int32_t mode, bool dec); +static int32_t procciph(const char* file, const char* key); +static int32_t proccomp(const char* file, int32_t mode, bool dec); +static int32_t prochash(const char* file, int32_t mode); +static int32_t procregex(const char* file, const char* pattern, const char* alt, int32_t opts); +static int32_t procconf(int32_t mode); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "hex")) { + rv = runhex(argc, argv); + } else if (!std::strcmp(argv[1], "enc")) { + rv = runenc(argc, argv); + } else if (!std::strcmp(argv[1], "ciph")) { + rv = runciph(argc, argv); + } else if (!std::strcmp(argv[1], "comp")) { + rv = runcomp(argc, argv); + } else if (!std::strcmp(argv[1], "hash")) { + rv = runhash(argc, argv); + } else if (!std::strcmp(argv[1], "regex")) { + rv = runregex(argc, argv); + } else if (!std::strcmp(argv[1], "conf")) { + rv = runconf(argc, argv); + } else if (!std::strcmp(argv[1], "version") || !std::strcmp(argv[1], "--version")) { + printversion(); + rv = 0; + } else { + usage(); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: command line interface of miscellaneous utilities of Kyoto Cabinet\n", + g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s hex [-d] [file]\n", g_progname); + eprintf(" %s enc [-hex|-url|-quote] [-d] [file]\n", g_progname); + eprintf(" %s ciph [-key str] [file]\n", g_progname); + eprintf(" %s comp [-def|-gz|-lzo|-lzma] [-d] [file]\n", g_progname); + eprintf(" %s hash [-fnv|-path|-crc] [file]\n", g_progname); + eprintf(" %s regex [-alt str] [-ic] pattern [file]\n", g_progname); + eprintf(" %s conf [-v|-i|-l|-p]\n", g_progname); + eprintf(" %s version\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// parse arguments of hex command +static int32_t runhex(int argc, char** argv) { + bool argbrk = false; + const char* file = NULL; + bool dec = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-d")) { + dec = true; + } else { + usage(); + } + } else if (!file) { + argbrk = true; + file = argv[i]; + } else { + usage(); + } + } + int32_t rv = prochex(file, dec); + return rv; +} + + +// parse arguments of enc command +static int32_t runenc(int argc, char** argv) { + bool argbrk = false; + const char* file = NULL; + int32_t mode = 0; + bool dec = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-hex")) { + mode = 1; + } else if (!std::strcmp(argv[i], "-url")) { + mode = 2; + } else if (!std::strcmp(argv[i], "-quote")) { + mode = 3; + } else if (!std::strcmp(argv[i], "-d")) { + dec = true; + } else { + usage(); + } + } else if (!file) { + argbrk = true; + file = argv[i]; + } else { + usage(); + } + } + int32_t rv = procenc(file, mode, dec); + return rv; +} + + +// parse arguments of ciph command +static int32_t runciph(int argc, char** argv) { + bool argbrk = false; + const char* file = NULL; + const char* key = ""; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-key")) { + if (++i >= argc) usage(); + key = argv[i]; + } else { + usage(); + } + } else if (!file) { + argbrk = true; + file = argv[i]; + } else { + usage(); + } + } + int32_t rv = procciph(file, key); + return rv; +} + + +// parse arguments of comp command +static int32_t runcomp(int argc, char** argv) { + bool argbrk = false; + const char* file = NULL; + int32_t mode = 0; + bool dec = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-def")) { + mode = 1; + } else if (!std::strcmp(argv[i], "-gz")) { + mode = 2; + } else if (!std::strcmp(argv[i], "-lzo")) { + mode = 3; + } else if (!std::strcmp(argv[i], "-lzma")) { + mode = 4; + } else if (!std::strcmp(argv[i], "-d")) { + dec = true; + } else { + usage(); + } + } else if (!file) { + argbrk = true; + file = argv[i]; + } else { + usage(); + } + } + int32_t rv = proccomp(file, mode, dec); + return rv; +} + + +// parse arguments of hash command +static int32_t runhash(int argc, char** argv) { + bool argbrk = false; + const char* file = NULL; + int32_t mode = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-fnv")) { + mode = 1; + } else if (!std::strcmp(argv[i], "-path")) { + mode = 2; + } else if (!std::strcmp(argv[i], "-crc")) { + mode = 3; + } else { + usage(); + } + } else if (!file) { + argbrk = true; + file = argv[i]; + } else { + usage(); + } + } + int32_t rv = prochash(file, mode); + return rv; +} + + +// parse arguments of regex command +static int32_t runregex(int argc, char** argv) { + bool argbrk = false; + const char* pattern = NULL; + const char* file = NULL; + const char* alt = NULL; + int32_t opts = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-alt")) { + if (++i >= argc) usage(); + alt = argv[i]; + } else if (!std::strcmp(argv[i], "-ic")) { + opts |= kc::Regex::IGNCASE; + } else { + usage(); + } + } else if (!pattern) { + argbrk = true; + pattern = argv[i]; + } else if (!file) { + file = argv[i]; + } else { + usage(); + } + } + if (!pattern) usage(); + int32_t rv = procregex(file, pattern, alt, opts); + return rv; +} + + +// parse arguments of conf command +static int32_t runconf(int argc, char** argv) { + bool argbrk = false; + int32_t mode = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-v")) { + mode = 'v'; + } else if (!std::strcmp(argv[i], "-i")) { + mode = 'i'; + } else if (!std::strcmp(argv[i], "-l")) { + mode = 'l'; + } else if (!std::strcmp(argv[i], "-p")) { + mode = 'p'; + } else { + usage(); + } + } else { + argbrk = true; + usage(); + } + } + int32_t rv = procconf(mode); + return rv; +} + + +// perform hex command +static int32_t prochex(const char* file, bool dec) { + const char* istr = file && *file == '@' ? file + 1 : NULL; + std::istream *is; + std::ifstream ifs; + std::istringstream iss(istr ? istr : ""); + if (file) { + if (istr) { + is = &iss; + } else { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + } else { + is = &std::cin; + } + if (dec) { + char c; + while (is->get(c)) { + int32_t cc = (unsigned char)c; + int32_t num = -1; + if (cc >= '0' && cc <= '9') { + num = cc - '0'; + } else if (cc >= 'a' && cc <= 'f') { + num = cc - 'a' + 10; + } else if (cc >= 'A' && cc <= 'F') { + num = cc - 'A' + 10; + } + if (num >= 0) { + if (is->get(c)) { + cc = (unsigned char)c; + if (cc >= '0' && cc <= '9') { + num = num * 0x10 + cc - '0'; + } else if (cc >= 'a' && cc <= 'f') { + num = num * 0x10 + cc - 'a' + 10; + } else if (cc >= 'A' && cc <= 'F') { + num = num * 0x10 + cc - 'A' + 10; + } + std::cout << (char)num; + } else { + std::cout << (char)num; + break; + } + } + } + if (istr) std::cout << std::endl; + } else { + bool mid = false; + char c; + while (is->get(c)) { + if (mid) std::cout << ' '; + int32_t cc = (unsigned char)c; + int32_t num = (cc >> 4); + if (num < 10) { + std::cout << (char)('0' + num); + } else { + std::cout << (char)('a' + num - 10); + } + num = (cc & 0x0f); + if (num < 10) { + std::cout << (char)('0' + num); + } else { + std::cout << (char)('a' + num - 10); + } + mid = true; + } + std::cout << std::endl; + } + return 0; +} + + +// perform enc command +static int32_t procenc(const char* file, int32_t mode, bool dec) { + const char* istr = file && *file == '@' ? file + 1 : NULL; + std::istream *is; + std::ifstream ifs; + std::istringstream iss(istr ? istr : ""); + if (file) { + if (istr) { + is = &iss; + } else { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + } else { + is = &std::cin; + } + std::ostringstream oss; + char c; + while (is->get(c)) { + oss.put(c); + } + const std::string& ostr = oss.str(); + bool err = false; + switch (mode) { + default: { + if (dec) { + size_t zsiz; + char* zbuf = kc::basedecode(ostr.c_str(), &zsiz); + std::cout.write(zbuf, zsiz); + delete[] zbuf; + if (istr) std::cout << std::endl; + } else { + char* zbuf = kc::baseencode(ostr.data(), ostr.size()); + std::cout << zbuf; + delete[] zbuf; + std::cout << std::endl; + } + break; + } + case 1: { + if (dec) { + size_t zsiz; + char* zbuf = kc::hexdecode(ostr.c_str(), &zsiz); + std::cout.write(zbuf, zsiz); + delete[] zbuf; + if (istr) std::cout << std::endl; + } else { + char* zbuf = kc::hexencode(ostr.data(), ostr.size()); + std::cout << zbuf; + delete[] zbuf; + std::cout << std::endl; + } + break; + } + case 2: { + if (dec) { + size_t zsiz; + char* zbuf = kc::urldecode(ostr.c_str(), &zsiz); + std::cout.write(zbuf, zsiz); + delete[] zbuf; + if (istr) std::cout << std::endl; + } else { + char* zbuf = kc::urlencode(ostr.data(), ostr.size()); + std::cout << zbuf; + delete[] zbuf; + std::cout << std::endl; + } + break; + } + case 3: { + if (dec) { + size_t zsiz; + char* zbuf = kc::quotedecode(ostr.c_str(), &zsiz); + std::cout.write(zbuf, zsiz); + delete[] zbuf; + if (istr) std::cout << std::endl; + } else { + char* zbuf = kc::quoteencode(ostr.data(), ostr.size()); + std::cout << zbuf; + delete[] zbuf; + std::cout << std::endl; + } + break; + } + } + return err ? 1 : 0; +} + + +// perform ciph command +static int32_t procciph(const char* file, const char* key) { + const char* istr = file && *file == '@' ? file + 1 : NULL; + std::istream *is; + std::ifstream ifs; + std::istringstream iss(istr ? istr : ""); + if (file) { + if (istr) { + is = &iss; + } else { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + } else { + is = &std::cin; + } + std::ostringstream oss; + char c; + while (is->get(c)) { + oss.put(c); + } + const std::string& ostr = oss.str(); + char* cbuf = new char[ostr.size()]; + kc::arccipher(ostr.data(), ostr.size(), key, std::strlen(key), cbuf); + std::cout.write(cbuf, ostr.size()); + delete[] cbuf; + return 0; +} + + +// perform comp command +static int32_t proccomp(const char* file, int32_t mode, bool dec) { + const char* istr = file && *file == '@' ? file + 1 : NULL; + std::istream *is; + std::ifstream ifs; + std::istringstream iss(istr ? istr : ""); + if (file) { + if (istr) { + is = &iss; + } else { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + } else { + is = &std::cin; + } + std::ostringstream oss; + char c; + while (is->get(c)) { + oss.put(c); + } + const std::string& ostr = oss.str(); + bool err = false; + switch (mode) { + default: { + kc::ZLIB::Mode zmode; + switch (mode) { + default: zmode = kc::ZLIB::RAW; break; + case 1: zmode = kc::ZLIB::DEFLATE; break; + case 2: zmode = kc::ZLIB::GZIP; break; + } + if (dec) { + size_t zsiz; + char* zbuf = kc::ZLIB::decompress(ostr.data(), ostr.size(), &zsiz, zmode); + if (zbuf) { + std::cout.write(zbuf, zsiz); + delete[] zbuf; + } else { + eprintf("%s: decompression failed\n", g_progname); + err = true; + } + } else { + size_t zsiz; + char* zbuf = kc::ZLIB::compress(ostr.data(), ostr.size(), &zsiz, zmode); + if (zbuf) { + std::cout.write(zbuf, zsiz); + delete[] zbuf; + } else { + eprintf("%s: compression failed\n", g_progname); + err = true; + } + } + break; + } + case 3: { + kc::LZO::Mode zmode = kc::LZO::RAW; + if (dec) { + size_t zsiz; + char* zbuf = kc::LZO::decompress(ostr.data(), ostr.size(), &zsiz, zmode); + if (zbuf) { + std::cout.write(zbuf, zsiz); + delete[] zbuf; + } else { + eprintf("%s: decompression failed\n", g_progname); + err = true; + } + } else { + size_t zsiz; + char* zbuf = kc::LZO::compress(ostr.data(), ostr.size(), &zsiz, zmode); + if (zbuf) { + std::cout.write(zbuf, zsiz); + delete[] zbuf; + } else { + eprintf("%s: compression failed\n", g_progname); + err = true; + } + } + break; + } + case 4: { + kc::LZMA::Mode zmode = kc::LZMA::RAW; + if (dec) { + size_t zsiz; + char* zbuf = kc::LZMA::decompress(ostr.data(), ostr.size(), &zsiz, zmode); + if (zbuf) { + std::cout.write(zbuf, zsiz); + delete[] zbuf; + } else { + eprintf("%s: decompression failed\n", g_progname); + err = true; + } + } else { + size_t zsiz; + char* zbuf = kc::LZMA::compress(ostr.data(), ostr.size(), &zsiz, zmode); + if (zbuf) { + std::cout.write(zbuf, zsiz); + delete[] zbuf; + } else { + eprintf("%s: compression failed\n", g_progname); + err = true; + } + } + break; + } + } + return err ? 1 : 0; +} + + +// perform hash command +static int32_t prochash(const char* file, int32_t mode) { + const char* istr = file && *file == '@' ? file + 1 : NULL; + std::istream *is; + std::ifstream ifs; + std::istringstream iss(istr ? istr : ""); + if (file) { + if (istr) { + is = &iss; + } else { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + } else { + is = &std::cin; + } + std::ostringstream oss; + char c; + while (is->get(c)) { + oss.put(c); + } + const std::string& ostr = oss.str(); + switch (mode) { + default: { + uint64_t hash = kc::hashmurmur(ostr.data(), ostr.size()); + oprintf("%016llx\n", (unsigned long long)hash); + break; + } + case 1: { + uint64_t hash = kc::hashfnv(ostr.data(), ostr.size()); + oprintf("%016llx\n", (unsigned long long)hash); + break; + } + case 2: { + char name[kc::NUMBUFSIZ]; + uint32_t hash = kc::hashpath(ostr.data(), ostr.size(), name); + oprintf("%s\t%08lx\n", name, (unsigned long)hash); + break; + } + case 3: { + uint32_t hash = kc::ZLIB::calculate_crc(ostr.data(), ostr.size()); + oprintf("%08x\n", (unsigned)hash); + break; + } + } + return 0; +} + + +// perform regex command +static int32_t procregex(const char* file, const char* pattern, const char* alt, int32_t opts) { + const char* istr = file && *file == '@' ? file + 1 : NULL; + std::istream *is; + std::ifstream ifs; + std::istringstream iss(istr ? istr : ""); + if (file) { + if (istr) { + is = &iss; + } else { + ifs.open(file, std::ios_base::in | std::ios_base::binary); + if (!ifs) { + eprintf("%s: %s: open error\n", g_progname, file); + return 1; + } + is = &ifs; + } + } else { + is = &std::cin; + } + if (alt) { + kc::Regex regex; + if (!regex.compile(pattern, opts)) { + eprintf("%s: %s: compilation failed\n", g_progname, pattern); + return 1; + } + std::string altstr(alt); + std::string line; + while (mygetline(is, &line)) { + + std::cout << regex.replace(line, altstr) << std::endl; + } + + + + } else { + kc::Regex regex; + if (!regex.compile(pattern, opts | kc::Regex::MATCHONLY)) { + eprintf("%s: %s: compilation failed\n", g_progname, pattern); + return 1; + } + std::string line; + while (mygetline(is, &line)) { + if (regex.match(line)) std::cout << line << std::endl; + } + } + return 0; +} + + +// perform conf command +static int32_t procconf(int32_t mode) { + switch (mode) { + case 'v': { + oprintf("%s\n", kc::VERSION); + break; + } + case 'i': { + oprintf("%s\n", _KC_APPINC); + break; + } + case 'l': { + oprintf("%s\n", _KC_APPLIBS); + break; + } + case 'p': { + oprintf("%s\n", _KC_BINDIR); + break; + } + default: { + oprintf("VERSION: %s\n", kc::VERSION); + oprintf("LIBVER: %d\n", kc::LIBVER); + oprintf("LIBREV: %d\n", kc::LIBREV); + oprintf("FMTVER: %d\n", kc::FMTVER); + oprintf("OSNAME: %s\n", kc::OSNAME); + oprintf("BIGEND: %d\n", kc::BIGEND); + oprintf("CLOCKTICK: %d\n", kc::CLOCKTICK); + oprintf("PAGESIZ: %d\n", kc::PAGESIZ); + oprintf("FEATURES: %s\n", kc::FEATURES); + oprintf("TYPES: void*=%d short=%d int=%d long=%d long_long=%d size_t=%d" + " float=%d double=%d long_double=%d\n", + (int)sizeof(void*), (int)sizeof(short), (int)sizeof(int), (int)sizeof(long), + (int)sizeof(long long), (int)sizeof(size_t), + (int)sizeof(float), (int)sizeof(double), (int)sizeof(long double)); + std::map<std::string, std::string> info; + kc::getsysinfo(&info); + if (info["mem_total"].size() > 0) + oprintf("MEMORY: total=%s free=%s cached=%s\n", + info["mem_total"].c_str(), info["mem_free"].c_str(), + info["mem_cached"].c_str()); + if (std::strcmp(_KC_PREFIX, "*")) { + oprintf("prefix: %s\n", _KC_PREFIX); + oprintf("includedir: %s\n", _KC_INCLUDEDIR); + oprintf("libdir: %s\n", _KC_LIBDIR); + oprintf("bindir: %s\n", _KC_BINDIR); + oprintf("libexecdir: %s\n", _KC_LIBEXECDIR); + oprintf("appinc: %s\n", _KC_APPINC); + oprintf("applibs: %s\n", _KC_APPLIBS); + } + break; + } + } + return 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kcutiltest.cc b/plugins/Dbx_kyoto/src/kyotocabinet/kcutiltest.cc new file mode 100644 index 0000000000..7ad66cac32 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kcutiltest.cc @@ -0,0 +1,2770 @@ +/************************************************************************************************* + * The test cases of the utility functions + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#include "cmdcommon.h" + + +// constants +const size_t LOCKSLOTNUM = 128; // number of lock slots +const size_t FILEIOUNIT = 50; // file I/O unit size + + +// global variables +const char* g_progname; // program name +uint32_t g_randseed; // random seed +int64_t g_memusage; // memory usage + + +// function prototypes +int main(int argc, char** argv); +static void usage(); +static void errprint(int32_t line, const char* format, ...); +static void fileerrprint(kc::File* file, int32_t line, const char* func); +static void filemetaprint(kc::File* file); +static int32_t runmutex(int argc, char** argv); +static int32_t runcond(int argc, char** argv); +static int32_t runpara(int argc, char** argv); +static int32_t runfile(int argc, char** argv); +static int32_t runlhmap(int argc, char** argv); +static int32_t runthmap(int argc, char** argv); +static int32_t runtalist(int argc, char** argv); +static int32_t runmisc(int argc, char** argv); +static int32_t procmutex(int64_t rnum, int32_t thnum, double iv); +static int32_t proccond(int64_t rnum, int32_t thnum, double iv); +static int32_t procpara(int64_t rnum, int32_t thnum, double iv); +static int32_t procfile(const char* path, int64_t rnum, int32_t thnum, bool rnd, int64_t msiz); +static int32_t proclhmap(int64_t rnum, bool rnd, int64_t bnum); +static int32_t procthmap(int64_t rnum, bool rnd, int64_t bnum); +static int32_t proctalist(int64_t rnum, bool rnd); +static int32_t procmisc(int64_t rnum); + + +// main routine +int main(int argc, char** argv) { + g_progname = argv[0]; + const char* ebuf = kc::getenv("KCRNDSEED"); + g_randseed = ebuf ? (uint32_t)kc::atoi(ebuf) : (uint32_t)(kc::time() * 1000); + mysrand(g_randseed); + g_memusage = memusage(); + kc::setstdiobin(); + if (argc < 2) usage(); + int32_t rv = 0; + if (!std::strcmp(argv[1], "mutex")) { + rv = runmutex(argc, argv); + } else if (!std::strcmp(argv[1], "cond")) { + rv = runcond(argc, argv); + } else if (!std::strcmp(argv[1], "para")) { + rv = runpara(argc, argv); + } else if (!std::strcmp(argv[1], "file")) { + rv = runfile(argc, argv); + } else if (!std::strcmp(argv[1], "lhmap")) { + rv = runlhmap(argc, argv); + } else if (!std::strcmp(argv[1], "thmap")) { + rv = runthmap(argc, argv); + } else if (!std::strcmp(argv[1], "talist")) { + rv = runtalist(argc, argv); + } else if (!std::strcmp(argv[1], "misc")) { + rv = runmisc(argc, argv); + } else { + usage(); + } + if (rv != 0) { + oprintf("FAILED: KCRNDSEED=%u PID=%ld", g_randseed, (long)kc::getpid()); + for (int32_t i = 0; i < argc; i++) { + oprintf(" %s", argv[i]); + } + oprintf("\n\n"); + } + return rv; +} + + +// print the usage and exit +static void usage() { + eprintf("%s: test cases of the utility functions of Kyoto Cabinet\n", g_progname); + eprintf("\n"); + eprintf("usage:\n"); + eprintf(" %s mutex [-th num] [-iv num] rnum\n", g_progname); + eprintf(" %s para [-th num] [-iv num] rnum\n", g_progname); + eprintf(" %s cond [-th num] [-iv num] rnum\n", g_progname); + eprintf(" %s file [-th num] [-rnd] [-msiz num] path rnum\n", g_progname); + eprintf(" %s lhmap [-rnd] [-bnum num] rnum\n", g_progname); + eprintf(" %s thmap [-rnd] [-bnum num] rnum\n", g_progname); + eprintf(" %s talist [-rnd] rnum\n", g_progname); + eprintf(" %s misc rnum\n", g_progname); + eprintf("\n"); + std::exit(1); +} + + +// print formatted error information string and flush the buffer +static void errprint(int32_t line, const char* format, ...) { + std::string msg; + kc::strprintf(&msg, "%s: %d: ", g_progname, line); + va_list ap; + va_start(ap, format); + kc::vstrprintf(&msg, format, ap); + va_end(ap); + kc::strprintf(&msg, "\n"); + std::cout << msg; + std::cout.flush(); +} + + +// print error message of file +static void fileerrprint(kc::File* file, int32_t line, const char* func) { + oprintf("%s: %d: %s: %s: %s\n", g_progname, line, func, file->path().c_str(), file->error()); +} + + +// print members of file +static void filemetaprint(kc::File* file) { + oprintf("size: %lld\n", (long long)file->size()); +} + + +// parse arguments of mutex command +static int32_t runmutex(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + double iv = 0.0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-iv")) { + if (++i >= argc) usage(); + iv = kc::atof(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procmutex(rnum, thnum, iv); + return rv; +} + + +// parse arguments of cond command +static int32_t runcond(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + double iv = 0.0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-iv")) { + if (++i >= argc) usage(); + iv = kc::atof(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = proccond(rnum, thnum, iv); + return rv; +} + + +// parse arguments of para command +static int32_t runpara(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + int32_t thnum = 1; + double iv = 0.0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-iv")) { + if (++i >= argc) usage(); + iv = kc::atof(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procpara(rnum, thnum, iv); + return rv; +} + + +// parse arguments of file command +static int32_t runfile(int argc, char** argv) { + bool argbrk = false; + const char* path = NULL; + const char* rstr = NULL; + int32_t thnum = 1; + bool rnd = false; + int64_t msiz = 0; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-th")) { + if (++i >= argc) usage(); + thnum = kc::atoix(argv[i]); + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-msiz")) { + if (++i >= argc) usage(); + msiz = kc::atoix(argv[i]); + } else { + usage(); + } + } else if (!path) { + argbrk = true; + path = argv[i]; + } else if (!rstr) { + rstr = argv[i]; + } else { + usage(); + } + } + if (!path || !rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1 || thnum < 1 || msiz < 0) usage(); + if (thnum > THREADMAX) thnum = THREADMAX; + int32_t rv = procfile(path, rnum, thnum, rnd, msiz); + return rv; +} + + +// parse arguments of lhmap command +static int32_t runlhmap(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + bool rnd = false; + int64_t bnum = -1; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1) usage(); + int32_t rv = proclhmap(rnum, rnd, bnum); + return rv; +} + + +// parse arguments of thmap command +static int32_t runthmap(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + bool rnd = false; + int64_t bnum = -1; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else if (!std::strcmp(argv[i], "-bnum")) { + if (++i >= argc) usage(); + bnum = kc::atoix(argv[i]); + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1) usage(); + int32_t rv = procthmap(rnum, rnd, bnum); + return rv; +} + + +// parse arguments of talist command +static int32_t runtalist(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + bool rnd = false; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else if (!std::strcmp(argv[i], "-rnd")) { + rnd = true; + } else { + usage(); + } + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1) usage(); + int32_t rv = proctalist(rnum, rnd); + return rv; +} + + +// parse arguments of misc command +static int32_t runmisc(int argc, char** argv) { + bool argbrk = false; + const char* rstr = NULL; + for (int32_t i = 2; i < argc; i++) { + if (!argbrk && argv[i][0] == '-') { + if (!std::strcmp(argv[i], "--")) { + argbrk = true; + } else usage(); + } else if (!rstr) { + argbrk = true; + rstr = argv[i]; + } else { + usage(); + } + } + if (!rstr) usage(); + int64_t rnum = kc::atoix(rstr); + if (rnum < 1) usage(); + int32_t rv = procmisc(rnum); + return rv; +} + + +// perform mutex command +static int32_t procmutex(int64_t rnum, int32_t thnum, double iv) { + oprintf("<Mutex Test>\n seed=%u rnum=%lld thnum=%d iv=%.3f\n\n", + g_randseed, (long long)rnum, thnum, iv); + bool err = false; + kc::Mutex mutex; + oprintf("mutex:\n"); + double stime = kc::time(); + class ThreadMutex : public kc::Thread { + public: + void setparams(int32_t id, kc::Mutex* mutex, int64_t rnum, int32_t thnum, double iv) { + id_ = id; + mutex_ = mutex; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + mutex_->lock(); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + mutex_->unlock(); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::Mutex* mutex_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadMutex threadmutexs[THREADMAX]; + if (thnum < 2) { + threadmutexs[0].setparams(0, &mutex, rnum, thnum, iv); + threadmutexs[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadmutexs[i].setparams(i, &mutex, rnum, thnum, iv); + threadmutexs[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadmutexs[i].join(); + } + } + double etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::SlottedMutex smutex(LOCKSLOTNUM); + oprintf("slotted mutex:\n"); + stime = kc::time(); + class ThreadSlottedMutex : public kc::Thread { + public: + void setparams(int32_t id, kc::SlottedMutex* smutex, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + smutex_ = smutex; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + size_t idx = i % LOCKSLOTNUM; + smutex_->lock(idx); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + smutex_->unlock(idx); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SlottedMutex* smutex_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSlottedMutex threadsmutexs[THREADMAX]; + if (thnum < 2) { + threadsmutexs[0].setparams(0, &smutex, rnum, thnum, iv); + threadsmutexs[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsmutexs[i].setparams(i, &smutex, rnum, thnum, iv); + threadsmutexs[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsmutexs[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::SpinLock spinlock; + oprintf("spin lock:\n"); + stime = kc::time(); + class ThreadSpinLock : public kc::Thread { + public: + void setparams(int32_t id, kc::SpinLock* spinlock, int64_t rnum, int32_t thnum, double iv) { + id_ = id; + spinlock_ = spinlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + spinlock_->lock(); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + spinlock_->unlock(); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SpinLock* spinlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSpinLock threadspins[THREADMAX]; + if (thnum < 2) { + threadspins[0].setparams(0, &spinlock, rnum, thnum, iv); + threadspins[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadspins[i].setparams(i, &spinlock, rnum, thnum, iv); + threadspins[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadspins[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::SlottedSpinLock sspinlock(LOCKSLOTNUM); + oprintf("slotted spin lock:\n"); + stime = kc::time(); + class ThreadSlottedSpinLock : public kc::Thread { + public: + void setparams(int32_t id, kc::SlottedSpinLock* sspinlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + sspinlock_ = sspinlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + size_t idx = i % LOCKSLOTNUM; + sspinlock_->lock(idx); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + sspinlock_->unlock(idx); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SlottedSpinLock* sspinlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSlottedSpinLock threadsspinlocks[THREADMAX]; + if (thnum < 2) { + threadsspinlocks[0].setparams(0, &sspinlock, rnum, thnum, iv); + threadsspinlocks[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsspinlocks[i].setparams(i, &sspinlock, rnum, thnum, iv); + threadsspinlocks[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsspinlocks[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::RWLock rwlock; + oprintf("reader-writer lock writer:\n"); + stime = kc::time(); + class ThreadRWLockWriter : public kc::Thread { + public: + void setparams(int32_t id, kc::RWLock* rwlock, int64_t rnum, int32_t thnum, double iv) { + id_ = id; + rwlock_ = rwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + rwlock_->lock_writer(); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + rwlock_->unlock(); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::RWLock* rwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadRWLockWriter threadrwlockwriters[THREADMAX]; + if (thnum < 2) { + threadrwlockwriters[0].setparams(0, &rwlock, rnum, thnum, iv); + threadrwlockwriters[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadrwlockwriters[i].setparams(i, &rwlock, rnum, thnum, iv); + threadrwlockwriters[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadrwlockwriters[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("reader-writer lock reader:\n"); + stime = kc::time(); + class ThreadRWLockReader : public kc::Thread { + public: + void setparams(int32_t id, kc::RWLock* rwlock, int64_t rnum, int32_t thnum, double iv) { + id_ = id; + rwlock_ = rwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + rwlock_->lock_reader(); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + rwlock_->unlock(); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::RWLock* rwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadRWLockReader threadrwlockreaders[THREADMAX]; + if (thnum < 2) { + threadrwlockreaders[0].setparams(0, &rwlock, rnum, thnum, iv); + threadrwlockreaders[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadrwlockreaders[i].setparams(i, &rwlock, rnum, thnum, iv); + threadrwlockreaders[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadrwlockreaders[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::SlottedRWLock srwlock(LOCKSLOTNUM); + oprintf("slotted reader-writer lock writer:\n"); + stime = kc::time(); + class ThreadSlottedRWLockWriter : public kc::Thread { + public: + void setparams(int32_t id, kc::SlottedRWLock* srwlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + srwlock_ = srwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + size_t idx = i % LOCKSLOTNUM; + srwlock_->lock_writer(idx); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + srwlock_->unlock(idx); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SlottedRWLock* srwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSlottedRWLockWriter threadsrwlockwriters[THREADMAX]; + if (thnum < 2) { + threadsrwlockwriters[0].setparams(0, &srwlock, rnum, thnum, iv); + threadsrwlockwriters[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsrwlockwriters[i].setparams(i, &srwlock, rnum, thnum, iv); + threadsrwlockwriters[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsrwlockwriters[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("slotted reader-writer lock reader:\n"); + stime = kc::time(); + class ThreadSlottedRWLockReader : public kc::Thread { + public: + void setparams(int32_t id, kc::SlottedRWLock* srwlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + srwlock_ = srwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + size_t idx = i % LOCKSLOTNUM; + srwlock_->lock_reader(idx); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + srwlock_->unlock(idx); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SlottedRWLock* srwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSlottedRWLockReader threadsrwlockreaders[THREADMAX]; + if (thnum < 2) { + threadsrwlockreaders[0].setparams(0, &srwlock, rnum, thnum, iv); + threadsrwlockreaders[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadsrwlockreaders[i].setparams(i, &srwlock, rnum, thnum, iv); + threadsrwlockreaders[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadsrwlockreaders[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::SpinRWLock spinrwlock; + oprintf("spin reader-writer lock writer:\n"); + stime = kc::time(); + class ThreadSpinRWLockWriter : public kc::Thread { + public: + void setparams(int32_t id, kc::SpinRWLock* spinrwlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + spinrwlock_ = spinrwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + spinrwlock_->lock_writer(); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + spinrwlock_->unlock(); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SpinRWLock* spinrwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSpinRWLockWriter threadspinrwlockwriters[THREADMAX]; + if (thnum < 2) { + threadspinrwlockwriters[0].setparams(0, &spinrwlock, rnum, thnum, iv); + threadspinrwlockwriters[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadspinrwlockwriters[i].setparams(i, &spinrwlock, rnum, thnum, iv); + threadspinrwlockwriters[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadspinrwlockwriters[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("spin reader-writer lock reader:\n"); + stime = kc::time(); + class ThreadSpinRWLockReader : public kc::Thread { + public: + void setparams(int32_t id, kc::SpinRWLock* spinrwlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + spinrwlock_ = spinrwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + spinrwlock_->lock_reader(); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + spinrwlock_->unlock(); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SpinRWLock* spinrwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSpinRWLockReader threadspinrwlockreaders[THREADMAX]; + if (thnum < 2) { + threadspinrwlockreaders[0].setparams(0, &spinrwlock, rnum, thnum, iv); + threadspinrwlockreaders[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadspinrwlockreaders[i].setparams(i, &spinrwlock, rnum, thnum, iv); + threadspinrwlockreaders[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadspinrwlockreaders[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("spin reader-writer lock wicked:\n"); + stime = kc::time(); + class ThreadSpinRWLockWicked : public kc::Thread { + public: + void setparams(int32_t id, kc::SpinRWLock* spinrwlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + spinrwlock_ = spinrwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + if (i % 4 == 0) { + spinrwlock_->lock_writer(); + if (i % 16 == 0) { + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + spinrwlock_->demote(); + } + } else { + spinrwlock_->lock_reader(); + if (i % 7 == 0) { + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + spinrwlock_->promote(); + } + } + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + spinrwlock_->unlock(); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SpinRWLock* spinrwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSpinRWLockWicked threadspinrwlockwickeds[THREADMAX]; + if (thnum < 2) { + threadspinrwlockwickeds[0].setparams(0, &spinrwlock, rnum, thnum, iv); + threadspinrwlockwickeds[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadspinrwlockwickeds[i].setparams(i, &spinrwlock, rnum, thnum, iv); + threadspinrwlockwickeds[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadspinrwlockwickeds[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::SlottedSpinRWLock ssrwlock(LOCKSLOTNUM); + oprintf("slotted spin reader-writer lock writer:\n"); + stime = kc::time(); + class ThreadSlottedSpinRWLockWriter : public kc::Thread { + public: + void setparams(int32_t id, kc::SlottedSpinRWLock* ssrwlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + ssrwlock_ = ssrwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + size_t idx = i % LOCKSLOTNUM; + ssrwlock_->lock_writer(idx); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + ssrwlock_->unlock(idx); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SlottedSpinRWLock* ssrwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSlottedSpinRWLockWriter threadssrwlockwriters[THREADMAX]; + if (thnum < 2) { + threadssrwlockwriters[0].setparams(0, &ssrwlock, rnum, thnum, iv); + threadssrwlockwriters[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadssrwlockwriters[i].setparams(i, &ssrwlock, rnum, thnum, iv); + threadssrwlockwriters[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadssrwlockwriters[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("slotted spin reader-writer lock reader:\n"); + stime = kc::time(); + class ThreadSlottedSpinRWLockReader : public kc::Thread { + public: + void setparams(int32_t id, kc::SlottedSpinRWLock* ssrwlock, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + ssrwlock_ = ssrwlock; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + size_t idx = i % LOCKSLOTNUM; + ssrwlock_->lock_reader(idx); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + ssrwlock_->unlock(idx); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::SlottedSpinRWLock* ssrwlock_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadSlottedSpinRWLockReader threadssrwlockreaders[THREADMAX]; + if (thnum < 2) { + threadssrwlockreaders[0].setparams(0, &ssrwlock, rnum, thnum, iv); + threadssrwlockreaders[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadssrwlockreaders[i].setparams(i, &ssrwlock, rnum, thnum, iv); + threadssrwlockreaders[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadssrwlockreaders[i].join(); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("atomic increment:\n"); + stime = kc::time(); + kc::AtomicInt64 anum; + anum = rnum * thnum; + class ThreadAtomic : public kc::Thread { + public: + void setparams(int32_t id, kc::AtomicInt64* anum, int64_t rnum, int32_t thnum, double iv) { + id_ = id; + anum_ = anum; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + anum_->add(1); + *anum_ += 1; + *anum_ -= 1; + while (true) { + int64_t num = anum_->get(); + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + if (anum_->cas(num, num + 1)) break; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::AtomicInt64* anum_; + int64_t rnum_; + int32_t thnum_; + double iv_; + }; + ThreadAtomic threadatomic[THREADMAX]; + if (thnum < 2) { + threadatomic[0].setparams(0, &anum, rnum, thnum, iv); + threadatomic[0].run(); + } else { + for (int32_t i = 0; i < thnum; i++) { + threadatomic[i].setparams(i, &anum, rnum, thnum, iv); + threadatomic[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadatomic[i].join(); + } + } + if (anum.get() != rnum * thnum * 3) { + errprint(__LINE__, "AtomicInt64::get: %lld", (long long)anum.get()); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform cond command +static int32_t proccond(int64_t rnum, int32_t thnum, double iv) { + oprintf("<Mutex Test>\n seed=%u rnum=%lld thnum=%d iv=%.3f\n\n", + g_randseed, (long long)rnum, thnum, iv); + bool err = false; + kc::Mutex mutex; + kc::CondVar cond; + oprintf("conditon variable:\n"); + double stime = kc::time(); + class ThreadCondVar : public kc::Thread { + public: + void setparams(int32_t id, kc::Mutex* mutex, kc::CondVar* cond, + int64_t rnum, int32_t thnum, double iv) { + id_ = id; + mutex_ = mutex; + cond_ = cond; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + active_ = 1; + } + bool active() { + return active_ > 0; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + mutex_->lock(); + if (i % 2 < 1) { + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + } + if (i % 7 == 0) { + cond_->wait(mutex_, 0.001); + } else { + cond_->wait(mutex_); + } + mutex_->unlock(); + if (i % 2 > 0) { + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + active_ = 0; + } + private: + int32_t id_; + kc::Mutex* mutex_; + kc::CondVar* cond_; + int64_t rnum_; + int32_t thnum_; + double iv_; + kc::AtomicInt64 active_; + }; + ThreadCondVar threadcondvars[THREADMAX]; + for (int32_t i = 0; i < thnum; i++) { + threadcondvars[i].setparams(i, &mutex, &cond, rnum, thnum, iv); + threadcondvars[i].start(); + } + int64_t cnt = 0; + while (true) { + if (iv > 0) { + kc::Thread::sleep(iv); + } else if (iv < 0) { + kc::Thread::yield(); + } + int32_t actnum = 0; + for (int32_t i = 0; i < thnum; i++) { + if (threadcondvars[i].active()) actnum++; + bool lock = (cnt + i) % 5 == 0; + if (lock) mutex.lock(); + if (cnt % (thnum + 1) < 1) { + cond.broadcast(); + } else { + cond.signal(); + } + if (lock) mutex.unlock(); + } + if (actnum < 1) break; + cnt++; + } + for (int32_t i = 0; i < thnum; i++) { + threadcondvars[i].join(); + } + double etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + kc::CondMap cmap; + oprintf("conditon map:\n"); + stime = kc::time(); + class ThreadCondMap : public kc::Thread { + public: + void setparams(int32_t id, kc::CondMap* cmap, int64_t rnum, int32_t thnum, double iv) { + id_ = id; + cmap_ = cmap; + rnum_ = rnum; + thnum_ = thnum; + iv_ = iv; + active_ = 1; + } + bool active() { + return active_ > 0; + } + void run() { + for (int64_t i = 1; i <= rnum_; i++) { + if (iv_ > 0) { + sleep(iv_); + } else if (iv_ < 0) { + yield(); + } + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08d", (int)(i % thnum_)); + cmap_->wait(kbuf, ksiz, 0.001); + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + active_ = 0; + } + private: + int32_t id_; + kc::CondMap* cmap_; + int64_t rnum_; + int32_t thnum_; + double iv_; + kc::AtomicInt64 active_; + }; + ThreadCondMap threadcondmaps[THREADMAX]; + for (int32_t i = 0; i < thnum; i++) { + threadcondmaps[i].setparams(i, &cmap, rnum, thnum, iv); + threadcondmaps[i].start(); + } + cnt = 0; + while (true) { + if (iv > 0) { + kc::Thread::sleep(iv); + } else if (iv < 0) { + kc::Thread::yield(); + } + int32_t actnum = 0; + for (int32_t i = 0; i < thnum; i++) { + if (threadcondmaps[i].active()) actnum++; + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08d", (int)i); + bool lock = (cnt + i) % 5 == 0; + if (lock) mutex.lock(); + if (cnt % (thnum + 1) < 1) { + cmap.broadcast(kbuf, ksiz); + } else { + cmap.signal(kbuf, ksiz); + } + if (lock) mutex.unlock(); + } + if (cnt % 1024 < 1) cmap.broadcast_all(); + if (actnum < 1) break; + cnt++; + } + for (int32_t i = 0; i < thnum; i++) { + threadcondmaps[i].join(); + } + if (cmap.count() != 0) { + errprint(__LINE__, "CondMap::count"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform para command +static int32_t procpara(int64_t rnum, int32_t thnum, double iv) { + oprintf("<Parallel Test>\n seed=%u rnum=%lld thnum=%d iv=%.3f\n\n", + g_randseed, (long long)rnum, thnum, iv); + bool err = false; + double stime = kc::time(); + class TaskQueueImpl : public kc::TaskQueue { + public: + void setparams(int32_t thnum, double iv) { + thnum_ = thnum; + iv_ = iv; + cnt_ = 0; + } + void do_task(kc::TaskQueue::Task* task) { + cnt_ += 1; + if (iv_ > 0) { + kc::Thread::sleep(iv_ * thnum_); + } else if (iv_ < 0) { + kc::Thread::yield(); + } + delete task; + } + int64_t done_count() { + return cnt_; + } + private: + int32_t thnum_; + double iv_; + kc::AtomicInt64 cnt_; + }; + TaskQueueImpl queue; + queue.setparams(thnum, iv); + queue.start(thnum); + for (int64_t i = 1; i <= rnum; i++) { + kc::TaskQueue::Task* task = new kc::TaskQueue::Task; + queue.add_task(task); + if (iv > 0) { + kc::Thread::sleep(iv); + } else if (iv < 0) { + kc::Thread::yield(); + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + oprintf("count: %lld\n", queue.count()); + oprintf("done: %lld\n", queue.done_count()); + queue.finish(); + if (queue.count() != 0) { + errprint(__LINE__, "TaskQueue::count"); + err = true; + } + if (queue.done_count() != rnum) { + errprint(__LINE__, "TaskQueueImpl::done_count"); + err = true; + } + double etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform file command +static int32_t procfile(const char* path, int64_t rnum, int32_t thnum, bool rnd, int64_t msiz) { + oprintf("<File Test>\n seed=%u path=%s rnum=%lld thnum=%d rnd=%d msiz=%lld\n\n", + g_randseed, path, (long long)rnum, thnum, rnd, (long long)msiz); + bool err = false; + kc::File file; + oprintf("opening the file:\n"); + double stime = kc::time(); + if (!file.open(path, kc::File::OWRITER | kc::File::OCREATE | kc::File::OTRUNCATE, msiz)) { + fileerrprint(&file, __LINE__, "File::open"); + err = true; + } + double etime = kc::time(); + filemetaprint(&file); + oprintf("time: %.3f\n", etime - stime); + oprintf("writing:\n"); + class ThreadWrite : public kc::Thread { + public: + void setparams(int32_t id, kc::File* file, int64_t rnum, int32_t thnum, bool rnd) { + id_ = id; + file_ = file; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + err_ = false; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + if (rnd_ && myrand(2) == 0) { + char rbuf[RECBUFSIZ]; + size_t rsiz = myrand(FILEIOUNIT); + if (rsiz > 0) std::memset(rbuf, '*', rsiz); + if (!file_->append(rbuf, rsiz)) { + fileerrprint(file_, __LINE__, "File::append"); + err_ = true; + } + } else { + int64_t num = rnd_ ? myrand(range) : base + i - 1; + int64_t off = num * FILEIOUNIT; + char rbuf[RECBUFSIZ]; + size_t rsiz = std::sprintf(rbuf, "[%048lld]", (long long)num); + if (!file_->write(off, rbuf, rsiz)) { + fileerrprint(file_, __LINE__, "File::write"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::File* file_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + bool err_; + }; + ThreadWrite threadwrites[THREADMAX]; + if (thnum < 2) { + threadwrites[0].setparams(0, &file, rnum, thnum, rnd); + threadwrites[0].run(); + if (threadwrites[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadwrites[i].setparams(i, &file, rnum, thnum, rnd); + threadwrites[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadwrites[i].join(); + if (threadwrites[i].error()) err = true; + } + } + etime = kc::time(); + filemetaprint(&file); + oprintf("time: %.3f\n", etime - stime); + oprintf("reading:\n"); + stime = kc::time(); + class ThreadRead : public kc::Thread { + public: + void setparams(int32_t id, kc::File* file, int64_t rnum, int32_t thnum, bool rnd) { + id_ = id; + file_ = file; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + err_ = false; + } + bool error() { + return err_; + } + void run() { + int64_t size = file_->size(); + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + int64_t num = rnd_ ? myrand(range) : base + i - 1; + int64_t off = num * FILEIOUNIT; + char rbuf[RECBUFSIZ]; + if (!file_->read(off, rbuf, FILEIOUNIT) && off + (int64_t)FILEIOUNIT < size) { + fileerrprint(file_, __LINE__, "File::read"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::File* file_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + bool err_; + }; + ThreadRead threadreads[THREADMAX]; + if (thnum < 2) { + threadreads[0].setparams(0, &file, rnum, thnum, rnd); + threadreads[0].run(); + if (threadreads[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadreads[i].setparams(i, &file, rnum, thnum, rnd); + threadreads[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadreads[i].join(); + if (threadreads[i].error()) err = true; + } + } + etime = kc::time(); + filemetaprint(&file); + oprintf("time: %.3f\n", etime - stime); + if (rnd) { + int64_t off = rnum * thnum * FILEIOUNIT; + char rbuf[RECBUFSIZ]; + std::memset(rbuf, '@', FILEIOUNIT); + if (!file.write(off, rbuf, FILEIOUNIT)) { + fileerrprint(&file, __LINE__, "File::write"); + err = true; + } + } + oprintf("fast writing:\n"); + stime = kc::time(); + class ThreadWriteFast : public kc::Thread { + public: + void setparams(int32_t id, kc::File* file, int64_t rnum, int32_t thnum, bool rnd) { + id_ = id; + file_ = file; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + err_ = false; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + int64_t num = rnd_ ? myrand(range) : base + i - 1; + int64_t off = num * FILEIOUNIT; + char rbuf[RECBUFSIZ]; + size_t rsiz = std::sprintf(rbuf, "[%048lld]", (long long)num); + if (!file_->write_fast(off, rbuf, rsiz)) { + fileerrprint(file_, __LINE__, "File::write_fast"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::File* file_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + bool err_; + }; + ThreadWriteFast threadwritefasts[THREADMAX]; + if (thnum < 2) { + threadwritefasts[0].setparams(0, &file, rnum, thnum, rnd); + threadwritefasts[0].run(); + if (threadwritefasts[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadwritefasts[i].setparams(i, &file, rnum, thnum, rnd); + threadwritefasts[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadwritefasts[i].join(); + if (threadwritefasts[i].error()) err = true; + } + } + etime = kc::time(); + filemetaprint(&file); + oprintf("time: %.3f\n", etime - stime); + oprintf("fast reading:\n"); + stime = kc::time(); + class ThreadReadFast : public kc::Thread { + public: + void setparams(int32_t id, kc::File* file, int64_t rnum, int32_t thnum, bool rnd) { + id_ = id; + file_ = file; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + err_ = false; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + int64_t num = rnd_ ? myrand(range) : base + i - 1; + int64_t off = num * FILEIOUNIT; + char rbuf[RECBUFSIZ]; + if (!file_->read_fast(off, rbuf, FILEIOUNIT)) { + fileerrprint(file_, __LINE__, "File::read_fast"); + err_ = true; + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::File* file_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + bool err_; + }; + ThreadReadFast threadreadfasts[THREADMAX]; + if (thnum < 2) { + threadreadfasts[0].setparams(0, &file, rnum, thnum, rnd); + threadreadfasts[0].run(); + if (threadreadfasts[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadreadfasts[i].setparams(i, &file, rnum, thnum, rnd); + threadreadfasts[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadreadfasts[i].join(); + if (threadreadfasts[i].error()) err = true; + } + } + etime = kc::time(); + filemetaprint(&file); + oprintf("time: %.3f\n", etime - stime); + oprintf("committing transaction:\n"); + stime = kc::time(); + int64_t qsiz = file.size() / 4; + if (!file.begin_transaction(rnd ? myrand(100) == 0 : false, qsiz)) { + fileerrprint(&file, __LINE__, "File::begin_transaction"); + err = true; + } + if (!file.write_transaction(0, qsiz)) { + fileerrprint(&file, __LINE__, "File::write_transaction"); + err = true; + } + int64_t fsiz = rnd ? myrand(rnum * thnum * FILEIOUNIT) : rnum * thnum / 2 * FILEIOUNIT + 5; + if (!file.truncate(fsiz)) { + fileerrprint(&file, __LINE__, "File::truncate"); + err = true; + } + if (file.size() != fsiz) { + fileerrprint(&file, __LINE__, "File::truncate"); + err = true; + } + class ThreadCommit : public kc::Thread { + public: + void setparams(int32_t id, kc::File* file, int64_t rnum, int32_t thnum, + bool rnd, int64_t fsiz) { + id_ = id; + file_ = file; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + fsiz_ = fsiz; + err_ = false; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + int64_t num = rnd_ ? myrand(range) : base + i - 1; + int64_t off = num * FILEIOUNIT; + char rbuf[RECBUFSIZ]; + size_t rsiz = std::sprintf(rbuf, "[%048lld]", (long long)num); + if (i % 2 == 0 || off > fsiz_ - (int64_t)FILEIOUNIT) { + if (!file_->write(off, rbuf, rsiz)) { + fileerrprint(file_, __LINE__, "File::write"); + err_ = true; + } + } else { + if (!file_->write_fast(off, rbuf, rsiz)) { + fileerrprint(file_, __LINE__, "File::write_fast"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::File* file_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t fsiz_; + bool err_; + }; + ThreadCommit threadcommits[THREADMAX]; + if (thnum < 2) { + threadcommits[0].setparams(0, &file, rnum, thnum, rnd, fsiz); + threadcommits[0].run(); + if (threadcommits[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadcommits[i].setparams(i, &file, rnum, thnum, rnd, fsiz); + threadcommits[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadcommits[i].join(); + if (threadcommits[i].error()) err = true; + } + } + if (!file.end_transaction(true)) { + fileerrprint(&file, __LINE__, "File::end_transaction"); + err = true; + } + etime = kc::time(); + filemetaprint(&file); + oprintf("time: %.3f\n", etime - stime); + oprintf("aborting transaction:\n"); + stime = kc::time(); + qsiz = file.size() / 4; + if (!file.begin_transaction(rnd ? myrand(100) == 0 : false, qsiz)) { + fileerrprint(&file, __LINE__, "File::begin_transaction"); + err = true; + } + if (!file.write_transaction(0, qsiz)) { + fileerrprint(&file, __LINE__, "File::write_transaction"); + err = true; + } + int64_t osiz = file.size(); + kc::StringTreeMap chkmap; + int64_t chknum = rnum / 100 + 1; + for (int64_t i = 0; i < chknum; i++) { + char rbuf[RECBUFSIZ]; + int64_t roff = myrand(osiz); + int32_t rsiz = myrand(RECBUFSIZ); + if (file.read(roff, rbuf, rsiz)) { + std::string key = kc::strprintf("%lld:%d", (long long)roff, rsiz); + chkmap[key] = std::string(rbuf, rsiz); + } + } + fsiz = rnd ? myrand(rnum * thnum * FILEIOUNIT) : rnum * thnum / 2 * FILEIOUNIT + 5; + if (!file.truncate(fsiz)) { + fileerrprint(&file, __LINE__, "File::truncate"); + err = true; + } + if (file.size() != fsiz) { + fileerrprint(&file, __LINE__, "File::truncate"); + err = true; + } + class ThreadAbort : public kc::Thread { + public: + void setparams(int32_t id, kc::File* file, int64_t rnum, int32_t thnum, + bool rnd, int64_t fsiz) { + id_ = id; + file_ = file; + rnum_ = rnum; + thnum_ = thnum; + rnd_ = rnd; + fsiz_ = fsiz; + err_ = false; + } + bool error() { + return err_; + } + void run() { + int64_t base = id_ * rnum_; + int64_t range = rnum_ * thnum_; + for (int64_t i = 1; !err_ && i <= rnum_; i++) { + int64_t num = rnd_ ? myrand(range) : base + i - 1; + int64_t off = num * FILEIOUNIT; + char rbuf[RECBUFSIZ]; + std::memset(rbuf, num, FILEIOUNIT); + if (i % 2 == 0 || off > fsiz_ - (int64_t)FILEIOUNIT) { + if (!file_->write(off, rbuf, FILEIOUNIT)) { + fileerrprint(file_, __LINE__, "File::write"); + err_ = true; + } + } else { + if (!file_->write_fast(off, rbuf, FILEIOUNIT)) { + fileerrprint(file_, __LINE__, "File::write_fast"); + err_ = true; + } + } + if (id_ < 1 && rnum_ > 250 && i % (rnum_ / 250) == 0) { + oputchar('.'); + if (i == rnum_ || i % (rnum_ / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + } + private: + int32_t id_; + kc::File* file_; + int64_t rnum_; + int32_t thnum_; + bool rnd_; + int64_t fsiz_; + bool err_; + }; + ThreadAbort threadaborts[THREADMAX]; + if (thnum < 2) { + threadaborts[0].setparams(0, &file, rnum, thnum, rnd, fsiz); + threadaborts[0].run(); + if (threadaborts[0].error()) err = true; + } else { + for (int32_t i = 0; i < thnum; i++) { + threadaborts[i].setparams(i, &file, rnum, thnum, rnd, fsiz); + threadaborts[i].start(); + } + for (int32_t i = 0; i < thnum; i++) { + threadaborts[i].join(); + if (threadaborts[i].error()) err = true; + } + } + if (!file.end_transaction(false)) { + fileerrprint(&file, __LINE__, "File::end_transaction"); + err = true; + } + if (file.size() != osiz) { + fileerrprint(&file, __LINE__, "File::end_transaction"); + err = true; + } + for (kc::StringTreeMap::iterator it = chkmap.begin(); it != chkmap.end(); ++it) { + const char* key = it->first.c_str(); + int64_t roff = kc::atoi(key); + int32_t rsiz = kc::atoi(std::strchr(key, ':') + 1); + char rbuf[RECBUFSIZ]; + if (file.read(roff, rbuf, rsiz)) { + if (it->second != std::string(rbuf, rsiz)) { + fileerrprint(&file, __LINE__, "File::end_transaction"); + err = true; + } + } else { + fileerrprint(&file, __LINE__, "File::end_transaction"); + err = true; + } + } + etime = kc::time(); + filemetaprint(&file); + oprintf("time: %.3f\n", etime - stime); + oprintf("closing the file:\n"); + stime = kc::time(); + if (!file.close()) { + fileerrprint(&file, __LINE__, "File::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("testing file utility functions:\n"); + stime = kc::time(); + std::string ostr = "_"; + for (int32_t i = 0; i < 100; i++) { + ostr.append(path); + } + ostr.append("_"); + if (!kc::File::write_file(path, ostr.c_str(), ostr.size())) { + errprint(__LINE__, "File::write_file"); + err = true; + } + int64_t isiz; + char* ibuf = kc::File::read_file(path, &isiz); + if (ibuf) { + if (ostr != ibuf) { + errprint(__LINE__, "File::read_file"); + err = true; + } + delete[] ibuf; + } else { + errprint(__LINE__, "File::read_file"); + err = true; + } + kc::File::Status sbuf; + if (!kc::File::status(path, &sbuf) || sbuf.isdir || sbuf.size < 1) { + errprint(__LINE__, "File::status"); + err = true; + } + if (!kc::File::status(kc::File::CDIRSTR, &sbuf) || !sbuf.isdir) { + errprint(__LINE__, "File::status"); + err = true; + } + const std::string& abspath = kc::File::absolute_path(path); + if (abspath.empty()) { + errprint(__LINE__, "File::absolute_path"); + err = true; + } + const std::string& tmppath = kc::strprintf("%s%ctmp", path, kc::File::EXTCHR); + if (!kc::File::rename(path, tmppath) || !kc::File::rename(tmppath, path)) { + errprint(__LINE__, "File::rename"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("testing directory utility functions:\n"); + stime = kc::time(); + std::vector<std::string> files; + if (!kc::File::read_directory(kc::File::CDIRSTR, &files)) { + errprint(__LINE__, "File::read_directory"); + err = true; + } + if (!kc::File::make_directory(tmppath)) { + errprint(__LINE__, "File::make_directory"); + err = true; + } + if (!kc::File::remove_directory(tmppath)) { + errprint(__LINE__, "File::remove_directory"); + err = true; + } + if (!kc::File::make_directory(tmppath)) { + errprint(__LINE__, "File::make_directory"); + err = true; + } + const std::string chldpath = tmppath + kc::File::PATHCHR + "tmp"; + if (!kc::File::write_file(chldpath, tmppath.c_str(), tmppath.size())) { + errprint(__LINE__, "File::write_file"); + err = true; + } + if (!kc::File::remove_recursively(tmppath)) { + errprint(__LINE__, "File::make_recursively"); + err = true; + } + const std::string& cwdpath = kc::File::get_current_directory(); + if (cwdpath.empty()) { + errprint(__LINE__, "File::get_current_directory"); + err = true; + } + if (!kc::File::set_current_directory(cwdpath)) { + errprint(__LINE__, "File::set_current_directory"); + err = true; + } + kc::DirStream dir; + if (!dir.open(cwdpath)) { + errprint(__LINE__, "DirStream::open"); + err = true; + } + std::string cpath; + while (dir.read(&cpath)) { + if (!kc::File::status(cpath, &sbuf)) { + errprint(__LINE__, "File::status"); + err = true; + } + } + if (!dir.close()) { + errprint(__LINE__, "DirStream::close"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform lhmap command +static int32_t proclhmap(int64_t rnum, bool rnd, int64_t bnum) { + oprintf("<Doubly-linked Hash Map Test>\n seed=%u rnum=%lld rnd=%d bnum=%lld\n\n", + g_randseed, (long long)rnum, rnd, (long long)bnum); + bool err = false; + if (bnum < 0) bnum = 0; + typedef kc::LinkedHashMap<std::string, std::string> Map; + Map map(bnum); + oprintf("setting records:\n"); + double stime = kc::time(); + for (int64_t i = 1; i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + map.set(kbuf, kbuf, Map::MCURRENT); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + double etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("getting records:\n"); + stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + Map::MoveMode mode = Map::MCURRENT; + if (rnd) { + switch (myrand(4)) { + case 0: mode = Map::MFIRST; + case 1: mode = Map::MLAST; + } + } + if (!map.get(kbuf, mode) && !rnd) { + errprint(__LINE__, "LinkedHashMap::get: %s", kbuf); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("traversing records:\n"); + stime = kc::time(); + int64_t cnt = 0; + for (Map::Iterator it = map.begin(); !err && it != map.end(); ++it) { + cnt++; + if (it.key() != it.value()) { + errprint(__LINE__, "LinkedHashMap::Iterator::key"); + err = true; + } + if (rnum > 250 && cnt % (rnum / 250) == 0) { + oputchar('.'); + if (cnt == rnum || cnt % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt); + } + } + if (rnd) oprintf(" (end)\n"); + if (cnt != (int64_t)map.count()) { + errprint(__LINE__, "LinkedHashMap::count"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + Map paramap(bnum + 31); + oprintf("migrating records:\n"); + stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + Map::MoveMode mode = Map::MCURRENT; + if (rnd) { + switch (myrand(4)) { + case 0: mode = Map::MFIRST; + case 1: mode = Map::MLAST; + } + } + if (!map.migrate(kbuf, ¶map, mode) && !rnd) { + errprint(__LINE__, "LinkedHashMap::migrate: %s", kbuf); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld,%lld\n", (long long)map.count(), (long long)paramap.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("removing records:\n"); + stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + if (!paramap.remove(kbuf) && !rnd) { + errprint(__LINE__, "LinkedHashMap::remove: %s", kbuf); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld,%lld\n", (long long)map.count(), (long long)paramap.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + if (rnd) { + oprintf("wicked testing:\n"); + stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + Map::MoveMode mode = Map::MCURRENT; + if (rnd) { + switch (myrand(4)) { + case 0: mode = Map::MFIRST; + case 1: mode = Map::MLAST; + } + } + Map *ptr = ↦ + Map *paraptr = ¶map; + if (myrand(2) == 0) { + ptr = ¶map; + paraptr = ↦ + } + switch (myrand(4)) { + case 0: { + ptr->set(kbuf, kbuf, mode); + break; + } + case 1: { + ptr->get(kbuf, mode); + break; + } + case 2: { + ptr->remove(kbuf); + break; + } + case 3: { + ptr->migrate(kbuf, paraptr, mode); + break; + } + } + if (myrand(rnum * 2 + 1) == 0) ptr->clear(); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + cnt = 0; + for (Map::Iterator it = map.begin(); !err && it != map.end(); ++it) { + cnt++; + if (it.key() != it.value()) { + errprint(__LINE__, "LinkedHashMap::Iterator::key"); + err = true; + } + if (rnum > 250 && cnt % (rnum / 250) == 0) { + oputchar('.'); + if (cnt == rnum || cnt % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt); + } + } + if (rnd) oprintf(" (end)\n"); + if (cnt != (int64_t)map.count()) { + errprint(__LINE__, "LinkedHashMap::count"); + err = true; + } + cnt = 0; + Map::Iterator it = map.end(); + while (!err && it != map.begin()) { + --it; + cnt++; + if (it.key() != it.value()) { + errprint(__LINE__, "LinkedHashMap::Iterator::key"); + err = true; + } + if (rnum > 250 && cnt % (rnum / 250) == 0) { + oputchar('.'); + if (cnt == rnum || cnt % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt); + } + } + if (rnd) oprintf(" (end)\n"); + if (cnt != (int64_t)map.count()) { + errprint(__LINE__, "LinkedHashMap::count"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform thmap command +static int32_t procthmap(int64_t rnum, bool rnd, int64_t bnum) { + oprintf("<Memory-saving Hash Map Test>\n seed=%u rnum=%lld rnd=%d bnum=%lld\n\n", + g_randseed, (long long)rnum, rnd, (long long)bnum); + bool err = false; + if (bnum < 0) bnum = 0; + kc::TinyHashMap map(bnum); + oprintf("setting records:\n"); + double stime = kc::time(); + for (int64_t i = 1; i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + map.set(kbuf, ksiz, kbuf, ksiz); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + double etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("getting records:\n"); + stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + size_t vsiz; + const char* vbuf = map.get(kbuf, ksiz, &vsiz); + if (!vbuf && !rnd) { + errprint(__LINE__, "TinyHashMap::get: %s", kbuf); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("appending records:\n"); + stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + map.append(kbuf, ksiz, kbuf, ksiz); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("traversing records:\n"); + stime = kc::time(); + int64_t cnt = 0; + kc::TinyHashMap::Iterator it(&map); + const char* kbuf, *vbuf; + size_t ksiz, vsiz; + while ((kbuf = it.get(&ksiz, &vbuf, &vsiz)) != NULL) { + cnt++; + it.step(); + if (rnum > 250 && cnt % (rnum / 250) == 0) { + oputchar('.'); + if (cnt == rnum || cnt % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt); + } + } + if (rnd) oprintf(" (end)\n"); + if (cnt != (int64_t)map.count()) { + errprint(__LINE__, "TinyHashMap::count"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("sorting records:\n"); + stime = kc::time(); + cnt = 0; + kc::TinyHashMap::Sorter sorter(&map); + while ((kbuf = sorter.get(&ksiz, &vbuf, &vsiz)) != NULL) { + cnt++; + sorter.step(); + if (rnum > 250 && cnt % (rnum / 250) == 0) { + oputchar('.'); + if (cnt == rnum || cnt % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)cnt); + } + } + if (rnd) oprintf(" (end)\n"); + if (cnt != (int64_t)map.count()) { + errprint(__LINE__, "TinyHashMap::count"); + err = true; + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("removing records:\n"); + stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%08lld", (long long)(rnd ? myrand(rnum) + 1 : i)); + if (!map.remove(kbuf, ksiz) && !rnd) { + errprint(__LINE__, "TinyHashMap::remove: %s", kbuf); + err = true; + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + if (rnd) { + oprintf("wicked testing:\n"); + stime = kc::time(); + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + for (int64_t i = 1; !err && i <= rnum; i++) { + char kbuf[RECBUFSIZ]; + size_t ksiz = std::sprintf(kbuf, "%lld", (long long)(myrand(rnum) + 1)); + size_t vsiz = myrand(sizeof(lbuf)); + switch (myrand(6)) { + case 0: { + map.set(kbuf, ksiz, lbuf, vsiz); + break; + } + case 1: { + map.add(kbuf, ksiz, lbuf, vsiz); + break; + } + case 2: { + map.replace(kbuf, ksiz, lbuf, vsiz); + break; + } + case 3: { + map.append(kbuf, ksiz, lbuf, vsiz); + break; + } + case 6: { + map.remove(kbuf, ksiz); + break; + } + default: { + map.get(kbuf, ksiz, &vsiz); + break; + } + } + if (myrand(rnum * 2 + 1) == 0) map.clear(); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)map.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform talist command +static int32_t proctalist(int64_t rnum, bool rnd) { + oprintf("<Memory-saving Array List Test>\n seed=%u rnum=%lld rnd=%d\n\n", + g_randseed, (long long)rnum, rnd); + bool err = false; + kc::TinyArrayList list; + oprintf("setting records:\n"); + double stime = kc::time(); + for (int64_t i = 1; i <= rnum; i++) { + char buf[RECBUFSIZ]; + size_t size = std::sprintf(buf, "%08lld", (long long)i); + if (rnd && myrand(2) == 0) { + list.unshift(buf, size); + } else { + list.push(buf, size); + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + double etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)list.count()); + int64_t musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("getting records:\n"); + stime = kc::time(); + size_t cnt = list.count(); + for (int64_t i = 1; i <= rnum; i++) { + size_t size; + list.get(rnd ? myrand(cnt) : i - 1, &size); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)list.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + oprintf("removing records:\n"); + stime = kc::time(); + for (int64_t i = 1; i <= rnum; i++) { + if (rnd && myrand(2) == 0) { + list.shift(); + } else { + list.pop(); + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)list.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + if (rnd) { + oprintf("wicked testing:\n"); + stime = kc::time(); + char lbuf[RECBUFSIZL]; + std::memset(lbuf, '*', sizeof(lbuf)); + for (int64_t i = 1; !err && i <= rnum; i++) { + size_t size = myrand(sizeof(lbuf)); + cnt = list.count(); + switch (myrand(10)) { + case 0: { + list.pop(); + break; + } + case 1: { + list.unshift(lbuf, size); + break; + } + case 2: { + list.shift(); + break; + } + case 3: { + list.insert(lbuf, size, cnt > 0 ? myrand(cnt) : 0); + break; + } + case 4: { + if (cnt > 0) list.remove(myrand(cnt)); + break; + } + case 5: { + if (cnt > 0) list.get(myrand(cnt), &size); + break; + } + case 6: { + if (myrand(100) == 0) list.clear(); + break; + } + default: { + list.push(lbuf, size); + break; + } + } + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + etime = kc::time(); + oprintf("time: %.3f\n", etime - stime); + oprintf("count: %lld\n", (long long)list.count()); + musage = memusage(); + if (musage > 0) oprintf("memory: %lld\n", (long long)(musage - g_memusage)); + } + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + +// perform misc command +static int32_t procmisc(int64_t rnum) { + oprintf("<Miscellaneous Test>\n seed=%u rnum=%lld\n\n", g_randseed, (long long)rnum); + bool err = false; + if (!kc::_dummytest()) { + errprint(__LINE__, "_dummytest"); + err = true; + } + double stime = kc::time(); + for (int64_t i = 1; !err && i <= rnum; i++) { + uint16_t num16 = (1ULL << myrand(sizeof(num16) * 8)) - 5 + myrand(10); + uint32_t num32 = (1ULL << myrand(sizeof(num32) * 8)) - 5 + myrand(10); + uint64_t num64 = (1ULL << myrand(sizeof(num64) * 8)) - 5 + myrand(10); + if (kc::ntoh16(kc::hton16(num16)) != num16) { + errprint(__LINE__, "ntoh16: %llu", (unsigned long long)num16); + err = true; + } + if (kc::ntoh32(kc::hton32(num32)) != num32) { + errprint(__LINE__, "ntoh32: %llu", (unsigned long long)num32); + err = true; + } + if (kc::ntoh64(kc::hton64(num64)) != num64) { + errprint(__LINE__, "ntoh64: %llu", (unsigned long long)num64); + err = true; + } + char fbuf[sizeof(num64)]; + num64 = (uint64_t)myrand(kc::INT32MAX) * myrand(kc::INT16MAX); + kc::writefixnum(fbuf, num64, 6); + uint64_t onum = kc::readfixnum(fbuf, 6); + if (onum != num64) { + errprint(__LINE__, "readfixnum: %llu:%llu", + (unsigned long long)num64, (unsigned long long)onum); + err = true; + } + unsigned char ubuf[RECBUFSIZ]; + unsigned char* uwp = ubuf; + if (kc::writevarnum(uwp, num32) != kc::sizevarnum(num32)) { + errprint(__LINE__, "sizevarnum: %llu", (unsigned long long)num32); + err = true; + } + uwp += kc::writevarnum(uwp, num16); + uwp += kc::writevarnum(uwp, num32); + uwp += kc::writevarnum(uwp, num64); + const unsigned char* urp = ubuf; + urp += kc::readvarnum(urp, uwp - urp, &onum); + if (onum != num16) { + errprint(__LINE__, "readvarnum: %llu:%llu", + (unsigned long long)num16, (unsigned long long)onum); + err = true; + } + urp += kc::readvarnum(urp, uwp - urp, &onum); + if (onum != num32) { + errprint(__LINE__, "readvarnum: %llu:%llu", + (unsigned long long)num32, (unsigned long long)onum); + err = true; + } + urp += kc::readvarnum(urp, uwp - urp, &onum); + if (onum != num64) { + errprint(__LINE__, "readvarnum: %llu:%llu", + (unsigned long long)num64, (unsigned long long)onum); + err = true; + } + if (urp != uwp) { + errprint(__LINE__, "readvarnum: %d", (int)(uwp - urp)); + err = true; + } + size_t usiz = urp - ubuf; + uint64_t hash = kc::hashmurmur(&num16, sizeof(num16)) + kc::hashmurmur(ubuf, usiz); + hash += kc::hashfnv(&num16, sizeof(num16)) + kc::hashfnv(ubuf, usiz); + char name[kc::NUMBUFSIZ]; + hash += kc::hashpath(ubuf, usiz, name); + hash = kc::nearbyprime(myrand(kc::INT32MAX)); + if (myrand(256) == 0) { + int32_t unum = myrand(64); + std::vector<uint32_t> oucs; + for (int32_t j = 0; j < unum; j++) { + uint32_t c = std::pow(2, myrand(31000000) / 1000000.0); + oucs.push_back(c); + } + std::string utf; + kc::strucstoutf(oucs, &utf); + std::vector<uint32_t> nucs; + kc::strutftoucs(utf, &nucs); + if (nucs.size() == oucs.size()) { + for (int32_t j = 0; j < (int32_t)nucs.size(); j++) { + if (nucs[j] != oucs[j]) { + errprint(__LINE__, "strutftoucs: %d:%d", (int)nucs[j], (int)oucs[j]); + err = true; + break; + } + } + } else { + errprint(__LINE__, "strutftoucs: %d:%d", (int)nucs.size(), (int)oucs.size()); + err = true; + } + uint32_t* cucs = new uint32_t[utf.size()+1]; + size_t cucsnum; + if (myrand(2) == 0) { + kc::strutftoucs(utf.c_str(), cucs, &cucsnum); + } else { + kc::strutftoucs(utf.data(), utf.size(), cucs, &cucsnum); + } + if (cucsnum == oucs.size()) { + char* cutf = new char[cucsnum*6+1]; + kc::strucstoutf(cucs, cucsnum, cutf); + if (std::strcmp(cutf, utf.c_str())) { + errprint(__LINE__, "strucstoutf"); + err = true; + } + delete[] cutf; + } else { + errprint(__LINE__, "strutftoucs"); + err = true; + } + delete[] cucs; + int32_t tnum = myrand(64); + std::vector<std::string> ovec; + std::map<std::string, std::string> omap; + for (int32_t j = 0; j < tnum; j++) { + char kbuf[RECBUFSIZ]; + std::sprintf(kbuf, "%lld", (long long)myrand(rnum)); + char vbuf[RECBUFSIZ]; + std::sprintf(vbuf, "%lld", (long long)myrand(rnum)); + ovec.push_back(vbuf); + omap[kbuf] = vbuf; + } + std::string vstr; + kc::strvecdump(ovec, &vstr); + std::vector<std::string> nvec; + kc::strvecload(vstr, &nvec); + if (nvec.size() != ovec.size()) { + errprint(__LINE__, "strvecload: %d:%d", (int)nvec.size(), (int)ovec.size()); + err = true; + } + std::string mstr; + kc::strmapdump(omap, &mstr); + std::map<std::string, std::string> nmap; + kc::strmapload(mstr, &nmap); + if (nmap.size() != omap.size()) { + errprint(__LINE__, "strmapload: %d:%d", (int)nvec.size(), (int)ovec.size()); + err = true; + } + } + char* ebuf = kc::hexencode(ubuf, usiz); + size_t osiz; + char* obuf = kc::hexdecode(ebuf, &osiz); + if (osiz != usiz || std::memcmp(obuf, ubuf, osiz)) { + errprint(__LINE__, "hexencode: %d:%d", (int)osiz, (int)usiz); + err = true; + } + delete[] obuf; + delete[] ebuf; + ebuf = kc::urlencode(ubuf, usiz); + obuf = kc::urldecode(ebuf, &osiz); + if (osiz != usiz || std::memcmp(obuf, ubuf, osiz)) { + errprint(__LINE__, "urlencode: %d:%d", (int)osiz, (int)usiz); + err = true; + } + delete[] obuf; + delete[] ebuf; + ebuf = kc::quoteencode(ubuf, usiz); + obuf = kc::quotedecode(ebuf, &osiz); + if (osiz != usiz || std::memcmp(obuf, ubuf, osiz)) { + errprint(__LINE__, "quoteencode: %d:%d", (int)osiz, (int)usiz); + err = true; + } + delete[] obuf; + delete[] ebuf; + ebuf = kc::baseencode(ubuf, usiz); + obuf = kc::basedecode(ebuf, &osiz); + if (osiz != usiz || std::memcmp(obuf, ubuf, osiz)) { + errprint(__LINE__, "baseencode: %d:%d", (int)osiz, (int)usiz); + err = true; + } + delete[] obuf; + delete[] ebuf; + size_t nsiz = std::strlen(name); + nsiz -= i % nsiz; + ebuf = new char[usiz]; + kc::arccipher(ubuf, usiz, name, nsiz, ebuf); + obuf = new char[usiz]; + kc::arccipher(ebuf, usiz, name, nsiz, obuf); + if (std::memcmp(obuf, ubuf, usiz)) { + errprint(__LINE__, "arccipher: %s", name); + err = true; + } + if (kc::memicmp(obuf, ubuf, usiz)) { + errprint(__LINE__, "memicmp"); + err = true; + } + if (!kc::memmem(obuf, osiz, ubuf, usiz)) { + errprint(__LINE__, "memmem"); + err = true; + } + if (!kc::memimem(obuf, osiz, ubuf, usiz)) { + errprint(__LINE__, "memimem"); + err = true; + } + if (kc::memdist(obuf, osiz, ubuf, usiz)) { + errprint(__LINE__, "memdist"); + err = true; + } + delete[] obuf; + delete[] ebuf; + ebuf = kc::memdup((char*)ubuf, usiz); + ebuf[usiz] = '\0'; + obuf = kc::strdup(ebuf); + switch (myrand(18)) { + case 0: kc::atoi(obuf); break; + case 1: kc::atoix(obuf); break; + case 2: kc::atoih(obuf); break; + case 3: kc::atoin((char*)ubuf, usiz); break; + case 4: kc::atof(obuf); break; + case 5: kc::atofn((char*)ubuf, usiz); break; + case 6: kc::strtoupper(obuf); break; + case 7: kc::strtolower(obuf); break; + case 8: kc::strtrim(obuf); break; + case 9: kc::strsqzspc(obuf); break; + case 10: kc::strnrmspc(obuf); break; + case 11: kc::stricmp(obuf, ebuf); break; + case 12: kc::stristr(obuf, ebuf); break; + case 13: kc::strfwm(obuf, ebuf); break; + case 14: kc::strifwm(obuf, ebuf); break; + case 15: kc::strbwm(obuf, ebuf); break; + case 16: kc::stribwm(obuf, ebuf); break; + case 17: kc::strutflen(obuf); break; + } + delete[] obuf; + delete[] ebuf; + kc::ZLIB::Mode zmode; + switch (myrand(3)) { + default: zmode = kc::ZLIB::RAW; break; + case 0: zmode = kc::ZLIB::DEFLATE; break; + case 1: zmode = kc::ZLIB::GZIP; break; + } + size_t zsiz; + char* zbuf = kc::ZLIB::compress(ubuf, usiz, &zsiz, zmode); + if (zbuf) { + obuf = kc::ZLIB::decompress(zbuf, zsiz, &osiz, zmode); + if (obuf) { + if (osiz != usiz || std::memcmp(obuf, ubuf, osiz)) { + errprint(__LINE__, "ZLIB::decompress"); + err = true; + } + delete[] obuf; + } else { + errprint(__LINE__, "ZLIB::decompress"); + err = true; + } + delete[] zbuf; + } else { + errprint(__LINE__, "ZLIB::compress"); + err = true; + } + zbuf = kc::LZO::compress(ubuf, usiz, &zsiz); + if (zbuf) { + obuf = kc::LZO::decompress(zbuf, zsiz, &osiz); + if (obuf) { + if (osiz != usiz || std::memcmp(obuf, ubuf, osiz)) { + errprint(__LINE__, "LZO::decompress"); + err = true; + } + delete[] obuf; + } else { + errprint(__LINE__, "LZO::decompress"); + err = true; + } + delete[] zbuf; + } else { + errprint(__LINE__, "LZO::compress"); + err = true; + } + std::string ustr((char*)ubuf, usiz); + kc::Regex::match(ustr, ".(\x80)."); + kc::Regex::replace(ustr, ".(\x80).", "[$0$1$2$&]"); + if (rnum > 250 && i % (rnum / 250) == 0) { + oputchar('.'); + if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08lld)\n", (long long)i); + } + } + oprintf("time: %.3f\n", kc::time() - stime); + oprintf("%s\n\n", err ? "error" : "ok"); + return err ? 1 : 0; +} + + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.def b/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.def new file mode 100644 index 0000000000..18a68edb82 --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.def @@ -0,0 +1,67 @@ +EXPORTS
+ KCVERSION = KCVERSION CONSTANT
+ KCVISNOP = KCVISNOP CONSTANT
+ KCVISREMOVE = KCVISREMOVE CONSTANT
+ kcatof = kcatof
+ kcatoi = kcatoi
+ kcatoix = kcatoix
+ kcchkinf = kcchkinf
+ kcchknan = kcchknan
+ kccuraccept = kccuraccept
+ kccurdb = kccurdb
+ kccurdel = kccurdel
+ kccurecode = kccurecode
+ kccuremsg = kccuremsg
+ kccurget = kccurget
+ kccurgetkey = kccurgetkey
+ kccurgetvalue = kccurgetvalue
+ kccurjump = kccurjump
+ kccurjumpback = kccurjumpback
+ kccurjumpbackkey = kccurjumpbackkey
+ kccurjumpkey = kccurjumpkey
+ kccurremove = kccurremove
+ kccursetvalue = kccursetvalue
+ kccurstep = kccurstep
+ kccurstepback = kccurstepback
+ kcdbaccept = kcdbaccept
+ kcdbadd = kcdbadd
+ kcdbappend = kcdbappend
+ kcdbbegintran = kcdbbegintran
+ kcdbbegintrantry = kcdbbegintrantry
+ kcdbcas = kcdbcas
+ kcdbclear = kcdbclear
+ kcdbclose = kcdbclose
+ kcdbcopy = kcdbcopy
+ kcdbcount = kcdbcount
+ kcdbcursor = kcdbcursor
+ kcdbdel = kcdbdel
+ kcdbdumpsnap = kcdbdumpsnap
+ kcdbecode = kcdbecode
+ kcdbemsg = kcdbemsg
+ kcdbendtran = kcdbendtran
+ kcdbget = kcdbget
+ kcdbgetbuf = kcdbgetbuf
+ kcdbincrdouble = kcdbincrdouble
+ kcdbincrint = kcdbincrint
+ kcdbiterate = kcdbiterate
+ kcdbloadsnap = kcdbloadsnap
+ kcdbmatchprefix = kcdbmatchprefix
+ kcdbmatchregex = kcdbmatchregex
+ kcdbmerge = kcdbmerge
+ kcdbnew = kcdbnew
+ kcdbopen = kcdbopen
+ kcdbpath = kcdbpath
+ kcdbremove = kcdbremove
+ kcdbreplace = kcdbreplace
+ kcdbset = kcdbset
+ kcdbsize = kcdbsize
+ kcdbstatus = kcdbstatus
+ kcdbsync = kcdbsync
+ kcecodename = kcecodename
+ kcfree = kcfree
+ kchashfnv = kchashfnv
+ kchashmurmur = kchashmurmur
+ kcinf = kcinf
+ kcmalloc = kcmalloc
+ kcnan = kcnan
+ kctime = kctime
diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.idl b/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.idl new file mode 100644 index 0000000000..41234ad13a --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.idl @@ -0,0 +1,263 @@ +/************************************************************************************************* + * IDL for bindings of scripting languages + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +/** + * namespace of Kyoto Cabinet + */ +module kyotocabinet { + //---------------------------------------------------------------- + // prediction + //---------------------------------------------------------------- + interface List; + interface Map; + interface Error; + interface Visitor; + interface FileProcessor; + interface Logger; + interface Cursor; + interface DB; + //---------------------------------------------------------------- + // list of strings (substituted by the native mechanism) + //---------------------------------------------------------------- + interface List { + string get(in long index); + }; + //---------------------------------------------------------------- + // map of strings (substituted by the native mechanism) + //---------------------------------------------------------------- + interface Map { + string get(in string key); + }; + //---------------------------------------------------------------- + // error information + //---------------------------------------------------------------- + interface Error { + const long SUCCESS = 0; + const long NOIMPL = 1; + const long INVALID = 2; + const long NOREPOS = 3; + const long NOPERM = 4; + const long BROKEN = 5; + const long DUPREC = 6; + const long NOREC = 7; + const long LOGIC = 8; + const long SYSTEM = 9; + const long MISC = 15; + long code(); + string name(); + string message(); + }; + //---------------------------------------------------------------- + // record visitor + //---------------------------------------------------------------- + interface Visitor { + const string NOP = ""; + const string REMOVE = ""; + string visit_full(in string key, in string value); + string visit_empty(in string key); + }; + //---------------------------------------------------------------- + // file processor + //---------------------------------------------------------------- + interface FileProcessor { + boolean process(in string path, in long long count, in long long size); + }; + //---------------------------------------------------------------- + // event logger + //---------------------------------------------------------------- + interface Logger { + const long INFO = 0; + const long WARN = 1; + const long ERROR = 2; + void log(in string file, in long line, in string func, in long kind, in string message); + }; + //---------------------------------------------------------------- + // meta operation trigger + //---------------------------------------------------------------- + interface MetaTrigger { + const long OPEN = 0; + const long CLOSE = 1; + const long CLEAR = 2; + const long ITERATE = 3; + const long SYNCHRONIZE = 4; + const long OCCUPY = 5; + const long BEGINTRAN = 6; + const long COMMITTRAN = 7; + const long ABORTTRAN = 8; + const long MISC = 15; + void trigger(in long kind, in string message); + }; + //---------------------------------------------------------------- + // cursor + //---------------------------------------------------------------- + interface Cursor { + boolean accept(inout Visitor visitor, in boolean writable, in boolean step); + boolean set_value(in string value, in boolean step); + boolean remove(); + string get_key(in boolean step); + string get_value(in boolean step); + boolean jump(); + boolean jump_(in string key); + boolean jump_back(); + boolean jump_back_(in string key); + boolean step(); + boolean step_back(); + DB db(); + Error error(); + }; + //---------------------------------------------------------------- + // common database operations + //---------------------------------------------------------------- + interface DB { + const long OREADER = 1 << 0; + const long OWRITER = 1 << 1; + const long OCREATE = 1 << 2; + const long OTRUNCATE = 1 << 3; + const long OAUTOTRAN = 1 << 4; + const long OAUTOSYNC = 1 << 5; + const long ONOLOCK = 1 << 6; + const long OTRYLOCK = 1 << 7; + const long ONOREPAIR = 1 << 8; + Error error(); + boolean open(in string path, in long mode); + boolean close(); + boolean accept(in string key, inout Visitor visitor, in boolean writable); + boolean accept_bulk(in List keys, inout Visitor visitor, in boolean writable); + boolean iterate(inout Visitor visitor, in boolean writable); + boolean scan_parallel(inout Visitor visitor, in long thnum); + boolean set(in string key, in string value); + boolean add(in string key, in string value); + boolean replace(in string key, in string value); + boolean append(in string key, in string value); + long long increment(in string key, in long long num, in long long orig); + double increment_double(in string key, in double num, in double orig); + boolean cas(in string key, in string oval, in string nval); + boolean remove(in string key); + string get(in string key); + long check(in string key); + string seize(in string key); + long long set_bulk(in Map recs); + long long remove_bulk(in List keys); + Map get_bulk(in List keys); + boolean clear(); + boolean synchronize(in boolean hard, inout FileProcessor proc); + boolean occupy(in boolean writable, inout FileProcessor proc); + boolean copy(in string dest); + boolean begin_transaction(in boolean hard); + boolean end_transaction(in boolean commit); + boolean dump_snapshot(in string dest); + boolean load_snapshot(in string src); + long long count(); + long long size(); + string path(); + Map status(); + Cursor cursor(); + boolean tune_logger(inout Logger logger); + boolean tune_meta_trigger(inout MetaTrigger trigger); + }; + //---------------------------------------------------------------- + // prototype hash database + //---------------------------------------------------------------- + interface ProtoHashDB :DB { + }; + //---------------------------------------------------------------- + // prototype tree database + //---------------------------------------------------------------- + interface ProtoTreeDB :DB { + }; + //---------------------------------------------------------------- + // stash database + //---------------------------------------------------------------- + interface StashDB :DB { + boolean tune_buckets(in long long bnum); + }; + //---------------------------------------------------------------- + // cache hash database + //---------------------------------------------------------------- + interface CacheDB :DB { + boolean tune_options(in long opts); + boolean tune_buckets(in long long bnum); + boolean cap_count(in long long count); + boolean cap_size(in long long size); + }; + //---------------------------------------------------------------- + // cache tree database + //---------------------------------------------------------------- + interface GrassDB :DB { + boolean tune_options(in long opts); + boolean tune_buckets(in long long bnum); + boolean tune_page(in long psiz); + boolean tune_page_cache(in long long pccap); + }; + //---------------------------------------------------------------- + // file hash database + //---------------------------------------------------------------- + interface HashDB :DB { + const long TSMALL = 1 << 0; + const long TLINEAR = 1 << 1; + const long TCOMPRESS = 1 << 2; + boolean tune_alignment(in long apow); + boolean tune_fbp(in long fpow); + boolean tune_options(in long opts); + boolean tune_buckets(in long long bnum); + boolean tune_map(in long long msiz); + boolean tune_defrag(in long dfunit); + }; + //---------------------------------------------------------------- + // file tree database + //---------------------------------------------------------------- + interface TreeDB :DB { + const long TSMALL = 1 << 0; + const long TLINEAR = 1 << 1; + const long TCOMPRESS = 1 << 2; + boolean tune_alignment(in long apow); + boolean tune_fbp(in long fpow); + boolean tune_options(in long opts); + boolean tune_buckets(in long long bnum); + boolean tune_page(in long psiz); + boolean tune_map(in long long msiz); + boolean tune_defrag(in long dfunit); + boolean tune_page_cache(in long long pccap); + }; + //---------------------------------------------------------------- + // directory hash database + //---------------------------------------------------------------- + interface DirDB :DB { + const long TCOMPRESS = 1 << 2; + boolean tune_options(in long opts); + }; + //---------------------------------------------------------------- + // directory tree database + //---------------------------------------------------------------- + interface ForestDB :DB { + const long TCOMPRESS = 1 << 2; + boolean tune_options(in long opts); + boolean tune_buckets(in long long bnum); + boolean tune_page(in long psiz); + boolean tune_page_cache(in long long pccap); + }; + //---------------------------------------------------------------- + // polymorphic database + //---------------------------------------------------------------- + interface PolyDB :DB { + List match_prefix(in string prefix, in long long max); + List match_regex(in string regex, in long long max); + List match_similar(in string origin, in long long range, in boolean utf, in long long max); + }; +}; + + + +/* END OF FILE */ diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.pc.in b/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.pc.in new file mode 100644 index 0000000000..a4459204ec --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/kyotocabinet.pc.in @@ -0,0 +1,15 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +datarootdir = @datarootdir@ +bindir=@bindir@ +libdir=@libdir@ +libexecdir=@libexecdir@ +includedir=@includedir@ +datadir=@datadir@ + +Name: Kyoto Cabinet +Description: a straightforward implementation of DBM +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lkyotocabinet +Libs.private: @LIBS@ +Cflags: -I${includedir} diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/myconf.h b/plugins/Dbx_kyoto/src/kyotocabinet/myconf.h new file mode 100644 index 0000000000..3bb8dd2f3b --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/myconf.h @@ -0,0 +1,242 @@ +/************************************************************************************************* + * System-dependent configurations + * Copyright (C) 2009-2012 FAL Labs + * This file is part of Kyoto Cabinet. + * This program is free software: you can redistribute it and/or modify it under the terms of + * the GNU General Public License as published by the Free Software Foundation, either version + * 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see <http://www.gnu.org/licenses/>. + *************************************************************************************************/ + + +#ifndef _MYCONF_H // duplication check +#define _MYCONF_H + + + +/************************************************************************************************* + * system discrimination + *************************************************************************************************/ + + +#if defined(__linux__) + +#define _SYS_LINUX_ +#define _KC_OSNAME "Linux" + +#elif defined(__FreeBSD__) + +#define _SYS_FREEBSD_ +#define _KC_OSNAME "FreeBSD" + +#elif defined(__NetBSD__) + +#define _SYS_NETBSD_ +#define _KC_OSNAME "NetBSD" + +#elif defined(__OpenBSD__) + +#define _SYS_OPENBSD_ +#define _KC_OSNAME "OpenBSD" + +#elif defined(__sun__) || defined(__sun) + +#define _SYS_SUNOS_ +#define _KC_OSNAME "SunOS" + +#elif defined(__hpux) + +#define _SYS_HPUX_ +#define _KC_OSNAME "HP-UX" + +#elif defined(__osf) + +#define _SYS_TRU64_ +#define _KC_OSNAME "Tru64" + +#elif defined(_AIX) + +#define _SYS_AIX_ +#define _KC_OSNAME "AIX" + +#elif defined(__APPLE__) && defined(__MACH__) + +#define _SYS_MACOSX_ +#define _KC_OSNAME "Mac OS X" + +#elif defined(_MSC_VER) + +#define _SYS_MSVC_ +#define _KC_OSNAME "Windows (VC++)" + +#elif defined(_WIN32) + +#define _SYS_MINGW_ +#define _KC_OSNAME "Windows (MinGW)" + +#elif defined(__CYGWIN__) + +#define _SYS_CYGWIN_ +#define _KC_OSNAME "Windows (Cygwin)" + +#else + +#define _SYS_GENERIC_ +#define _KC_OSNAME "Generic" + +#endif + +#define _KC_VERSION "1.2.76" +#define _KC_LIBVER 16 +#define _KC_LIBREV 13 +#define _KC_FMTVER 5 + +#if defined(_MYBIGEND) +#define _KC_BIGEND 1 +#else +#define _KC_BIGEND 0 +#endif + +#if defined(_MYGCCATOMIC) +#define _KC_GCCATOMIC 1 +#else +#define _KC_GCCATOMIC 0 +#endif + +#if defined(_MYZLIB) +#define _KC_ZLIB 1 +#else +#define _KC_ZLIB 0 +#endif + +#if defined(_MYLZO) +#define _KC_LZO 1 +#else +#define _KC_LZO 0 +#endif + +#if defined(_MYLZMA) +#define _KC_LZMA 1 +#else +#define _KC_LZMA 0 +#endif + +#if defined(_SYS_MSVC_) +#define _KC_PXREGEX 0 +#else +#define _KC_PXREGEX 1 +#endif + + + +/************************************************************************************************* + * notation of the file system + *************************************************************************************************/ + + +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + +#define MYPATHCHR '\\' +#define MYPATHSTR "\\" +#define MYEXTCHR '.' +#define MYEXTSTR "." +#define MYCDIRSTR "." +#define MYPDIRSTR ".." + +#else + +#define MYPATHCHR '/' +#define MYPATHSTR "/" +#define MYEXTCHR '.' +#define MYEXTSTR "." +#define MYCDIRSTR "." +#define MYPDIRSTR ".." + +#endif + + + +/************************************************************************************************* + * general headers + *************************************************************************************************/ + + +extern "C" { +#include <assert.h> +#include <ctype.h> +#include <errno.h> +#include <float.h> +#include <limits.h> +#include <locale.h> +#include <math.h> +#include <setjmp.h> +#include <stdarg.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> +#include <string.h> +#include <time.h> +} + +extern "C" { +#include <stdint.h> +} + +#if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) + +#include <windows.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <direct.h> +#include <io.h> +#include <process.h> + +#else + +extern "C" { +#include <unistd.h> +#include <sys/param.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/mman.h> +#include <sys/time.h> +#include <sys/times.h> +#include <sys/wait.h> +#include <sys/resource.h> +#include <fcntl.h> +#include <dirent.h> +} + +extern "C" { +#include <pthread.h> +#include <sched.h> +} + +#endif + +#if defined(_SYS_FREEBSD_) || defined(_SYS_OPENBSD_) || defined(_SYS_NETBSD_) || \ + defined(_SYS_MACOSX_) +#define pthread_spinlock_t pthread_mutex_t +#define pthread_spin_init(KC_a, KC_b) \ + pthread_mutex_init(KC_a, NULL) +#define pthread_spin_destroy(KC_a) \ + pthread_mutex_destroy(KC_a) +#define pthread_spin_lock(KC_a) \ + pthread_mutex_lock(KC_a) +#define pthread_spin_trylock(KC_a) \ + pthread_mutex_trylock(KC_a) +#define pthread_spin_unlock(KC_a) \ + pthread_mutex_unlock(KC_a) +#endif + + +#endif // duplication check + + +// END OF FILE diff --git a/plugins/Dbx_kyoto/src/kyotocabinet/overview b/plugins/Dbx_kyoto/src/kyotocabinet/overview new file mode 100644 index 0000000000..18e7e2a07c --- /dev/null +++ b/plugins/Dbx_kyoto/src/kyotocabinet/overview @@ -0,0 +1,327 @@ +/** + +@mainpage Kyoto Cabinet: a straightforward implementation of DBM + +@section Introduction + +Kyoto Cabinet is a library of routines for managing a database. The database is a simple data file containing records, each is a pair of a key and a value. Every key and value is serial bytes with variable length. Both binary data and character string can be used as a key and a value. Each key must be unique within a database. There is neither concept of data tables nor data types. Records are organized in hash table or B+ tree. + +The following access methods are provided to the database: storing a record with a key and a value, deleting a record by a key, retrieving a record by a key. Moreover, traversal access to every key are provided. These access methods are similar to ones of the original DBM (and its followers: NDBM and GDBM) library defined in the UNIX standard. Kyoto Cabinet is an alternative for the DBM because of its higher performance. + +Each operation of the hash database has the time complexity of "O(1)". Therefore, in theory, the performance is constant regardless of the scale of the database. In practice, the performance is determined by the speed of the main memory or the storage device. If the size of the database is less than the capacity of the main memory, the performance will seem on-memory speed, which is faster than std::map of STL. Of course, the database size can be greater than the capacity of the main memory and the upper limit is 8 exabytes. Even in that case, each operation needs only one or two seeking of the storage device. + +Each operation of the B+ tree database has the time complexity of "O(log N)". Therefore, in theory, the performance is logarithmic to the scale of the database. Although the performance of random access of the B+ tree database is slower than that of the hash database, the B+ tree database supports sequential access in order of the keys, which realizes forward matching search for strings and range search for integers. The performance of sequential access is much faster than that of random access. + +As the API is based on object-oriented design, the hash database and the the B+ tree database have same methods which inherited from the upper abstract class. Beside them, seven kinds of databases are provided under the same base class. The prototype hash database is powered by the standard container of std::unordered_map. The prototype tree database is powered by the standard container of std::map. The stash database is powered by the original implementation of naive hash map saving memory. The cache hash database is powered by the original implementation of doubly-linked hash map with LRU deletion algorithm. The cache tree database is powered by the cache hash database and provides B+ tree mechanism. The directory hash database is powered by the directory mechanism of the file system and stores records as respective files in a directory. The directory tree database is powered by the directory hash database and provides B+ tree mechanism. All databases have practical utility methods related to transaction and cursor. Programs for command line interface are also included in the package. + +All databases have practical utility methods related to transaction and cursor. Programs for command line interface are also included in the package. + +The following classes are most important. If you are new to Kyoto Cabinet, learn the polymorphic database first. + +@li kyotocabinet::BasicDB -- common interface of concrete databases +@li kyotocabinet::ProtoHashDB -- on-memory hash database based on std::unordered_map +@li kyotocabinet::ProtoTreeDB -- on-memory hash database based on std::map +@li kyotocabinet::StashDB -- economical on-memory hash database for cache. +@li kyotocabinet::CacheDB -- on-memory hash database for cache with LRU deletion +@li kyotocabinet::GrassDB -- on-memory tree database for cache in order +@li kyotocabinet::HashDB -- file hash database, which implements hash table on a file +@li kyotocabinet::TreeDB -- file tree database, which implements B+ tree on a file +@li kyotocabinet::DirDB -- directory hash database, which handles respective files in a directory +@li kyotocabinet::ForestDB -- directory tree database, which implements B+ tree on a directory +@li kyotocabinet::TextDB -- plain text database, which treats a text file as a database +@li kyotocabinet::PolyDB -- polymorphic database, dynamic binding of the above databases + +@li kyotocabinet::MapReduce -- MapReduce framework to process records in each database +@li kyotocabinet::IndexDB -- wrapper for efficient appending operations to each database + +@li kclangc.h -- C language binding of the polymorphic database + +See the project homepage ( http://fallabs.com/kyotocabinet/ ) for details. + +@section Example + +The following code is an example to use a polymorphic database. + +@code +#include <kcpolydb.h> + +using namespace std; +using namespace kyotocabinet; + +// main routine +int main(int argc, char** argv) { + + // create the database object + PolyDB db; + + // open the database + if (!db.open("casket.kch", PolyDB::OWRITER | PolyDB::OCREATE)) { + cerr << "open error: " << db.error().name() << endl; + } + + // store records + if (!db.set("foo", "hop") || + !db.set("bar", "step") || + !db.set("baz", "jump")) { + cerr << "set error: " << db.error().name() << endl; + } + + // retrieve a record + string value; + if (db.get("foo", &value)) { + cout << value << endl; + } else { + cerr << "get error: " << db.error().name() << endl; + } + + // traverse records + DB::Cursor* cur = db.cursor(); + cur->jump(); + string ckey, cvalue; + while (cur->get(&ckey, &cvalue, true)) { + cout << ckey << ":" << cvalue << endl; + } + delete cur; + + // close the database + if (!db.close()) { + cerr << "close error: " << db.error().name() << endl; + } + + return 0; +} +@endcode + +The following code is a more complex example, which uses the Visitor pattern. + +@code +#include <kcpolydb.h> + +using namespace std; +using namespace kyotocabinet; + +// main routine +int main(int argc, char** argv) { + + // create the database object + PolyDB db; + + // open the database + if (!db.open("casket.kch", PolyDB::OREADER)) { + cerr << "open error: " << db.error().name() << endl; + } + + // define the visitor + class VisitorImpl : public DB::Visitor { + // call back function for an existing record + const char* visit_full(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t *sp) { + cout << string(kbuf, ksiz) << ":" << string(vbuf, vsiz) << endl; + return NOP; + } + // call back function for an empty record space + const char* visit_empty(const char* kbuf, size_t ksiz, size_t *sp) { + cerr << string(kbuf, ksiz) << " is missing" << endl; + return NOP; + } + } visitor; + + // retrieve a record with visitor + if (!db.accept("foo", 3, &visitor, false) || + !db.accept("dummy", 5, &visitor, false)) { + cerr << "accept error: " << db.error().name() << endl; + } + + // traverse records with visitor + if (!db.iterate(&visitor, false)) { + cerr << "iterate error: " << db.error().name() << endl; + } + + // close the database + if (!db.close()) { + cerr << "close error: " << db.error().name() << endl; + } + + return 0; +} +@endcode + +The following code is an example of word counting with the MapReduce framework. + +@code +#include <kcpolydb.h> +#include <kcdbext.h> + +using namespace std; +using namespace kyotocabinet; + +// main routine +int main(int argc, char** argv) { + + // create the database object + PolyDB db; + + // open the database + if (!db.open()) { + cerr << "open error: " << db.error().name() << endl; + } + + // store records + db.set("1", "this is a pen"); + db.set("2", "what a beautiful pen this is"); + db.set("3", "she is beautiful"); + + // define the mapper and the reducer + class MapReduceImpl : public MapReduce { + // call back function of the mapper + bool map(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { + vector<string> words; + strsplit(string(vbuf, vsiz), ' ', &words); + for (vector<string>::iterator it = words.begin(); + it != words.end(); it++) { + emit(it->data(), it->size(), "", 0); + } + return true; + } + // call back function of the reducer + bool reduce(const char* kbuf, size_t ksiz, ValueIterator* iter) { + size_t count = 0; + const char* vbuf; + size_t vsiz; + while ((vbuf = iter->next(&vsiz)) != NULL) { + count++; + } + cout << string(kbuf, ksiz) << ": " << count << endl; + return true; + } + } mr; + + // execute the MapReduce process + if (!mr.execute(&db)) { + cerr << "MapReduce error: " << db.error().name() << endl; + } + + // close the database + if (!db.close()) { + cerr << "close error: " << db.error().name() << endl; + } + + return 0; +} +@endcode + +The C language binding is also provided as a wrapper of the polymorphic database API. The following code is an example. + +@code +#include <kclangc.h> + +/* call back function for an existing record */ +const char* visitfull(const char* kbuf, size_t ksiz, + const char* vbuf, size_t vsiz, size_t *sp, void* opq) { + fwrite(kbuf, 1, ksiz, stdout); + printf(":"); + fwrite(vbuf, 1, vsiz, stdout); + printf("\n"); + return KCVISNOP; +} + +/* call back function for an empty record space */ +const char* visitempty(const char* kbuf, size_t ksiz, size_t *sp, void* opq) { + fwrite(kbuf, 1, ksiz, stdout); + printf(" is missing\n"); + return KCVISNOP; +} + +/* main routine */ +int main(int argc, char** argv) { + KCDB* db; + KCCUR* cur; + char *kbuf, *vbuf; + size_t ksiz, vsiz; + const char *cvbuf; + + /* create the database object */ + db = kcdbnew(); + + /* open the database */ + if (!kcdbopen(db, "casket.kch", KCOWRITER | KCOCREATE)) { + fprintf(stderr, "open error: %s\n", kcecodename(kcdbecode(db))); + } + + /* store records */ + if (!kcdbset(db, "foo", 3, "hop", 3) || + !kcdbset(db, "bar", 3, "step", 4) || + !kcdbset(db, "baz", 3, "jump", 4)) { + fprintf(stderr, "set error: %s\n", kcecodename(kcdbecode(db))); + } + + /* retrieve a record */ + vbuf = kcdbget(db, "foo", 3, &vsiz); + if (vbuf) { + printf("%s\n", vbuf); + kcfree(vbuf); + } else { + fprintf(stderr, "get error: %s\n", kcecodename(kcdbecode(db))); + } + + /* traverse records */ + cur = kcdbcursor(db); + kccurjump(cur); + while ((kbuf = kccurget(cur, &ksiz, &cvbuf, &vsiz, 1)) != NULL) { + printf("%s:%s\n", kbuf, cvbuf); + kcfree(kbuf); + } + kccurdel(cur); + + /* retrieve a record with visitor */ + if (!kcdbaccept(db, "foo", 3, visitfull, visitempty, NULL, 0) || + !kcdbaccept(db, "dummy", 5, visitfull, visitempty, NULL, 0)) { + fprintf(stderr, "accept error: %s\n", kcecodename(kcdbecode(db))); + } + + /* traverse records with visitor */ + if (!kcdbiterate(db, visitfull, NULL, 0)) { + fprintf(stderr, "iterate error: %s\n", kcecodename(kcdbecode(db))); + } + + /* close the database */ + if (!kcdbclose(db)) { + fprintf(stderr, "close error: %s\n", kcecodename(kcdbecode(db))); + } + + /* delete the database object */ + kcdbdel(db); + + return 0; +} +@endcode + +*/ + + +/** + * Common namespace of Kyoto Cabinet. + */ +namespace kyotocabinet {} + + +/** + * @file kccommon.h common symbols for the library + * @file kcutil.h utility functions + * @file kcdb.h database interface + * @file kcthread.h threading devices + * @file kcfile.h filesystem abstraction + * @file kccompress.h data compressor and decompressor + * @file kccompare.h comparator functions + * @file kcmap.h data mapping structures + * @file kcregex.h regular expression + * @file kcplantdb.h plant database + * @file kcprotodb.h prototype database + * @file kccachedb.h cache hash database + * @file kchashdb.h file hash database + * @file kcdirdb.h directory hash database + * @file kctextdb.h plain text database + * @file kcpolydb.h polymorphic database + * @file kcdbext.h database extension + * @file kclangc.h C language binding + */ diff --git a/plugins/Dbx_kyoto/src/resource.h b/plugins/Dbx_kyoto/src/resource.h new file mode 100644 index 0000000000..be74c3e5e8 --- /dev/null +++ b/plugins/Dbx_kyoto/src/resource.h @@ -0,0 +1,32 @@ +//{{NO_DEPENDENCIES}}
+// Microsoft Visual C++ generated include file.
+// Used by D:\Myranda\plugins\Db3x_mmap\res\db3x_mmap.rc
+//
+
+#define IDREMOVE 3
+
+#define IDI_ICONPASS 100
+#define IDI_LOGO 101
+#define IDD_LOGIN 102
+#define IDD_NEWPASS 103
+#define IDD_CHANGEPASS 104
+#define IDD_OPTIONS 105
+#define IDC_HEADERBAR 1001
+#define IDC_LANG 1002
+#define IDC_USERPASS 1003
+#define IDC_USERPASS1 1004
+#define IDC_USERPASS2 1005
+#define IDC_OLDPASS 1006
+#define IDC_STANDARD 1007
+#define IDC_TOTAL 1008
+
+// Next default values for new objects
+//
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_NEXT_RESOURCE_VALUE 106
+#define _APS_NEXT_COMMAND_VALUE 40001
+#define _APS_NEXT_CONTROL_VALUE 1009
+#define _APS_NEXT_SYMED_VALUE 101
+#endif
+#endif
diff --git a/plugins/Dbx_kyoto/src/stdafx.cpp b/plugins/Dbx_kyoto/src/stdafx.cpp new file mode 100644 index 0000000000..048b14e9d2 --- /dev/null +++ b/plugins/Dbx_kyoto/src/stdafx.cpp @@ -0,0 +1,18 @@ +/*
+Copyright (C) 2012-15 Miranda NG project (http://miranda-ng.org)
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation version 2
+of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "commonheaders.h"
\ No newline at end of file diff --git a/plugins/Dbx_kyoto/src/ui.cpp b/plugins/Dbx_kyoto/src/ui.cpp new file mode 100644 index 0000000000..a4ba3d12c5 --- /dev/null +++ b/plugins/Dbx_kyoto/src/ui.cpp @@ -0,0 +1,340 @@ +/*
+
+Miranda NG: the free IM client for Microsoft* Windows*
+
+Copyright (ñ) 2012-15 Miranda NG project (http://miranda-ng.org)
+all portions of this codebase are copyrighted to the people
+listed in contributors.txt.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+*/
+
+#include "commonheaders.h"
+
+struct DlgChangePassParam
+{
+ CDbxKV *db;
+ TCHAR newPass[100];
+ int wrongPass;
+};
+
+#define MS_DB_CHANGEPASSWORD "DB/UI/ChangePassword"
+
+static IconItem iconList[] =
+{
+ { LPGEN("Logo"), "logo", IDI_LOGO },
+ { LPGEN("Password"), "password", IDI_ICONPASS }
+};
+
+static HGENMENU hSetPwdMenu;
+
+static UINT oldLangID;
+void LanguageChanged(HWND hwndDlg)
+{
+ UINT LangID = (UINT)GetKeyboardLayout(0);
+ char Lang[3] = { 0 };
+ if (LangID != oldLangID) {
+ oldLangID = LangID;
+ GetLocaleInfoA(MAKELCID((LangID & 0xffffffff), SORT_DEFAULT), LOCALE_SABBREVLANGNAME, Lang, 2);
+ Lang[0] = toupper(Lang[0]);
+ Lang[1] = tolower(Lang[1]);
+ SetDlgItemTextA(hwndDlg, IDC_LANG, Lang);
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+static INT_PTR CALLBACK sttEnterPassword(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+{
+ DlgChangePassParam *param = (DlgChangePassParam*)GetWindowLongPtr(hwndDlg, GWLP_USERDATA);
+
+ switch (uMsg) {
+ case WM_INITDIALOG:
+ TranslateDialogDefault(hwndDlg);
+ SendDlgItemMessage(hwndDlg, IDC_HEADERBAR, WM_SETICON, ICON_SMALL, (LPARAM)LoadIcon(g_hInst, MAKEINTRESOURCE(iconList[0].defIconID)));
+
+ param = (DlgChangePassParam*)lParam;
+ SetWindowLongPtr(hwndDlg, GWLP_USERDATA, lParam);
+
+ if (param->wrongPass) {
+ if (param->wrongPass > 2) {
+ HWND hwndCtrl = GetDlgItem(hwndDlg, IDC_USERPASS);
+ EnableWindow(hwndCtrl, FALSE);
+ hwndCtrl = GetDlgItem(hwndDlg, IDOK);
+ EnableWindow(hwndCtrl, FALSE);
+ SetDlgItemText(hwndDlg, IDC_HEADERBAR, TranslateT("Too many errors!"));
+ }
+ else SetDlgItemText(hwndDlg, IDC_HEADERBAR, TranslateT("Password is not correct!"));
+ }
+ else SetDlgItemText(hwndDlg, IDC_HEADERBAR, TranslateT("Please type in your password"));
+
+ oldLangID = 0;
+ SetTimer(hwndDlg, 1, 200, NULL);
+ LanguageChanged(hwndDlg);
+ return TRUE;
+
+ case WM_CTLCOLORSTATIC:
+ if ((HWND)lParam == GetDlgItem(hwndDlg, IDC_LANG)) {
+ SetTextColor((HDC)wParam, GetSysColor(COLOR_HIGHLIGHTTEXT));
+ SetBkMode((HDC)wParam, TRANSPARENT);
+ return (BOOL)GetSysColorBrush(COLOR_HIGHLIGHT);
+ }
+ return FALSE;
+
+ case WM_COMMAND:
+ switch (LOWORD(wParam)) {
+ case IDCANCEL:
+ EndDialog(hwndDlg, IDCANCEL);
+ break;
+
+ case IDOK:
+ GetDlgItemText(hwndDlg, IDC_USERPASS, param->newPass, SIZEOF(param->newPass));
+ EndDialog(hwndDlg, IDOK);
+ }
+ break;
+
+ case WM_TIMER:
+ LanguageChanged(hwndDlg);
+ return FALSE;
+
+ case WM_DESTROY:
+ KillTimer(hwndDlg, 1);
+ DestroyIcon((HICON)SendMessage(hwndDlg, WM_GETICON, ICON_SMALL, 0));
+ }
+
+ return FALSE;
+}
+
+bool CDbxKV::EnterPassword(const BYTE *pKey, const size_t keyLen)
+{
+ DlgChangePassParam param = { this };
+ while (true) {
+ // Esc pressed
+ if (IDOK != DialogBoxParam(g_hInst, MAKEINTRESOURCE(IDD_LOGIN), 0, sttEnterPassword, (LPARAM)¶m))
+ return false;
+
+ m_crypto->setPassword(ptrA(mir_utf8encodeT(param.newPass)));
+ if (m_crypto->setKey(pKey, keyLen)) {
+ m_bUsesPassword = true;
+ SecureZeroMemory(¶m, sizeof(param));
+ return true;
+ }
+
+ param.wrongPass++;
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+static bool CheckOldPassword(HWND hwndDlg, CDbxKV *db)
+{
+ if (db->usesPassword()) {
+ TCHAR buf[100];
+ GetDlgItemText(hwndDlg, IDC_OLDPASS, buf, SIZEOF(buf));
+ ptrA oldPass(mir_utf8encodeT(buf));
+ if (!db->m_crypto->checkPassword(oldPass)) {
+ SetDlgItemText(hwndDlg, IDC_HEADERBAR, TranslateT("Wrong old password entered!"));
+ return false;
+ }
+ }
+ return true;
+}
+
+static INT_PTR CALLBACK sttChangePassword(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
+{
+ DlgChangePassParam *param = (DlgChangePassParam*)GetWindowLongPtr(hwndDlg, GWLP_USERDATA);
+ TCHAR buf[100];
+
+ switch (uMsg) {
+ case WM_INITDIALOG:
+ TranslateDialogDefault(hwndDlg);
+ SendDlgItemMessage(hwndDlg, IDC_HEADERBAR, WM_SETICON, ICON_SMALL, (LPARAM)Skin_GetIconByHandle(iconList[0].hIcolib, true));
+
+ param = (DlgChangePassParam*)lParam;
+ SetWindowLongPtr(hwndDlg, GWLP_USERDATA, lParam);
+
+ oldLangID = 0;
+ SetTimer(hwndDlg, 1, 200, NULL);
+ LanguageChanged(hwndDlg);
+ return TRUE;
+
+ case WM_CTLCOLORSTATIC:
+ if ((HWND)lParam == GetDlgItem(hwndDlg, IDC_LANG)) {
+ SetTextColor((HDC)wParam, GetSysColor(COLOR_HIGHLIGHTTEXT));
+ SetBkMode((HDC)wParam, TRANSPARENT);
+ return (BOOL)GetSysColorBrush(COLOR_HIGHLIGHT);
+ }
+ return FALSE;
+
+ case WM_COMMAND:
+ switch (LOWORD(wParam)) {
+ case IDCANCEL:
+ EndDialog(hwndDlg, IDCANCEL);
+ break;
+
+ case IDREMOVE:
+ if (!CheckOldPassword(hwndDlg, param->db)) {
+ LBL_Error:
+ SendDlgItemMessage(hwndDlg, IDC_HEADERBAR, WM_NCPAINT, 0, 0);
+ SetDlgItemTextA(hwndDlg, IDC_USERPASS1, "");
+ SetDlgItemTextA(hwndDlg, IDC_USERPASS2, "");
+ }
+ else {
+ // param->db->WriteSignature(dbSignatureU);
+ param->db->SetPassword(NULL);
+ param->db->StoreKey();
+ EndDialog(hwndDlg, IDREMOVE);
+ }
+ break;
+
+ case IDOK:
+ TCHAR buf2[100];
+ GetDlgItemText(hwndDlg, IDC_USERPASS1, buf2, SIZEOF(buf2));
+ if (_tcslen(buf2) < 3) {
+ SetDlgItemText(hwndDlg, IDC_HEADERBAR, TranslateT("Password is too short!"));
+ goto LBL_Error;
+ }
+
+ GetDlgItemText(hwndDlg, IDC_USERPASS2, buf, SIZEOF(buf));
+ if (_tcscmp(buf2, buf)) {
+ SetDlgItemText(hwndDlg, IDC_HEADERBAR, TranslateT("Passwords do not match!"));
+ goto LBL_Error;
+ }
+
+ if (!CheckOldPassword(hwndDlg, param->db))
+ goto LBL_Error;
+
+ // param->db->WriteSignature(dbSignatureE);
+ param->db->SetPassword(buf2);
+ param->db->StoreKey();
+ SecureZeroMemory(buf2, sizeof(buf2));
+ EndDialog(hwndDlg, IDOK);
+ }
+ break;
+
+ case WM_TIMER:
+ LanguageChanged(hwndDlg);
+ return FALSE;
+
+ case WM_DESTROY:
+ KillTimer(hwndDlg, 1);
+ Skin_ReleaseIcon((HICON)SendMessage(hwndDlg, WM_GETICON, ICON_SMALL, 0));
+ }
+
+ return FALSE;
+}
+
+static INT_PTR ChangePassword(void* obj, WPARAM, LPARAM)
+{
+ CDbxKV *db = (CDbxKV*)obj;
+ DlgChangePassParam param = { db };
+ DialogBoxParam(g_hInst, MAKEINTRESOURCE(db->usesPassword() ? IDD_CHANGEPASS : IDD_NEWPASS), 0, sttChangePassword, (LPARAM)¶m);
+ return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+INT_PTR CALLBACK DlgProcOptions(HWND hwndDlg, UINT msg, WPARAM wParam, LPARAM lParam)
+{
+ CDbxKV *db = (CDbxKV *)GetWindowLongPtr(hwndDlg, GWLP_USERDATA);
+
+ switch (msg) {
+ case WM_INITDIALOG:
+ TranslateDialogDefault(hwndDlg);
+ SetWindowLongPtr(hwndDlg, GWLP_USERDATA, lParam);
+
+ db = (CDbxKV*)lParam;
+ CheckRadioButton(hwndDlg, IDC_STANDARD, IDC_TOTAL, IDC_STANDARD + db->isEncrypted());
+ return TRUE;
+
+ case WM_COMMAND:
+ if (HIWORD(wParam) == BN_CLICKED && (HWND)lParam == GetFocus()) {
+ if (LOWORD(wParam) == IDC_USERPASS)
+ CallService(MS_DB_CHANGEPASSWORD, 0, 0);
+ else
+ SendMessage(GetParent(hwndDlg), PSM_CHANGED, 0, 0);
+ }
+ break;
+
+ case WM_NOTIFY:
+ if (((LPNMHDR)lParam)->code == PSN_APPLY) {
+ if (IsDlgButtonChecked(hwndDlg, IDC_TOTAL) != (UINT)db->isEncrypted()) {
+ db->ToggleEncryption();
+ CheckRadioButton(hwndDlg, IDC_STANDARD, IDC_TOTAL, IDC_STANDARD + db->isEncrypted());
+ }
+ break;
+ }
+ break;
+ }
+
+ return FALSE;
+}
+
+static int OnOptionsInit(PVOID obj, WPARAM wParam, LPARAM)
+{
+ OPTIONSDIALOGPAGE odp = { sizeof(odp) };
+ odp.position = -790000000;
+ odp.hInstance = g_hInst;
+ odp.pszTemplate = MAKEINTRESOURCEA(IDD_OPTIONS);
+ odp.flags = ODPF_BOLDGROUPS;
+ odp.pszTitle = LPGEN("Database");
+ odp.pfnDlgProc = DlgProcOptions;
+ odp.dwInitParam = (LPARAM)obj;
+ Options_AddPage(wParam, &odp);
+ return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+void CDbxKV::UpdateMenuItem()
+{
+ CLISTMENUITEM mi = { sizeof(mi) };
+ mi.flags = CMIM_NAME;
+ mi.icolibItem = iconList[1].hIcolib;
+ mi.pszName = GetMenuTitle();
+ Menu_ModifyItem(hSetPwdMenu, &mi);
+}
+
+static int OnModulesLoaded(PVOID obj, WPARAM, LPARAM)
+{
+ CDbxKV *db = (CDbxKV*)obj;
+
+ Icon_Register(g_hInst, LPGEN("Database"), iconList, SIZEOF(iconList), "mmap");
+
+ HookEventObj(ME_OPT_INITIALISE, OnOptionsInit, db);
+
+ // main menu item
+ CLISTMENUITEM mi = { sizeof(mi) };
+ mi.pszName = LPGEN("Database");
+ mi.position = 500000000;
+ mi.flags = CMIF_ROOTHANDLE;
+ mi.icolibItem = iconList[0].hIcolib;
+ HGENMENU hMenuRoot = Menu_AddMainMenuItem(&mi);
+
+ mi.icolibItem = iconList[1].hIcolib;
+ mi.pszName = db->GetMenuTitle();
+ mi.hParentMenu = hMenuRoot;
+ mi.pszService = MS_DB_CHANGEPASSWORD;
+ hSetPwdMenu = Menu_AddMainMenuItem(&mi);
+ return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+
+void CDbxKV::InitDialogs()
+{
+ hService = CreateServiceFunctionObj(MS_DB_CHANGEPASSWORD, ChangePassword, this);
+ hHook = HookEventObj(ME_SYSTEM_MODULESLOADED, OnModulesLoaded, this);
+}
diff --git a/plugins/Dbx_kyoto/src/version.h b/plugins/Dbx_kyoto/src/version.h new file mode 100644 index 0000000000..8522a9aa5d --- /dev/null +++ b/plugins/Dbx_kyoto/src/version.h @@ -0,0 +1,14 @@ +#define __MAJOR_VERSION 0
+#define __MINOR_VERSION 95
+#define __RELEASE_NUM 4
+#define __BUILD_NUM 1
+
+#include <stdver.h>
+
+#define __PLUGIN_NAME "Miranda NG KyotoCabinet database driver"
+#define __FILENAME "Dbx_kyoto.dll"
+#define __DESCRIPTION "Provides Miranda database support: global settings, contacts, history, settings per contact."
+#define __AUTHOR "Miranda-NG project"
+#define __AUTHOREMAIL "ghazan@miranda.im"
+#define __AUTHORWEB "http://miranda-ng.org/p/Dbx_kyoto/"
+#define __COPYRIGHT "© 2015 Miranda NG project"
|