From 3ed0cd54cf2db670166af44296694a1c9679ec03 Mon Sep 17 00:00:00 2001 From: George Hazan Date: Tue, 13 Jan 2015 17:36:14 +0000 Subject: - fix for ssize_t; - various optimizations git-svn-id: http://svn.miranda-ng.org/main/trunk@11848 1316c22d-e87f-b044-9b9b-93d7a3e3ba9c --- plugins/Dbx_mdb/src/dbcontacts.cpp | 26 ++++--- plugins/Dbx_mdb/src/dbsettings.cpp | 80 +++++++++++---------- plugins/Dbx_mdb/src/lmdb/mdb.c | 142 ++++++++++++++++++++++++------------- 3 files changed, 151 insertions(+), 97 deletions(-) (limited to 'plugins/Dbx_mdb') diff --git a/plugins/Dbx_mdb/src/dbcontacts.cpp b/plugins/Dbx_mdb/src/dbcontacts.cpp index 26c245e8cb..b02c4cb596 100644 --- a/plugins/Dbx_mdb/src/dbcontacts.cpp +++ b/plugins/Dbx_mdb/src/dbcontacts.cpp @@ -85,9 +85,12 @@ STDMETHODIMP_(LONG) CDbxMdb::DeleteContact(MCONTACT contactID) NotifyEventHooks(hContactDeletedEvent, contactID, 0); // delete + mir_cslock lck(m_csDbAccess); + MDB_val key = { sizeof(DWORD), &contactID }; txn_lock trnlck(m_pMdbEnv); + mdb_open(trnlck, "contacts", MDB_INTEGERKEY, &m_dbContacts); mdb_del(trnlck, m_dbContacts, &key, NULL); trnlck.commit(); return 0; @@ -95,21 +98,26 @@ STDMETHODIMP_(LONG) CDbxMdb::DeleteContact(MCONTACT contactID) STDMETHODIMP_(MCONTACT) CDbxMdb::AddContact() { + DWORD dwContactId; + DBContact dbc; dbc.signature = DBCONTACT_SIGNATURE; dbc.eventCount = 0; - - DWORD dwContactId = m_dwMaxContactId++; + { + mir_cslock lck(m_csDbAccess); + dwContactId = m_dwMaxContactId++; - MDB_val key = { sizeof(DWORD), &dwContactId }; - MDB_val data = { sizeof(DBContact), &dbc }; + MDB_val key = { sizeof(DWORD), &dwContactId }; + MDB_val data = { sizeof(DBContact), &dbc }; - txn_lock trnlck(m_pMdbEnv); - mdb_put(trnlck, m_dbContacts, &key, &data, 0); - trnlck.commit(); + txn_lock trnlck(m_pMdbEnv); + mdb_open(trnlck, "contacts", MDB_INTEGERKEY, &m_dbContacts); + mdb_put(trnlck, m_dbContacts, &key, &data, 0); + trnlck.commit(); - DBCachedContact *cc = m_cache->AddContactToCache(dwContactId); - cc->dwDriverData = 0; + DBCachedContact *cc = m_cache->AddContactToCache(dwContactId); + cc->dwDriverData = 0; + } NotifyEventHooks(hContactAddedEvent, dwContactId, 0); return dwContactId; diff --git a/plugins/Dbx_mdb/src/dbsettings.cpp b/plugins/Dbx_mdb/src/dbsettings.cpp index 7b0825d8c9..1269c25c76 100644 --- a/plugins/Dbx_mdb/src/dbsettings.cpp +++ b/plugins/Dbx_mdb/src/dbsettings.cpp @@ -120,7 +120,7 @@ LBL_Seek: DBCachedContact *cc = (contactID) ? m_cache->GetCachedContact(contactID) : NULL; txn_lock trnlck(m_pMdbEnv); - mdb_open(trnlck, "settings", 0, &m_dbSettings); + //mdb_open(trnlck, "settings", 0, &m_dbSettings); DBSettingKey keySearch; keySearch.dwContactID = contactID; @@ -146,46 +146,46 @@ LBL_Seek: return 1; int varLen; - BYTE iType = dbv->type = pBlob[0]; + BYTE iType = dbv->type = pBlob[0]; pBlob++; switch (iType) { case DBVT_DELETED: /* this setting is deleted */ dbv->type = DBVT_DELETED; return 2; - case DBVT_BYTE: dbv->bVal = pBlob[1]; break; - case DBVT_WORD: memmove(&(dbv->wVal), (PWORD)(pBlob + 1), 2); break; - case DBVT_DWORD: memmove(&(dbv->dVal), (PDWORD)(pBlob + 1), 4); break; + case DBVT_BYTE: dbv->bVal = *pBlob; break; + case DBVT_WORD: dbv->wVal = *(WORD*)pBlob; break; + case DBVT_DWORD: dbv->dVal = *(DWORD*)pBlob; break; case DBVT_UTF8: case DBVT_ASCIIZ: - varLen = *(PWORD)(pBlob + 1); - pBlob += 3 + varLen; + varLen = *(WORD*)pBlob; + pBlob += 2; if (isStatic) { dbv->cchVal--; if (varLen < dbv->cchVal) dbv->cchVal = varLen; - memmove(dbv->pszVal, pBlob + 3, dbv->cchVal); // decode + memmove(dbv->pszVal, pBlob, dbv->cchVal); // decode dbv->pszVal[dbv->cchVal] = 0; dbv->cchVal = varLen; } else { dbv->pszVal = (char*)mir_alloc(1 + varLen); - memmove(dbv->pszVal, pBlob + 3, varLen); + memmove(dbv->pszVal, pBlob, varLen); dbv->pszVal[varLen] = 0; } break; case DBVT_BLOB: - varLen = *(PWORD)(pBlob + 1); - pBlob += 3 + varLen; + varLen = *(WORD*)pBlob; + pBlob += 2; if (isStatic) { if (varLen < dbv->cpbVal) dbv->cpbVal = varLen; - memmove(dbv->pbVal, pBlob + 3, dbv->cpbVal); + memmove(dbv->pbVal, pBlob, dbv->cpbVal); } else { dbv->pbVal = (BYTE *)mir_alloc(varLen); - memmove(dbv->pbVal, pBlob + 3, varLen); + memmove(dbv->pbVal, pBlob, varLen); } dbv->cpbVal = varLen; break; @@ -193,29 +193,29 @@ LBL_Seek: case DBVT_ENCRYPTED: if (m_crypto == NULL) return 1; - else { - varLen = *(PWORD)(pBlob + 1); - pBlob += 3 + varLen; - size_t realLen; - ptrA decoded(m_crypto->decodeString(pBlob + 3, varLen, &realLen)); - if (decoded == NULL) - return 1; - varLen = (WORD)realLen; - dbv->type = DBVT_UTF8; - if (isStatic) { - dbv->cchVal--; - if (varLen < dbv->cchVal) - dbv->cchVal = varLen; - memmove(dbv->pszVal, decoded, dbv->cchVal); - dbv->pszVal[dbv->cchVal] = 0; + varLen = *(WORD*)pBlob; + pBlob += 2; + + size_t realLen; + ptrA decoded(m_crypto->decodeString(pBlob, varLen, &realLen)); + if (decoded == NULL) + return 1; + + varLen = (WORD)realLen; + dbv->type = DBVT_UTF8; + if (isStatic) { + dbv->cchVal--; + if (varLen < dbv->cchVal) dbv->cchVal = varLen; - } - else { - dbv->pszVal = (char*)mir_alloc(1 + varLen); - memmove(dbv->pszVal, decoded, varLen); - dbv->pszVal[varLen] = 0; - } + memmove(dbv->pszVal, decoded, dbv->cchVal); + dbv->pszVal[dbv->cchVal] = 0; + dbv->cchVal = varLen; + } + else { + dbv->pszVal = (char*)mir_alloc(1 + varLen); + memmove(dbv->pszVal, decoded, varLen); + dbv->pszVal[varLen] = 0; } break; } @@ -474,7 +474,7 @@ STDMETHODIMP_(BOOL) CDbxMdb::WriteContactSetting(MCONTACT contactID, DBCONTACTWR else m_cache->GetCachedValuePtr(contactID, szCachedSettingName, -1); txn_lock trnlck(m_pMdbEnv); - mdb_open(trnlck, "settings", 0, &m_dbSettings); + //mdb_open(trnlck, "settings", 0, &m_dbSettings); DBSettingKey keySearch; keySearch.dwContactID = contactID; @@ -521,6 +521,7 @@ STDMETHODIMP_(BOOL) CDbxMdb::WriteContactSetting(MCONTACT contactID, DBCONTACTWR memcpy(pBlob, dbcwWork.value.pbVal, dbcwWork.value.cpbVal); } + trnlck.commit(); lck.unlock(); // notify @@ -555,7 +556,7 @@ STDMETHODIMP_(BOOL) CDbxMdb::DeleteContactSetting(MCONTACT contactID, LPCSTR szM char *szCachedSettingName = m_cache->GetCachedSetting(szModule, szSetting, moduleNameLen, settingNameLen); if (szCachedSettingName[-1] == 0) { // it's not a resident variable txn_lock trnlck(m_pMdbEnv); - mdb_open(trnlck, "settings", 0, &m_dbSettings); + //mdb_open(trnlck, "settings", 0, &m_dbSettings); DBSettingKey keySearch; keySearch.dwContactID = contactID; @@ -588,9 +589,9 @@ STDMETHODIMP_(BOOL) CDbxMdb::EnumContactSettings(MCONTACT contactID, DBCONTACTEN int result = 0; mir_cslock lck(m_csDbAccess); - /* + txn_lock trnlck(m_pMdbEnv); - mdb_open(trnlck, "settings", 0, &m_dbSettings); + //mdb_open(trnlck, "settings", 0, &m_dbSettings); MDB_cursor *cursor; mdb_cursor_open(trnlck, m_dbSettings, &cursor); @@ -601,6 +602,7 @@ STDMETHODIMP_(BOOL) CDbxMdb::EnumContactSettings(MCONTACT contactID, DBCONTACTEN keySearch.szSettingName[0] = 0; MDB_val key = { 2 * sizeof(DWORD), &keySearch }, data; + mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY); while (mdb_cursor_get(cursor, &key, &data, MDB_NEXT) == 0) { DBSettingKey *pKey = (DBSettingKey*)key.mv_data; if (pKey->dwContactID != contactID || pKey->dwOfsModule != keySearch.dwOfsModule) @@ -611,7 +613,7 @@ STDMETHODIMP_(BOOL) CDbxMdb::EnumContactSettings(MCONTACT contactID, DBCONTACTEN result = (dbces->pfnEnumProc)(szSetting, dbces->lParam); } - mdb_cursor_close(cursor); */ + mdb_cursor_close(cursor); return result; } diff --git a/plugins/Dbx_mdb/src/lmdb/mdb.c b/plugins/Dbx_mdb/src/lmdb/mdb.c index 996683cd08..10339c8388 100644 --- a/plugins/Dbx_mdb/src/lmdb/mdb.c +++ b/plugins/Dbx_mdb/src/lmdb/mdb.c @@ -42,6 +42,12 @@ * as int64 which is wrong. MSVC doesn't define it at all, so just * don't use it. */ +#ifdef _WIN64 +typedef signed __int64 ssize_t; +#else +typedef signed int ssize_t; +#endif + #define MDB_PID_T int #define MDB_THR_T DWORD #include @@ -79,6 +85,14 @@ extern int cacheflush(char *addr, int nbytes, int cache); #define CACHEFLUSH(addr, bytes, cache) #endif +#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS) +/** fdatasync is broken on ext3/ext4fs on older kernels, see + * description in #mdb_env_open2 comments. You can safely + * define MDB_FDATASYNC_WORKS if this code will only be run + * on kernels 3.6 and newer. + */ +//#define BROKEN_FDATASYNC +#endif #include #include @@ -367,7 +381,6 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc); */ #ifndef MDB_FDATASYNC # define MDB_FDATASYNC fdatasync -# define HAVE_FDATASYNC 1 #endif #ifndef MDB_MSYNC @@ -1138,6 +1151,8 @@ struct MDB_env { #define MDB_ENV_ACTIVE 0x20000000U /** me_txkey is set */ #define MDB_ENV_TXKEY 0x10000000U + /** fdatasync is unreliable */ +#define MDB_FSYNCONLY 0x08000000U uint32_t me_flags; /**< @ref mdb_env */ unsigned int me_psize; /**< DB page size, inited from me_os_psize */ unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ @@ -1154,7 +1169,7 @@ struct MDB_env { MDB_txn *me_txn; /**< current write transaction */ MDB_txn *me_txn0; /**< prealloc'd write transaction */ size_t me_mapsize; /**< size of the data memory map */ - size_t me_size; /**< current file size */ + off_t me_size; /**< current file size */ pgno_t me_maxpg; /**< me_mapsize / me_psize */ MDB_dbx *me_dbxs; /**< array of static DB info */ uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ @@ -1203,7 +1218,7 @@ typedef struct MDB_ntxn { #endif /** max bytes to write in one call */ -#define MAX_WRITE (0x80000000U >> (sizeof(size_t) == 4)) +#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) /** Check \b txn and \b dbi arguments to a function */ #define TXN_DBI_EXIST(txn, dbi) \ @@ -2341,19 +2356,12 @@ fail: return rc; } -/* internal env_sync flags: */ -#define FORCE 1 /* as before, force a flush */ -#define FGREW 0x8000 /* file has grown, do a full fsync instead of just - fdatasync. We shouldn't have to do this, according to the POSIX spec. - But common Linux FSs violate the spec and won't sync required metadata - correctly when the file grows. This only makes a difference if the - platform actually distinguishes fdatasync from fsync. - http://www.openldap.org/lists/openldap-devel/201411/msg00000.html */ - -static int -mdb_env_sync0(MDB_env *env, int flag) +int +mdb_env_sync(MDB_env *env, int force) { - int rc = 0, force = flag & FORCE; + int rc = 0; + if (env->me_flags & MDB_RDONLY) + return EACCES; if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { if (env->me_flags & MDB_WRITEMAP) { int flags = ((env->me_flags & MDB_MAPASYNC) && !force) @@ -2365,9 +2373,9 @@ mdb_env_sync0(MDB_env *env, int flag) rc = ErrCode(); #endif } else { -#ifdef HAVE_FDATASYNC - if (flag & FGREW) { - if (fsync(env->me_fd)) /* Avoid ext-fs bugs, do full sync */ +#ifdef BROKEN_FDATASYNC + if (env->me_flags & MDB_FSYNCONLY) { + if (fsync(env->me_fd)) rc = ErrCode(); } else #endif @@ -2378,12 +2386,6 @@ mdb_env_sync0(MDB_env *env, int flag) return rc; } -int -mdb_env_sync(MDB_env *env, int force) -{ - return mdb_env_sync0(env, force != 0); -} - /** Back up parent txn's cursors, then grab the originals for tracking */ static int mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) @@ -2939,7 +2941,7 @@ mdb_freelist_save(MDB_txn *txn) int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; txnid_t pglast = 0, head_id = 0; pgno_t freecnt = 0, *free_pgs, *mop; - size_t head_room = 0, total_room = 0, mop_len, clean_limit; + ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; mdb_cursor_init(&mc, txn, FREE_DBI, NULL); @@ -2971,7 +2973,7 @@ mdb_freelist_save(MDB_txn *txn) /* Come back here after each Put() in case freelist changed */ MDB_val key, data; pgno_t *pgs; - size_t j; + ssize_t j; /* If using records from freeDB which we have not yet * deleted, delete them and any we reserved for me_pghead. @@ -3095,7 +3097,7 @@ mdb_freelist_save(MDB_txn *txn) rc = mdb_cursor_first(&mc, &key, &data); for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { txnid_t id = *(txnid_t *)key.mv_data; - size_t len = (size_t)(data.mv_size / sizeof(MDB_ID)) - 1; + ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; MDB_ID save; mdb_tassert(txn, len >= 0 && id <= env->me_pglast); @@ -3135,7 +3137,7 @@ mdb_page_flush(MDB_txn *txn, int keep) OVERLAPPED ov; #else struct iovec iov[MDB_COMMIT_PAGES]; - size_t wpos = 0, wsize = 0, wres; + ssize_t wpos = 0, wsize = 0, wres; size_t next_pos = 1; /* impossible pos, so pos != next_pos */ int n = 0; #endif @@ -3458,15 +3460,8 @@ mdb_txn_commit(MDB_txn *txn) mdb_audit(txn); #endif - i = 0; -#ifdef HAVE_FDATASYNC - if (txn->mt_next_pgno * env->me_psize > env->me_size) { - i |= FGREW; - env->me_size = txn->mt_next_pgno * env->me_psize; - } -#endif if ((rc = mdb_page_flush(txn, 0)) || - (rc = mdb_env_sync0(env, i)) || + (rc = mdb_env_sync(env, 0)) || (rc = mdb_env_write_meta(txn))) goto fail; @@ -3946,6 +3941,11 @@ mdb_fsize(HANDLE fd, size_t *size) return MDB_SUCCESS; } +#ifdef BROKEN_FDATASYNC +#include +#include +#endif + /** Further setup required for opening an LMDB environment */ static int ESECT @@ -3964,6 +3964,54 @@ mdb_env_open2(MDB_env *env) env->me_pidquery = PROCESS_QUERY_INFORMATION; #endif /* _WIN32 */ +#ifdef BROKEN_FDATASYNC + /* ext3/ext4 fdatasync is broken on some older Linux kernels. + * https://lkml.org/lkml/2012/9/3/83 + * Kernels after 3.6-rc6 are known good. + * https://lkml.org/lkml/2012/9/10/556 + * See if the DB is on ext3/ext4, then check for new enough kernel + * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known + * to be patched. + */ + { + struct statfs st; + fstatfs(env->me_fd, &st); + while (st.f_type == 0xEF53) { + struct utsname uts; + int i; + uname(&uts); + if (uts.release[0] < '3') { + if (!strncmp(uts.release, "2.6.32.", 7)) { + i = atoi(uts.release+7); + if (i >= 60) + break; /* 2.6.32.60 and newer is OK */ + } else if (!strncmp(uts.release, "2.6.34.", 7)) { + i = atoi(uts.release+7); + if (i >= 15) + break; /* 2.6.34.15 and newer is OK */ + } + } else if (uts.release[0] == '3') { + i = atoi(uts.release+2); + if (i > 5) + break; /* 3.6 and newer is OK */ + if (i == 5) { + i = atoi(uts.release+4); + if (i >= 4) + break; /* 3.5.4 and newer is OK */ + } else if (i == 2) { + i = atoi(uts.release+4); + if (i >= 30) + break; /* 3.2.30 and newer is OK */ + } + } else { /* 4.x and newer is OK */ + break; + } + env->me_flags |= MDB_FSYNCONLY; + break; + } + } +#endif + if ((i = mdb_env_read_header(env, &meta)) != 0) { if (i != ENOENT) return i; @@ -4007,10 +4055,6 @@ mdb_env_open2(MDB_env *env) newenv = 0; } - rc = mdb_fsize(env->me_fd, &env->me_size); - if (rc) - return rc; - rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); if (rc) return rc; @@ -4844,11 +4888,11 @@ static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) { int diff; - size_t len_diff; + ssize_t len_diff; unsigned int len; len = a->mv_size; - len_diff = (size_t) a->mv_size - (size_t) b->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; if (len_diff > 0) { len = b->mv_size; len_diff = 1; @@ -4863,14 +4907,14 @@ static int mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) { const unsigned char *p1, *p2, *p1_lim; - size_t len_diff; + ssize_t len_diff; int diff; p1_lim = (const unsigned char *)a->mv_data; p1 = (const unsigned char *)a->mv_data + a->mv_size; p2 = (const unsigned char *)b->mv_data + b->mv_size; - len_diff = (size_t) a->mv_size - (size_t) b->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; if (len_diff > 0) { p1_lim += len_diff; len_diff = 1; @@ -6803,7 +6847,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, { unsigned int i; size_t node_size = NODESIZE; - size_t room; + ssize_t room; indx_t ofs; MDB_node *node; MDB_page *mp = mc->mc_pg[mc->mc_top]; @@ -6834,7 +6878,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, return MDB_SUCCESS; } - room = (size_t)SIZELEFT(mp) - (size_t)sizeof(indx_t); + room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); if (key != NULL) node_size += key->mv_size; if (IS_LEAF(mp)) { @@ -6849,7 +6893,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", data->mv_size, node_size+data->mv_size)); node_size = EVEN(node_size + sizeof(pgno_t)); - if ((size_t)node_size > room) + if ((ssize_t)node_size > room) goto full; if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) return rc; @@ -6861,7 +6905,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, } } node_size = EVEN(node_size); - if ((size_t)node_size > room) + if ((ssize_t)node_size > room) goto full; update: @@ -8768,7 +8812,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) DWORD len, w2; #define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) #else - size_t len; + ssize_t len; size_t w2; #define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) #endif -- cgit v1.2.3