From fbf164f781503ee2e538156bc701a957e3dea713 Mon Sep 17 00:00:00 2001 From: George Hazan Date: Tue, 9 Jan 2024 13:52:26 +0300 Subject: Netlib_DownloadFile - chunk gatherer which writes down data directly to a file, without storing anything in memory --- src/mir_app/src/mir_app.def | 1 + src/mir_app/src/mir_app64.def | 1 + src/mir_app/src/netlib.h | 28 +++++------ src/mir_app/src/netlib_http.cpp | 92 +++++++++++++++++++++++++++++++++---- src/mir_app/src/netlib_openconn.cpp | 8 ++-- src/mir_app/src/stdafx.h | 1 + 6 files changed, 100 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/mir_app/src/mir_app.def b/src/mir_app/src/mir_app.def index 81b4f9d53a..5c41efdb66 100644 --- a/src/mir_app/src/mir_app.def +++ b/src/mir_app/src/mir_app.def @@ -961,3 +961,4 @@ _Netlib_HttpCookies@4 @1095 NONAME _Netlib_HttpResult@4 @1096 NONAME ?SetData@MHttpRequest@@QAEXPBXI@Z @1097 NONAME ?DeleteHeader@MHttpHeaders@@QAEXPBD@Z @1098 NONAME +_Netlib_DownloadFile@20 @1099 NONAME diff --git a/src/mir_app/src/mir_app64.def b/src/mir_app/src/mir_app64.def index 86f013224d..a5f85d044b 100644 --- a/src/mir_app/src/mir_app64.def +++ b/src/mir_app/src/mir_app64.def @@ -961,3 +961,4 @@ Netlib_HttpCookies @1089 NONAME Netlib_HttpResult @1090 NONAME ?SetData@MHttpRequest@@QEAAXPEBX_K@Z @1091 NONAME ?DeleteHeader@MHttpHeaders@@QEAAXPEBD@Z @1092 NONAME +Netlib_DownloadFile @1093 NONAME diff --git a/src/mir_app/src/netlib.h b/src/mir_app/src/netlib.h index 1eed2ced9d..0a764c5f0f 100644 --- a/src/mir_app/src/netlib.h +++ b/src/mir_app/src/netlib.h @@ -75,25 +75,16 @@ struct NetlibUrl struct MChunkHandler { - virtual void updateChunk(const void *pData, size_t cbLen) = 0; virtual void apply(MHttpResponse *nlhr) = 0; + virtual bool updateChunk(const void *pData, size_t cbLen) = 0; }; class MMemoryChunkStorage : public MChunkHandler { MBinBuffer buf; - void updateChunk(const void *pData, size_t cbLen) override - { - buf.append(pData, cbLen); - } - - void apply(MHttpResponse *nlhr) override - { - unsigned dataLen = (unsigned)buf.length(); - nlhr->body.Truncate(dataLen); - memcpy(nlhr->body.GetBuffer(), buf.data(), dataLen); - } + void apply(MHttpResponse *nlhr) override; + bool updateChunk(const void *pData, size_t cbLen) override; public: MMemoryChunkStorage() {} @@ -101,15 +92,18 @@ public: class MFileChunkStorage : public MChunkHandler { - CMStringW filePath; + int fileId, prevBlocks = 0; + + pfnDownloadCallback pCallback; + void *pCallbackInfo; - void updateChunk(const void *pData, size_t cbLen) override; void apply(MHttpResponse *nlhr) override; + bool updateChunk(const void *pData, size_t cbLen) override; public: - MFileChunkStorage(const wchar_t *pwszFileName) : - filePath(pwszFileName) - {} + MFileChunkStorage(const MFilePath &pwszFileName, pfnDownloadCallback, void*); + + __forceinline operator bool() const { return fileId != -1; } }; ///////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/mir_app/src/netlib_http.cpp b/src/mir_app/src/netlib_http.cpp index 37b9023dad..69946a0c8a 100644 --- a/src/mir_app/src/netlib_http.cpp +++ b/src/mir_app/src/netlib_http.cpp @@ -82,7 +82,7 @@ ProxyAuthList proxyAuthList; ///////////////////////////////////////////////////////////////////////////////////////// // Module exports -MIR_APP_DLL(bool) Netlib_FreeHttpRequest(MHttpResponse *nlhr) +EXTERN_C MIR_APP_DLL(bool) Netlib_FreeHttpRequest(MHttpResponse *nlhr) { if (nlhr == nullptr) { SetLastError(ERROR_INVALID_PARAMETER); @@ -958,15 +958,18 @@ next: if (recvResult == SOCKET_ERROR) return nullptr; - if (recvResult <= dataLen) { - pHandler.updateChunk(tmpBuf, recvResult); + if (!pHandler.updateChunk(tmpBuf, recvResult)) + return nullptr; + dataLen -= recvResult; if (!dataLen) break; } else { - pHandler.updateChunk(tmpBuf, dataLen); + if (!pHandler.updateChunk(tmpBuf, dataLen)) + return nullptr; + nlc->foreBuf.appendBefore(tmpBuf.get() + dataLen, recvResult - dataLen); break; } @@ -1029,7 +1032,7 @@ next: ///////////////////////////////////////////////////////////////////////////////////////// // Module entry point -MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpRequest *nlhr) +static MHttpResponse* HttpTransactionWorker(HNETLIBUSER nlu, MHttpRequest *nlhr, MChunkHandler &pHandler) { if (GetNetlibHandleType(nlu) != NLH_USER || !(nlu->user.flags & NUF_OUTGOING) || !nlhr || nlhr->m_szUrl.IsEmpty()) { SetLastError(ERROR_INVALID_PARAMETER); @@ -1070,9 +1073,8 @@ MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpReques if (!nlhr->FindHeader("Accept-Encoding")) nlhr->AddHeader("Accept-Encoding", "deflate, gzip"); - - MMemoryChunkStorage storage; - if (Netlib_SendHttpRequest(nlc, nlhr, storage) == SOCKET_ERROR) { + + if (Netlib_SendHttpRequest(nlc, nlhr, pHandler) == SOCKET_ERROR) { Netlib_CloseHandle(nlc); return nullptr; } @@ -1089,7 +1091,7 @@ MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpReques if (nlhr->requestType == REQUEST_HEAD) nlhrReply = Netlib_RecvHttpHeaders(nlc, 0); else - nlhrReply = NetlibHttpRecv(nlc, hflags, dflags, storage); + nlhrReply = NetlibHttpRecv(nlc, hflags, dflags, pHandler); if (nlhrReply) { nlhrReply->szUrl = nlc->szNewUrl; @@ -1105,3 +1107,75 @@ MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpReques return nlhrReply; } + +///////////////////////////////////////////////////////////////////////////////////////// + +void MMemoryChunkStorage::apply(MHttpResponse *nlhr) +{ + unsigned dataLen = (unsigned)buf.length(); + nlhr->body.Truncate(dataLen); + memcpy(nlhr->body.GetBuffer(), buf.data(), dataLen); +} + +bool MMemoryChunkStorage::updateChunk(const void *pData, size_t cbLen) +{ + buf.append(pData, cbLen); + return true; +} + +MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpRequest *nlhr) +{ + MMemoryChunkStorage storage; + return HttpTransactionWorker(nlu, nlhr, storage); +} + +///////////////////////////////////////////////////////////////////////////////////////// + +MFileChunkStorage::MFileChunkStorage(const MFilePath &_1, pfnDownloadCallback _2, void *_3) : + pCallback(_2), + pCallbackInfo(_3) +{ + fileId = _wopen(_1, _O_WRONLY | _O_TRUNC | _O_BINARY | _O_CREAT, _S_IREAD | _S_IWRITE); +} + +void MFileChunkStorage::apply(MHttpResponse *nlhr) +{ + if (fileId != -1) { + nlhr->resultCode = 200; + nlhr->body = "OK"; + _close(fileId); + } + else nlhr->resultCode = 500; +} + +bool MFileChunkStorage::updateChunk(const void *pData, size_t cbLen) +{ + if (cbLen != _write(fileId, pData, unsigned(cbLen))) { + _close(fileId); + fileId = -1; + return false; + } + + if (pCallback) { + int nBlocks = _filelength(fileId) / 65536; + if (nBlocks != prevBlocks) { + prevBlocks = nBlocks; + pCallback(pCallbackInfo); + } + } + return true; +} + +MIR_APP_DLL(MHttpResponse *) Netlib_DownloadFile( + HNETLIBUSER nlu, + MHttpRequest *nlhr, + const MFilePath &wszFileName, + pfnDownloadCallback pCallback, + void *pCallbackInfo) +{ + MFileChunkStorage storage(wszFileName, pCallback, pCallbackInfo); + if (!storage) + return nullptr; + + return HttpTransactionWorker(nlu, nlhr, storage); +} diff --git a/src/mir_app/src/netlib_openconn.cpp b/src/mir_app/src/netlib_openconn.cpp index 58443d090e..31878d4c24 100644 --- a/src/mir_app/src/netlib_openconn.cpp +++ b/src/mir_app/src/netlib_openconn.cpp @@ -316,13 +316,12 @@ static bool NetlibInitHttpsConnection(NetlibConnection *nlc) if (Netlib_SendHttpRequest(nlc, &nlhrSend, storage) == SOCKET_ERROR) return false; - auto *nlhrReply = NetlibHttpRecv(nlc, MSG_DUMPPROXY | MSG_RAW, MSG_DUMPPROXY | MSG_RAW, storage, true); + NLHR_PTR nlhrReply(NetlibHttpRecv(nlc, MSG_DUMPPROXY | MSG_RAW, MSG_DUMPPROXY | MSG_RAW, storage, true)); if (nlhrReply == nullptr) return false; if (nlhrReply->resultCode < 200 || nlhrReply->resultCode >= 300) { if (nlhrReply->resultCode == 403 && nlc->dnsThroughProxy) { - Netlib_FreeHttpRequest(nlhrReply); nlc->dnsThroughProxy = 0; return NetlibInitHttpsConnection(nlc); } @@ -330,10 +329,9 @@ static bool NetlibInitHttpsConnection(NetlibConnection *nlc) NetlibHttpSetLastErrorUsingHttpResult(nlhrReply->resultCode); Netlib_Logf(nlc->nlu, "%s %d: %s request failed (%u %s)", __FILE__, __LINE__, nlc->nlu->settings.proxyType == PROXYTYPE_HTTP ? "HTTP" : "HTTPS", nlhrReply->resultCode, nlhrReply->szResultDescr); - Netlib_FreeHttpRequest(nlhrReply); - return 0; + return false; } - Netlib_FreeHttpRequest(nlhrReply); + return true; // connected } diff --git a/src/mir_app/src/stdafx.h b/src/mir_app/src/stdafx.h index 131c61dc7a..9c91bb225b 100644 --- a/src/mir_app/src/stdafx.h +++ b/src/mir_app/src/stdafx.h @@ -39,6 +39,7 @@ typedef struct SslHandle *HSSL; #include #include + #include #include #include #else -- cgit v1.2.3