Netlib_DownloadFile - chunk gatherer which writes down data directly to a file, without storing anything in memory

6 files changed, 100 insertions, 31 deletions

diff --git a/src/mir_app/src/mir_app.def b/src/mir_app/src/mir_app.def
index 81b4f9d53a..5c41efdb66 100644
--- a/src/mir_app/src/mir_app.def
+++ b/src/mir_app/src/mir_app.def
@@ -961,3 +961,4 @@ _Netlib_HttpCookies@4 @1095 NONAME
 _Netlib_HttpResult@4 @1096 NONAME

 ?SetData@MHttpRequest@@QAEXPBXI@Z @1097 NONAME

 ?DeleteHeader@MHttpHeaders@@QAEXPBD@Z @1098 NONAME

+_Netlib_DownloadFile@20 @1099 NONAME

diff --git a/src/mir_app/src/mir_app64.def b/src/mir_app/src/mir_app64.def
index 86f013224d..a5f85d044b 100644
--- a/src/mir_app/src/mir_app64.def
+++ b/src/mir_app/src/mir_app64.def
@@ -961,3 +961,4 @@ Netlib_HttpCookies @1089 NONAME
 Netlib_HttpResult @1090 NONAME

 ?SetData@MHttpRequest@@QEAAXPEBX_K@Z @1091 NONAME

 ?DeleteHeader@MHttpHeaders@@QEAAXPEBD@Z @1092 NONAME

+Netlib_DownloadFile @1093 NONAME

diff --git a/src/mir_app/src/netlib.h b/src/mir_app/src/netlib.h
index 1eed2ced9d..0a764c5f0f 100644
--- a/src/mir_app/src/netlib.h
+++ b/src/mir_app/src/netlib.h
@@ -75,25 +75,16 @@ struct NetlibUrl
 

 struct MChunkHandler

 {

-	virtual void updateChunk(const void *pData, size_t cbLen) = 0;

 	virtual void apply(MHttpResponse *nlhr) = 0;

+	virtual bool updateChunk(const void *pData, size_t cbLen) = 0;

 };

 

 class MMemoryChunkStorage : public MChunkHandler

 {

 	MBinBuffer buf;

 

-	void updateChunk(const void *pData, size_t cbLen) override

-	{

-		buf.append(pData, cbLen);

-	}

-	

-	void apply(MHttpResponse *nlhr) override

-	{

-		unsigned dataLen = (unsigned)buf.length();

-		nlhr->body.Truncate(dataLen);

-		memcpy(nlhr->body.GetBuffer(), buf.data(), dataLen);

-	}

+	void apply(MHttpResponse *nlhr) override;

+	bool updateChunk(const void *pData, size_t cbLen) override;

 

 public:

 	MMemoryChunkStorage() {}

@@ -101,15 +92,18 @@ public:
 

 class MFileChunkStorage : public MChunkHandler

 {

-	CMStringW filePath;

+	int fileId, prevBlocks = 0;

+

+	pfnDownloadCallback pCallback;

+	void *pCallbackInfo;

 

-	void updateChunk(const void *pData, size_t cbLen) override;

 	void apply(MHttpResponse *nlhr) override;

+	bool updateChunk(const void *pData, size_t cbLen) override;

 

 public:

-	MFileChunkStorage(const wchar_t *pwszFileName) :

-		filePath(pwszFileName)

-	{}

+	MFileChunkStorage(const MFilePath &pwszFileName, pfnDownloadCallback, void*);

+

+	__forceinline operator bool() const { return fileId != -1; }

 };

 

 /////////////////////////////////////////////////////////////////////////////////////////

diff --git a/src/mir_app/src/netlib_http.cpp b/src/mir_app/src/netlib_http.cpp
index 37b9023dad..69946a0c8a 100644
--- a/src/mir_app/src/netlib_http.cpp
+++ b/src/mir_app/src/netlib_http.cpp
@@ -82,7 +82,7 @@ ProxyAuthList proxyAuthList;
 /////////////////////////////////////////////////////////////////////////////////////////

 // Module exports

 

-MIR_APP_DLL(bool) Netlib_FreeHttpRequest(MHttpResponse *nlhr)

+EXTERN_C MIR_APP_DLL(bool) Netlib_FreeHttpRequest(MHttpResponse *nlhr)

 {

 	if (nlhr == nullptr) {

 		SetLastError(ERROR_INVALID_PARAMETER);

@@ -958,15 +958,18 @@ next:
 				if (recvResult == SOCKET_ERROR)

 					return nullptr;

 

-

 				if (recvResult <= dataLen) {

-					pHandler.updateChunk(tmpBuf, recvResult);

+					if (!pHandler.updateChunk(tmpBuf, recvResult))

+						return nullptr;

+

 					dataLen -= recvResult;

 					if (!dataLen)

 						break;

 				}

 				else {

-					pHandler.updateChunk(tmpBuf, dataLen);

+					if (!pHandler.updateChunk(tmpBuf, dataLen))

+						return nullptr;

+

 					nlc->foreBuf.appendBefore(tmpBuf.get() + dataLen, recvResult - dataLen);

 					break;

 				}

@@ -1029,7 +1032,7 @@ next:
 /////////////////////////////////////////////////////////////////////////////////////////

 // Module entry point

 

-MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpRequest *nlhr)

+static MHttpResponse* HttpTransactionWorker(HNETLIBUSER nlu, MHttpRequest *nlhr, MChunkHandler &pHandler)

 {

 	if (GetNetlibHandleType(nlu) != NLH_USER || !(nlu->user.flags & NUF_OUTGOING) || !nlhr || nlhr->m_szUrl.IsEmpty()) {

 		SetLastError(ERROR_INVALID_PARAMETER);

@@ -1070,9 +1073,8 @@ MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpReques
 

 	if (!nlhr->FindHeader("Accept-Encoding"))

 		nlhr->AddHeader("Accept-Encoding", "deflate, gzip");

-	

-	MMemoryChunkStorage storage;

-	if (Netlib_SendHttpRequest(nlc, nlhr, storage) == SOCKET_ERROR) {

+

+	if (Netlib_SendHttpRequest(nlc, nlhr, pHandler) == SOCKET_ERROR) {

 		Netlib_CloseHandle(nlc);

 		return nullptr;

 	}

@@ -1089,7 +1091,7 @@ MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpReques
 	if (nlhr->requestType == REQUEST_HEAD)

 		nlhrReply = Netlib_RecvHttpHeaders(nlc, 0);

 	else

-		nlhrReply = NetlibHttpRecv(nlc, hflags, dflags, storage);

+		nlhrReply = NetlibHttpRecv(nlc, hflags, dflags, pHandler);

 

 	if (nlhrReply) {

 		nlhrReply->szUrl = nlc->szNewUrl;

@@ -1105,3 +1107,75 @@ MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpReques
 

 	return nlhrReply;

 }

+

+/////////////////////////////////////////////////////////////////////////////////////////

+

+void MMemoryChunkStorage::apply(MHttpResponse *nlhr)

+{

+	unsigned dataLen = (unsigned)buf.length();

+	nlhr->body.Truncate(dataLen);

+	memcpy(nlhr->body.GetBuffer(), buf.data(), dataLen);

+}

+

+bool MMemoryChunkStorage::updateChunk(const void *pData, size_t cbLen)

+{

+	buf.append(pData, cbLen);

+	return true;

+}

+

+MIR_APP_DLL(MHttpResponse *) Netlib_HttpTransaction(HNETLIBUSER nlu, MHttpRequest *nlhr)

+{

+	MMemoryChunkStorage storage;

+	return HttpTransactionWorker(nlu, nlhr, storage);

+}

+

+/////////////////////////////////////////////////////////////////////////////////////////

+

+MFileChunkStorage::MFileChunkStorage(const MFilePath &_1, pfnDownloadCallback _2, void *_3) :

+	pCallback(_2),

+	pCallbackInfo(_3)

+{

+	fileId = _wopen(_1, _O_WRONLY | _O_TRUNC | _O_BINARY | _O_CREAT, _S_IREAD | _S_IWRITE);

+}

+

+void MFileChunkStorage::apply(MHttpResponse *nlhr)

+{

+	if (fileId != -1) {

+		nlhr->resultCode = 200;

+		nlhr->body = "OK";

+		_close(fileId);

+	}

+	else nlhr->resultCode = 500;

+}

+

+bool MFileChunkStorage::updateChunk(const void *pData, size_t cbLen)

+{

+	if (cbLen != _write(fileId, pData, unsigned(cbLen))) {

+		_close(fileId);

+		fileId = -1;

+		return false;

+	}

+

+	if (pCallback) {

+		int nBlocks = _filelength(fileId) / 65536;

+		if (nBlocks != prevBlocks) {

+			prevBlocks = nBlocks;

+			pCallback(pCallbackInfo);

+		}

+	}

+	return true;

+}

+

+MIR_APP_DLL(MHttpResponse *) Netlib_DownloadFile(

+	HNETLIBUSER nlu,

+	MHttpRequest *nlhr,

+	const MFilePath &wszFileName,

+	pfnDownloadCallback pCallback,

+	void *pCallbackInfo)

+{

+	MFileChunkStorage storage(wszFileName, pCallback, pCallbackInfo);

+	if (!storage)

+		return nullptr;

+

+	return HttpTransactionWorker(nlu, nlhr, storage);

+}

diff --git a/src/mir_app/src/netlib_openconn.cpp b/src/mir_app/src/netlib_openconn.cpp
index 58443d090e..31878d4c24 100644
--- a/src/mir_app/src/netlib_openconn.cpp
+++ b/src/mir_app/src/netlib_openconn.cpp
@@ -316,13 +316,12 @@ static bool NetlibInitHttpsConnection(NetlibConnection *nlc)
 	if (Netlib_SendHttpRequest(nlc, &nlhrSend, storage) == SOCKET_ERROR)

 		return false;

 

-	auto *nlhrReply = NetlibHttpRecv(nlc, MSG_DUMPPROXY | MSG_RAW, MSG_DUMPPROXY | MSG_RAW, storage, true);

+	NLHR_PTR nlhrReply(NetlibHttpRecv(nlc, MSG_DUMPPROXY | MSG_RAW, MSG_DUMPPROXY | MSG_RAW, storage, true));

 	if (nlhrReply == nullptr)

 		return false;

 

 	if (nlhrReply->resultCode < 200 || nlhrReply->resultCode >= 300) {

 		if (nlhrReply->resultCode == 403 && nlc->dnsThroughProxy) {

-			Netlib_FreeHttpRequest(nlhrReply);

 			nlc->dnsThroughProxy = 0;

 			return NetlibInitHttpsConnection(nlc);

 		}

@@ -330,10 +329,9 @@ static bool NetlibInitHttpsConnection(NetlibConnection *nlc)
 		NetlibHttpSetLastErrorUsingHttpResult(nlhrReply->resultCode);

 		Netlib_Logf(nlc->nlu, "%s %d: %s request failed (%u %s)", __FILE__, __LINE__, 

 			nlc->nlu->settings.proxyType == PROXYTYPE_HTTP ? "HTTP" : "HTTPS", nlhrReply->resultCode, nlhrReply->szResultDescr);

-		Netlib_FreeHttpRequest(nlhrReply);

-		return 0;

+		return false;

 	}

-	Netlib_FreeHttpRequest(nlhrReply);

+

 	return true; // connected

 }

 

diff --git a/src/mir_app/src/stdafx.h b/src/mir_app/src/stdafx.h
index 131c61dc7a..9c91bb225b 100644
--- a/src/mir_app/src/stdafx.h
+++ b/src/mir_app/src/stdafx.h
@@ -39,6 +39,7 @@ typedef struct SslHandle *HSSL;
 	#include <Richedit.h>

 	

 	#include <io.h>

+	#include <fcntl.h>

 	#include <direct.h>

 	#include <process.h>

 #else