Facebook: initial changes to support loading of unread chat messages (#1095)

* Facebook: initial changes to support loading of unread chat messages

* Facebook: load unread messages in one request + load last messages when chat opens (if is set)

* shrinked regex pattern, commented unused code

* Facebook: fixed loading of whole history

5 files changed, 163 insertions, 122 deletions

diff --git a/protocols/FacebookRM/src/client.h b/protocols/FacebookRM/src/client.h
index bcc161d3fc..a71fab6aff 100644
--- a/protocols/FacebookRM/src/client.h
+++ b/protocols/FacebookRM/src/client.h
@@ -223,7 +223,7 @@ public:
 	HttpRequest* memoriesRequest();

 

 	// history.cpp

-	HttpRequest* threadInfoRequest(bool isChat, const char *id, const char* timestamp = nullptr, int limit = -1);

+	HttpRequest* threadInfoRequest(bool isChat, const char *id, const char* timestamp = nullptr, int limit = -1, bool loadMessages = false);

 	HttpRequest* threadInfoRequest(const LIST<char> &ids, int offset, int limit);

 	HttpRequest* unreadThreadsRequest();

 

diff --git a/protocols/FacebookRM/src/history.cpp b/protocols/FacebookRM/src/history.cpp
index eb1e480766..82d37ddae1 100644
--- a/protocols/FacebookRM/src/history.cpp
+++ b/protocols/FacebookRM/src/history.cpp
@@ -22,7 +22,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #include "stdafx.h"
 
-HttpRequest* facebook_client::threadInfoRequest(bool isChat, const char *id, const char* timestamp, int limit)
+HttpRequest* facebook_client::threadInfoRequest(bool isChat, const char *id, const char* timestamp, int limit, bool loadMessages)
 {
 	HttpRequest *p = new HttpRequest(REQUEST_POST, FACEBOOK_SERVER_REGULAR "/api/graphqlbatch/");
 
@@ -47,13 +47,13 @@ HttpRequest* facebook_client::threadInfoRequest(bool isChat, const char *id, con
 	query_params
 		<< CHAR_PARAM("id", id_.c_str())
 		<< INT_PARAM("message_limit", (limit == -1) ? 50 : limit)
-		<< INT_PARAM("load_messages", 0)
+		<< BOOL_PARAM("load_messages", loadMessages)
 		<< BOOL_PARAM("load_read_receipts", false);
 
-	if (timestamp != nullptr)
-		query_params << CHAR_PARAM("before", timestamp);
-	else
+	if (timestamp == nullptr || strcmp(timestamp, "") == 0)
 		query_params << NULL_PARAM("before");
+	else
+		query_params << CHAR_PARAM("before", timestamp);
 
 	o0 << CHAR_PARAM("doc_id", "1549485615075443") << JSON_PARAM("query_params", query_params);
 	root << JSON_PARAM("o0", o0);
@@ -79,20 +79,28 @@ HttpRequest* facebook_client::threadInfoRequest(const LIST<char> &ids, int offse
 		<< CHAR_PARAM("__rev", __rev())
 		<< CHAR_PARAM("fb_dtsg", dtsg_.c_str());
 
+	JSONNode root;
 	for (int i = 0; i < ids.getCount(); i++) {
 		// NOTE: Remove "id." prefix as here we need to give threadFbId and not threadId
 		std::string id_ = ids[i]; // FIXME: Rewrite this without std::string...
 		if (id_.substr(0, 3) == "id.")
 			id_ = id_.substr(3);
 
-		// Load messages
-		CMStringA begin(::FORMAT, "messages[%s][%s]", "thread_fbids", ptrA(mir_urlEncode(id_.c_str())).get());
-		p->Body
-			<< INT_PARAM(CMStringA(::FORMAT, "%s[offset]", begin.c_str()), offset)
-			<< INT_PARAM(CMStringA(::FORMAT, "%s[limit]", begin.c_str()), limit)
-			<< CHAR_PARAM(CMStringA(::FORMAT, "threads[%s][%i]", "thread_fbids", i), id_.c_str());
+		// Build query
+		JSONNode oX, query_params;
+		query_params
+			<< CHAR_PARAM("id", id_.c_str())
+			<< INT_PARAM("message_limit", limit)
+			<< BOOL_PARAM("load_messages", true)
+			<< BOOL_PARAM("load_read_receipts", false)
+			<< NULL_PARAM("before");
+
+		oX << CHAR_PARAM("doc_id", "1508526735892416") << JSON_PARAM("query_params", query_params);
+		root << JSON_PARAM(("o" + std::to_string(i)).c_str(), oX);
 	}
 
+	p->Body << CHAR_PARAM("queries", root.write().c_str());
+
 	return p;
 }
 
diff --git a/protocols/FacebookRM/src/json.cpp b/protocols/FacebookRM/src/json.cpp
index af232b9bd6..1ffd30b203 100644
--- a/protocols/FacebookRM/src/json.cpp
+++ b/protocols/FacebookRM/src/json.cpp
@@ -21,6 +21,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "stdafx.h"
+#include <regex>
 
 LRESULT CALLBACK PopupDlgProc(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam);
 
@@ -1165,127 +1166,159 @@ int FacebookProto::ParseUnreadThreads(std::string *data, std::vector< std::strin
 
 int FacebookProto::ParseThreadMessages(std::string *data, std::vector< facebook_message >* messages, bool unreadOnly)
 {
-	std::string jsonData = data->substr(9);
+	/*size_t len = data->find("\r\n");
+	if (len != data->npos)
+		data->erase(len);*/
+
+	// since it could loop over multiple queries not all can be valid
+	// so return EXIT_FAILURE only if none is processed
+	bool hasResult = false;
+
+	// pattern for one query
+	std::regex r("\\{\"o\\d\":\\{\"data\":\\{\"message_thread\":\\{.+\\}{4,5}"); // (\\{|$)
+	std::sregex_iterator i = std::sregex_iterator(data->begin(), data->end(), r);
+	std::sregex_iterator end;
+	// loop over queries
+	for (; i != end; ++i) {
+
+		std::smatch m = *i;
+		std::string match = m.str();
+
+		JSONNode root = JSONNode::parse(match.c_str());
+		if (!root)
+			//return EXIT_FAILURE;
+			continue;
 
-	JSONNode root = JSONNode::parse(jsonData.c_str());
-	if (!root)
-		return EXIT_FAILURE;
+		// query number "o0", "o1", .. but they are not ordered
+		std::string oX = std::string("o") + std::string(1, match.at(3));
 
-	const JSONNode &payload = root["payload"];
-	if (!payload)
-		return EXIT_FAILURE;
+		const JSONNode &thread = root[oX.c_str()]["data"]["message_thread"];
+		if (!thread)
+			//return EXIT_FAILURE;
+			continue;
 
-	const JSONNode &actions = payload["actions"];
-	const JSONNode &threads = payload["threads"];
-	if (!actions || !threads)
-		return EXIT_FAILURE;
+		const JSONNode &nodes = thread["messages"]["nodes"];
+		if (!nodes)
+			//return EXIT_FAILURE;
+			continue;
 
-	for (auto &it : actions) {
-		const JSONNode &author_ = it["author"];
-		const JSONNode &other_user_fbid_ = it["other_user_fbid"];
-		const JSONNode &body_ = it["body"];
-		const JSONNode &thread_id_ = it["thread_id"];
-		const JSONNode &thread_fbid_ = it["thread_fbid"];
-		const JSONNode &mid_ = it["message_id"];
-		const JSONNode &timestamp_ = it["timestamp"];
-		const JSONNode &filtered_ = it["is_filtered_content"];
-		const JSONNode &is_unread_ = it["is_unread"];
+		// TODO! process commented sections and better pair json (this is just quick attempt, + I do not know what everything means yet)
 
-		// Either there is "body" (for classic messages), or "log_message_type" and "log_message_body" (for log messages)
-		const JSONNode &log_type_ = it["log_message_type"];
-		const JSONNode &log_body_ = it["log_message_body"];
-		const JSONNode &log_data_ = it["log_message_data"]; // additional data for this log message
+		const JSONNode &other_user_fbid_ = thread["thread_key"]["other_user_id"];
+		const JSONNode &thread_fbid_ = thread["thread_key"]["thread_fbid"];
 
-		if (!author_ || (!body_ && !log_body_) || !mid_ || (!thread_fbid_ && !thread_id_) || !timestamp_) {
-			debugLogA("ParseThreadMessages: ignoring message (%s) - missing attribute", mid_.as_string().c_str());
-			continue;
-		}
+		for (auto it = nodes.begin(); it != nodes.end(); ++it) {
+			const JSONNode &author_ = (*it)["message_sender"]["id"];
+			const JSONNode &body_ = (*it)["message"]["text"];
+			const JSONNode &thread_id_ = (*it)["offline_threading_id"];
+			const JSONNode &mid_ = (*it)["message_id"];
+			const JSONNode &timestamp_ = (*it)["timestamp_precise"];
+			// const JSONNode &filtered_ = (*it)["is_filtered_content"];
+			const JSONNode &is_unread_ = (*it)["unread"];
 
-		std::string thread_id = thread_id_.as_string();
-		std::string thread_fbid = thread_fbid_.as_string();
-		std::string message_id = mid_.as_string();
-		std::string message_text = body_ ? body_.as_string() : log_body_.as_string();
-		std::string author_id = author_.as_string();
-		std::string other_user_fbid = other_user_fbid_ ? other_user_fbid_.as_string() : "";
-		std::string::size_type pos = author_id.find(":"); // strip "fbid:" prefix
-		if (pos != std::string::npos)
-			author_id = author_id.substr(pos + 1);
+			// Either there is "body" (for classic messages), or "log_message_type" and "log_message_body" (for log messages)
+			const JSONNode &log_type_ = (*it)["log_message_type"];
+			const JSONNode &log_body_ = (*it)["log_message_body"];
+			const JSONNode &log_data_ = (*it)["log_message_data"]; // additional data for this log message
 
-		// Process attachements and stickers
-		ParseAttachments(message_text, it, other_user_fbid, true);
+			if (!author_ || (!body_ && !log_body_) || !mid_ || (!thread_fbid_ && !thread_id_) || !timestamp_) {
+				debugLogA("ParseThreadMessages: ignoring message (%s) - missing attribute", mid_.as_string().c_str());
+				continue;
+			}
 
-		if (filtered_.as_bool() && message_text.empty())
-			message_text = Translate("This message is no longer available, because it was marked as abusive or spam.");
+			std::string thread_id = thread_id_.as_string();
+			std::string thread_fbid = thread_fbid_.as_string();
+			std::string message_id = mid_.as_string();
+			std::string message_text = body_ ? body_.as_string() : log_body_.as_string();
+			std::string author_id = author_.as_string();
+			std::string other_user_fbid = other_user_fbid_ ? other_user_fbid_.as_string() : "";
+			std::string::size_type pos = author_id.find(":"); // strip "fbid:" prefix
+			if (pos != std::string::npos)
+				author_id = author_id.substr(pos + 1);
+
+			// Process attachements and stickers
+			ParseAttachments(message_text, (*it), other_user_fbid, true);
+
+			//if (filtered_.as_bool() && message_text.empty())
+			//	message_text = Translate("This message is no longer available, because it was marked as abusive or spam.");
+
+			message_text = utils::text::trim(utils::text::slashu_to_utf8(message_text), true);
+			if (message_text.empty()) {
+				debugLogA("ParseThreadMessages: ignoring message (%s) - empty message text", mid_.as_string().c_str());
+				continue;
+			}
 
-		message_text = utils::text::trim(utils::text::slashu_to_utf8(message_text), true);
-		if (message_text.empty()) {
-			debugLogA("ParseThreadMessages: ignoring message (%s) - empty message text", mid_.as_string().c_str());
-			continue;
-		}
+			bool isUnread = is_unread_.as_bool();
 
-		bool isUnread = is_unread_.as_bool();
+			// Ignore read messages if we want only unread messages
+			if (unreadOnly && !isUnread)
+				continue;
 
-		// Ignore read messages if we want only unread messages
-		if (unreadOnly && !isUnread)
-			continue;
+			facebook_message message;
+			message.message_text = message_text;
+			message.time = utils::time::from_string(timestamp_.as_string());
+			message.message_id = message_id;
+			message.isIncoming = (author_id != facy.self_.user_id);
+			message.isUnread = isUnread;
+
+			message.isChat = other_user_fbid.empty();
+			if (message.isChat) {
+				message.user_id = author_id;
+				message.thread_id = "id." + thread_fbid;
+			}
+			else {
+				message.user_id = other_user_fbid;
+				message.thread_id = thread_id;
+			}
 
-		facebook_message message;
-		message.message_text = message_text;
-		message.time = utils::time::from_string(timestamp_.as_string());
-		message.message_id = message_id;
-		message.isIncoming = (author_id != facy.self_.user_id);
-		message.isUnread = isUnread;
+			ParseMessageType(message, log_type_, log_body_, log_data_);
 
-		message.isChat = other_user_fbid.empty();
-		if (message.isChat) {
-			message.user_id = author_id;
-			message.thread_id = "id." + thread_fbid;
+			messages->push_back(message);
+			hasResult = true;
 		}
-		else {
-			message.user_id = other_user_fbid;
-			message.thread_id = thread_id;
-		}
-
-		ParseMessageType(message, log_type_, log_body_, log_data_);
-
-		messages->push_back(message);
 	}
 
-	return EXIT_SUCCESS;
+	return hasResult ? EXIT_SUCCESS : EXIT_FAILURE;
 }
 
-int FacebookProto::ParseHistory(std::string *data, std::vector< facebook_message >* messages, std::string *firstTimestamp)
+int FacebookProto::ParseHistory(std::string *data, std::vector< facebook_message > *messages, std::string* firstTimestamp)
 {
-	std::string jsonData = data->substr(9);
+	size_t len = data->find("\r\n");
+	if (len != data->npos)
+		data->erase(len);
 
-	JSONNode root = JSONNode::parse(jsonData.c_str());
+	JSONNode root = JSONNode::parse(data->c_str());
 	if (!root)
 		return EXIT_FAILURE;
 
-	const JSONNode &payload = root["payload"];
-	if (!payload)
+	const JSONNode &thread = root["o0"]["data"]["message_thread"];
+	if (!thread)
 		return EXIT_FAILURE;
 
-	const JSONNode &actions = payload["actions"];
-	if (!actions)
+	const JSONNode &nodes = thread["messages"]["nodes"];
+	if (!nodes)
 		return EXIT_FAILURE;
 
+	// TODO! process commented sections and better pair json (this is just quick attempt, + I do not know what everything means yet)
+
 	bool first = true;
 
-	for (auto &it : actions) {
-		const JSONNode &author = it["author"];
-		const JSONNode &other_user_fbid = it["other_user_fbid"];
-		const JSONNode &body = it["body"];
-		const JSONNode &tid = it["thread_id"];
-		const JSONNode &mid = it["message_id"];
-		const JSONNode &timestamp = it["timestamp"];
-		const JSONNode &filtered = it["is_filtered_content"];
-		const JSONNode &is_unread = it["is_unread"];
+	const JSONNode &other_user_fbid_ = thread["thread_key"]["other_user_id"];
+	const JSONNode &thread_fbid_ = thread["thread_key"]["thread_fbid"];
+
+	for (auto it = nodes.begin(); it != nodes.end(); ++it) {
+		const JSONNode &author = (*it)["message_sender"]["id"];
+		const JSONNode &body = (*it)["message"]["text"];
+		const JSONNode &tid = (*it)["offline_threading_id"];
+		const JSONNode &mid = (*it)["message_id"];
+		const JSONNode &timestamp = (*it)["timestamp_precise"];
+		// const JSONNode &filtered = (*it)["is_filtered_content"];
+		const JSONNode &is_unread = (*it)["unread"];
 
 		// Either there is "body" (for classic messages), or "log_message_type" and "log_message_body" (for log messages)
-		const JSONNode &log_type_ = it["log_message_type"];
-		const JSONNode &log_body_ = it["log_message_body"];
-		const JSONNode &log_data_ = it["log_message_data"];
+		const JSONNode &log_type_ = (*it)["log_message_type"];
+		const JSONNode &log_body_ = (*it)["log_message_body"];
+		const JSONNode &log_data_ = (*it)["log_message_data"];
 
 		if (!author || (!body && !log_body_) || !mid || !tid || !timestamp) {
 			debugLogA("ParseHistory: ignoring message (%s) - missing attribute", mid.as_string().c_str());
@@ -1301,16 +1334,16 @@ int FacebookProto::ParseHistory(std::string *data, std::vector< facebook_message
 		std::string message_id = mid.as_string();
 		std::string message_text = body ? body.as_string() : log_body_.as_string();
 		std::string author_id = author.as_string();
-		std::string other_user_id = other_user_fbid ? other_user_fbid.as_string() : "";
+		std::string other_user_id = other_user_fbid_ ? other_user_fbid_.as_string() : "";
 		std::string::size_type pos = author_id.find(":"); // strip "fbid:" prefix
 		if (pos != std::string::npos)
 			author_id = author_id.substr(pos + 1);
 
 		// Process attachements and stickers
-		ParseAttachments(message_text, it, other_user_id, true);
+		ParseAttachments(message_text, (*it), other_user_id, true);
 
-		if (filtered.as_bool() && message_text.empty())
-			message_text = Translate("This message is no longer available, because it was marked as abusive or spam.");
+		//if (filtered.as_bool() && message_text.empty())
+		//	message_text = Translate("This message is no longer available, because it was marked as abusive or spam.");
 
 		message_text = utils::text::trim(utils::text::slashu_to_utf8(message_text), true);
 		if (message_text.empty()) {
@@ -1395,26 +1428,26 @@ int FacebookProto::ParseUserInfo(std::string *data, facebook_user* fbu)
 
 int FacebookProto::ParseMessagesCount(std::string *data, int *messagesCount, int *unreadCount)
 {
-	std::string jsonData = data->substr(9);
+	size_t len = data->find("\r\n");
+	if (len != data->npos)
+		data->erase(len);
 
-	JSONNode root = JSONNode::parse(jsonData.c_str());
+	JSONNode root = JSONNode::parse(data->c_str());
 	if (!root)
 		return EXIT_FAILURE;
 
-	const JSONNode &threads = root["payload"].at("threads");
-	if (!threads)
+	const JSONNode &thread = root["o0"]["data"]["message_thread"];
+	if (!thread)
 		return EXIT_FAILURE;
 
-	for (auto &it : threads) {
-		const JSONNode &message_count_ = it["message_count"];
-		const JSONNode &unread_count_ = it["unread_count"];
+	const JSONNode &message_count_ = thread["messages_count"];
+	const JSONNode &unread_count_ = thread["unread_count"];
 
-		if (!message_count_ || !unread_count_)
-			return EXIT_FAILURE;
+	if (!message_count_ || !unread_count_)
+		return EXIT_FAILURE;
 
-		*messagesCount = message_count_.as_int();
-		*unreadCount = unread_count_.as_int();
-	}
+	*messagesCount = message_count_.as_int();
+	*unreadCount = unread_count_.as_int();
 
 	return EXIT_SUCCESS;
 }
diff --git a/protocols/FacebookRM/src/process.cpp b/protocols/FacebookRM/src/process.cpp
index 39053d2d8a..9ec6e093f6 100644
--- a/protocols/FacebookRM/src/process.cpp
+++ b/protocols/FacebookRM/src/process.cpp
@@ -228,8 +228,8 @@ void FacebookProto::ProcessUnreadMessage(void *pParam)
 		}
 		else facy.handle_error("ProcessUnreadMessage");
 
-		offset += limit;
-		limit = 20; // TODO: use better limits?
+		// offset += limit;
+		// limit = 20; // TODO: use better limits?
 
 		threads->clear(); // TODO: if we have limit messages from one user, there may be more unread messages... continue with it... otherwise remove that threadd from threads list -- or do it in json parser? hm			 = allow more than "limit" unread messages to be parsed
 	}
@@ -264,7 +264,7 @@ void FacebookProto::LoadLastMessages(void *pParam)
 
 	int count = min(FACEBOOK_MESSAGES_ON_OPEN_LIMIT, getByte(FACEBOOK_KEY_MESSAGES_ON_OPEN_COUNT, DEFAULT_MESSAGES_ON_OPEN_COUNT));
 
-	http::response resp = facy.sendRequest(facy.threadInfoRequest(isChat, (const char*)item_id, nullptr, count));
+	http::response resp = facy.sendRequest(facy.threadInfoRequest(isChat, (const char*)item_id, nullptr, count, true));
 	if (resp.code != HTTP_CODE_OK || resp.data.empty()) {
 		facy.handle_error("LoadLastMessages");
 		return;
@@ -362,7 +362,7 @@ void FacebookProto::LoadHistory(void *pParam)
 			break;
 
 		// Load batch of messages
-		resp = facy.sendRequest(facy.threadInfoRequest(isChat, item_id, firstTimestamp.c_str(), messagesPerBatch));
+		resp = facy.sendRequest(facy.threadInfoRequest(isChat, item_id, firstTimestamp.c_str(), messagesPerBatch, true));
 
 		if (resp.code != HTTP_CODE_OK || resp.data.empty()) {
 			facy.handle_error("LoadHistory");
diff --git a/protocols/FacebookRM/src/proto.h b/protocols/FacebookRM/src/proto.h
index d5e3617546..a9b2e798dc 100644
--- a/protocols/FacebookRM/src/proto.h
+++ b/protocols/FacebookRM/src/proto.h
@@ -36,7 +36,7 @@ class FacebookProto : public PROTO<FacebookProto>
 	int ParseChatInfo(std::string* data, facebook_chatroom* fbc);
 	int ParseChatParticipants(std::string *data, std::map<std::string, chatroom_participant>* participants);
 	int ParseFriends(std::string*, std::map< std::string, facebook_user* >*, bool);
-	int ParseHistory(std::string*, std::vector< facebook_message >*, std::string *);
+	int ParseHistory(std::string* data, std::vector<facebook_message>* messages, std::string* firstTimestamp);
 	int ParseMessages(std::string*, std::vector< facebook_message >*, std::map< std::string, facebook_notification* >*);
 	int ParseMessagesCount(std::string *data, int *messagesCount, int *unreadCount);
 	int ParseNotifications(std::string*, std::map< std::string, facebook_notification* >*);