From efc336e60cf1331bf5f3213d296981b87b8b2a6c Mon Sep 17 00:00:00 2001 From: George Hazan Date: Sun, 4 Jun 2023 19:24:05 +0300 Subject: =?UTF-8?q?fixes=20#3537=20(Telegram:=2032-=D1=80=D0=B0=D0=B7?= =?UTF-8?q?=D1=80=D1=8F=D0=B4=D0=BD=D0=B0=D1=8F=20=D0=B2=D0=B5=D1=80=D1=81?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=BF=D0=B0=D0=B4=D0=B0=D0=B5=D1=82=20=D0=B2=20?= =?UTF-8?q?64-=D1=80=D0=B0=D0=B7=D1=80=D1=8F=D0=B4=D0=BD=D0=BE=D0=B9=20Win?= =?UTF-8?q?dows)=20+=20update=20to=20the=20fresh=20TDLIB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tdlib/td/example/web/tdweb/src/index.js | 680 +++++++++++++++++++++ 1 file changed, 680 insertions(+) create mode 100644 protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js (limited to 'protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js') diff --git a/protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js b/protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js new file mode 100644 index 0000000000..cf365e9b5e --- /dev/null +++ b/protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js @@ -0,0 +1,680 @@ +import MyWorker from './worker.js'; +//import localforage from 'localforage'; +import BroadcastChannel from 'broadcast-channel'; +import uuid4 from 'uuid/v4'; +import log from './logger.js'; + +const sleep = ms => new Promise(res => setTimeout(res, ms)); + +/** + * TDLib in a browser + * + * TDLib can be compiled to WebAssembly or asm.js using Emscripten compiler and used in a browser from JavaScript. + * This is a convenient wrapper for TDLib in a browser which controls TDLib instance creation, handles interaction + * with TDLib and manages a filesystem for persistent TDLib data. + * TDLib instance is created in a Web Worker to run it in a separate thread. + * TdClient just sends queries to the Web Worker and receives updates and results from it. + *
+ *
+ * Differences from the TDLib JSON API:
+ * 1. Added the update updateFatalError error:string = Update; which is sent whenever TDLib encounters a fatal error.
+ * 2. Added the method setJsLogVerbosityLevel new_verbosity_level:string = Ok;, which allows to change the verbosity level of tdweb logging.
+ * 3. Added the possibility to use blobs as input files via the constructor inputFileBlob data: = InputFile;.
+ * 4. The class filePart contains data as a JavaScript blob instead of a base64-encoded string.
+ * 5. The methods getStorageStatistics, getStorageStatisticsFast, optimizeStorage, addProxy and getFileDownloadedPrefixSize are not supported.
+ *
+ */ +class TdClient { + /** + * @callback TdClient~updateCallback + * @param {Object} update The update. + */ + + /** + * Create TdClient. + * @param {Object} options - Options for TDLib instance creation. + * @param {TdClient~updateCallback} options.onUpdate - Callback for all incoming updates. + * @param {string} [options.instanceName=tdlib] - Name of the TDLib instance. Currently only one instance of TdClient with a given name is allowed. All but one instances with the same name will be automatically closed. Usually, the newest non-background instance is kept alive. Files will be stored in an IndexedDb table with the same name. + * @param {boolean} [options.isBackground=false] - Pass true if the instance is opened from the background. + * @param {string} [options.jsLogVerbosityLevel=info] - The initial verbosity level of the JavaScript part of the code (one of 'error', 'warning', 'info', 'log', 'debug'). + * @param {number} [options.logVerbosityLevel=2] - The initial verbosity level for the TDLib internal logging (0-1023). + * @param {boolean} [options.useDatabase=true] - Pass false to use TDLib without database and secret chats. It will significantly improve loading time, but some functionality will be unavailable. + * @param {boolean} [options.readOnly=false] - For debug only. Pass true to open TDLib database in read-only mode + * @param {string} [options.mode=auto] - For debug only. The type of the TDLib build to use. 'asmjs' for asm.js and 'wasm' for WebAssembly. If mode == 'auto' WebAbassembly will be used if supported by browser, asm.js otherwise. + */ + constructor(options) { + log.setVerbosity(options.jsLogVerbosityLevel); + this.worker = new MyWorker(); + this.worker.onmessage = e => { + this.onResponse(e.data); + }; + this.query_id = 0; + this.query_callbacks = new Map(); + if ('onUpdate' in options) { + this.onUpdate = options.onUpdate; + delete options.onUpdate; + } + options.instanceName = options.instanceName || 'tdlib'; + this.fileManager = new FileManager(options.instanceName, this); + this.worker.postMessage({ '@type': 'init', options: options }); + this.closeOtherClients(options); + } + + /** + * Send a query to TDLib. + * + * If the query contains the field '@extra', the same field will be added into the result. + * + * @param {Object} query - The query for TDLib. See the [td_api.tl]{@link https://github.com/tdlib/td/blob/master/td/generate/scheme/td_api.tl} scheme or + * the automatically generated [HTML documentation]{@link https://core.telegram.org/tdlib/docs/td__api_8h.html} + * for a list of all available TDLib [methods]{@link https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_function.html} and + * [classes]{@link https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_object.html}. + * @returns {Promise} Promise object represents the result of the query. + */ + send(query) { + return this.doSend(query, true); + } + + /** @private */ + sendInternal(query) { + return this.doSend(query, false); + } + /** @private */ + doSend(query, isExternal) { + this.query_id++; + if (query['@extra']) { + query['@extra'] = { + '@old_extra': JSON.parse(JSON.stringify(query['@extra'])), + query_id: this.query_id + }; + } else { + query['@extra'] = { + query_id: this.query_id + }; + } + if (query['@type'] === 'setJsLogVerbosityLevel') { + log.setVerbosity(query.new_verbosity_level); + } + + log.debug('send to worker: ', query); + const res = new Promise((resolve, reject) => { + this.query_callbacks.set(this.query_id, [resolve, reject]); + }); + if (isExternal) { + this.externalPostMessage(query); + } else { + this.worker.postMessage(query); + } + return res; + } + + /** @private */ + externalPostMessage(query) { + const unsupportedMethods = [ + 'getStorageStatistics', + 'getStorageStatisticsFast', + 'optimizeStorage', + 'addProxy', + 'init', + 'start' + ]; + if (unsupportedMethods.includes(query['@type'])) { + this.onResponse({ + '@type': 'error', + '@extra': query['@extra'], + code: 400, + message: "Method '" + query['@type'] + "' is not supported" + }); + return; + } + if (query['@type'] === 'readFile' || query['@type'] === 'readFilePart') { + this.readFile(query); + return; + } + if (query['@type'] === 'deleteFile') { + this.deleteFile(query); + return; + } + this.worker.postMessage(query); + } + + /** @private */ + async readFile(query) { + const response = await this.fileManager.readFile(query); + this.onResponse(response); + } + + /** @private */ + async deleteFile(query) { + const response = this.fileManager.deleteFile(query); + try { + if (response.idb_key) { + await this.sendInternal({ + '@type': 'deleteIdbKey', + idb_key: response.idb_key + }); + delete response.idb_key; + } + await this.sendInternal({ + '@type': 'deleteFile', + file_id: query.file_id + }); + } catch (e) {} + this.onResponse(response); + } + + /** @private */ + onResponse(response) { + log.debug( + 'receive from worker: ', + JSON.parse( + JSON.stringify(response, (key, value) => { + if (key === 'arr' || key === 'data') { + return undefined; + } + return value; + }) + ) + ); + + // for FileManager + response = this.prepareResponse(response); + + if ('@extra' in response) { + const query_id = response['@extra'].query_id; + const [resolve, reject] = this.query_callbacks.get(query_id); + this.query_callbacks.delete(query_id); + if ('@old_extra' in response['@extra']) { + response['@extra'] = response['@extra']['@old_extra']; + } + if (resolve) { + if (response['@type'] === 'error') { + reject(response); + } else { + resolve(response); + } + } + } else { + if (response['@type'] === 'inited') { + this.onInited(); + return; + } + if (response['@type'] === 'fsInited') { + this.onFsInited(); + return; + } + if ( + response['@type'] === 'updateAuthorizationState' && + response.authorization_state['@type'] === 'authorizationStateClosed' + ) { + this.onClosed(); + } + this.onUpdate(response); + } + } + + /** @private */ + prepareFile(file) { + return this.fileManager.registerFile(file); + } + + /** @private */ + prepareResponse(response) { + if (response['@type'] === 'file') { + if (false && Math.random() < 0.1) { + (async () => { + log.warn('DELETE FILE', response.id); + try { + await this.send({ '@type': 'deleteFile', file_id: response.id }); + } catch (e) {} + })(); + } + return this.prepareFile(response); + } + for (const key in response) { + const field = response[key]; + if ( + field && + typeof field === 'object' && + key !== 'data' && + key !== 'arr' + ) { + response[key] = this.prepareResponse(field); + } + } + return response; + } + + /** @private */ + onBroadcastMessage(e) { + //const message = e.data; + const message = e; + if (message.uid === this.uid) { + log.info('ignore self broadcast message: ', message); + return; + } + log.info('receive broadcast message: ', message); + if (message.isBackground && !this.isBackground) { + // continue + } else if ( + (!message.isBackground && this.isBackground) || + message.timestamp > this.timestamp + ) { + this.close(); + return; + } + if (message.state === 'closed') { + this.waitSet.delete(message.uid); + if (this.waitSet.size === 0) { + log.info('onWaitSetEmpty'); + this.onWaitSetEmpty(); + this.onWaitSetEmpty = () => {}; + } + } else { + this.waitSet.add(message.uid); + if (message.state !== 'closing') { + this.postState(); + } + } + } + + /** @private */ + postState() { + const state = { + uid: this.uid, + state: this.state, + timestamp: this.timestamp, + isBackground: this.isBackground + }; + log.info('Post state: ', state); + this.channel.postMessage(state); + } + + /** @private */ + onWaitSetEmpty() { + // nop + } + + /** @private */ + onFsInited() { + this.fileManager.init(); + } + + /** @private */ + onInited() { + this.isInited = true; + this.doSendStart(); + } + + /** @private */ + sendStart() { + this.wantSendStart = true; + this.doSendStart(); + } + + /** @private */ + doSendStart() { + if (!this.isInited || !this.wantSendStart || this.state !== 'start') { + return; + } + this.wantSendStart = false; + this.state = 'active'; + const query = { '@type': 'start' }; + log.info('send to worker: ', query); + this.worker.postMessage(query); + } + + /** @private */ + onClosed() { + this.isClosing = true; + this.worker.terminate(); + log.info('worker is terminated'); + this.state = 'closed'; + this.postState(); + } + + /** @private */ + close() { + if (this.isClosing) { + return; + } + this.isClosing = true; + + log.info('close state: ', this.state); + + if (this.state === 'start') { + this.onClosed(); + this.onUpdate({ + '@type': 'updateAuthorizationState', + authorization_state: { + '@type': 'authorizationStateClosed' + } + }); + return; + } + + const query = { '@type': 'close' }; + log.info('send to worker: ', query); + this.worker.postMessage(query); + + this.state = 'closing'; + this.postState(); + } + + /** @private */ + async closeOtherClients(options) { + this.uid = uuid4(); + this.state = 'start'; + this.isBackground = !!options.isBackground; + this.timestamp = Date.now(); + this.waitSet = new Set(); + + log.info('close other clients'); + this.channel = new BroadcastChannel(options.instanceName, { + webWorkerSupport: false + }); + + this.postState(); + + this.channel.onmessage = message => { + this.onBroadcastMessage(message); + }; + + await sleep(300); + if (this.waitSet.size !== 0) { + await new Promise(resolve => { + this.onWaitSetEmpty = resolve; + }); + } + this.sendStart(); + } + + /** @private */ + onUpdate(update) { + log.info('ignore onUpdate'); + //nop + } +} + +/** @private */ +class ListNode { + constructor(value) { + this.value = value; + this.clear(); + } + + erase() { + this.prev.connect(this.next); + this.clear(); + } + clear() { + this.prev = this; + this.next = this; + } + + connect(other) { + this.next = other; + other.prev = this; + } + + onUsed(other) { + other.usedAt = Date.now(); + other.erase(); + other.connect(this.next); + log.debug('LRU: used file_id: ', other.value); + this.connect(other); + } + + getLru() { + if (this === this.next) { + throw new Error('popLru from empty list'); + } + return this.prev; + } +} + +/** @private */ +class FileManager { + constructor(instanceName, client) { + this.instanceName = instanceName; + this.cache = new Map(); + this.pending = []; + this.transaction_id = 0; + this.totalSize = 0; + this.lru = new ListNode(-1); + this.client = client; + } + + init() { + this.idb = new Promise((resolve, reject) => { + const request = indexedDB.open(this.instanceName); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + //this.store = localforage.createInstance({ + //name: instanceName + //}); + this.isInited = true; + } + + unload(info) { + if (info.arr) { + log.debug( + 'LRU: delete file_id: ', + info.node.value, + ' with arr.length: ', + info.arr.length + ); + this.totalSize -= info.arr.length; + delete info.arr; + } + if (info.node) { + info.node.erase(); + delete info.node; + } + } + + registerFile(file) { + if (file.idb_key || file.arr) { + file.local.is_downloading_completed = true; + } else { + file.local.is_downloading_completed = false; + } + let info = {}; + const cached_info = this.cache.get(file.id); + if (cached_info) { + info = cached_info; + } else { + this.cache.set(file.id, info); + } + if (file.idb_key) { + info.idb_key = file.idb_key; + delete file.idb_key; + } else { + delete info.idb_key; + } + if (file.arr) { + const now = Date.now(); + while (this.totalSize > 100000000) { + const node = this.lru.getLru(); + // immunity for 60 seconds + if (node.usedAt + 60 * 1000 > now) { + break; + } + const lru_info = this.cache.get(node.value); + this.unload(lru_info); + } + + if (info.arr) { + log.warn('Receive file.arr at least twice for the same file'); + this.totalSize -= info.arr.length; + } + info.arr = file.arr; + delete file.arr; + this.totalSize += info.arr.length; + if (!info.node) { + log.debug( + 'LRU: create file_id: ', + file.id, + ' with arr.length: ', + info.arr.length + ); + info.node = new ListNode(file.id); + } + this.lru.onUsed(info.node); + log.info('Total file.arr size: ', this.totalSize); + } + info.file = file; + return file; + } + + async flushLoad() { + const pending = this.pending; + this.pending = []; + const idb = await this.idb; + const transaction_id = this.transaction_id++; + const read = idb + .transaction(['keyvaluepairs'], 'readonly') + .objectStore('keyvaluepairs'); + log.debug('Load group of files from idb', pending.length); + for (const query of pending) { + const request = read.get(query.key); + request.onsuccess = event => { + const blob = event.target.result; + if (blob) { + if (blob.size === 0) { + log.error('Receive empty blob from db ', query.key); + } + query.resolve({ data: blob, transaction_id: transaction_id }); + } else { + query.reject(); + } + }; + request.onerror = () => query.reject(request.error); + } + } + + load(key, resolve, reject) { + if (this.pending.length === 0) { + setTimeout(() => { + this.flushLoad(); + }, 1); + } + this.pending.push({ key: key, resolve: resolve, reject: reject }); + } + + async doLoadFull(info) { + if (info.arr) { + return { data: new Blob([info.arr]), transaction_id: -1 }; + } + if (info.idb_key) { + const idb_key = info.idb_key; + //return this.store.getItem(idb_key); + return await new Promise((resolve, reject) => { + this.load(idb_key, resolve, reject); + }); + } + throw new Error('File is not loaded'); + } + async doLoad(info, offset, size) { + if (!info.arr && !info.idb_key && info.file.local.path) { + try { + const count = await this.client.sendInternal({ + '@type': 'getFileDownloadedPrefixSize', + file_id: info.file.id, + offset: offset + }); + //log.error(count, size); + if (!size) { + size = count.count; + } else if (size > count.count) { + throw new Error('File not loaded yet'); + } + const res = await this.client.sendInternal({ + '@type': 'readFilePart', + path: info.file.local.path, + offset: offset, + count: size + }); + res.data = new Blob([res.data]); + res.transaction_id = -2; + //log.error(res); + return res; + } catch (e) { + log.info('readFilePart failed', info, offset, size, e); + } + } + + const res = await this.doLoadFull(info); + + // return slice(size, offset + size) + const data_size = res.data.size; + if (!size) { + size = data_size; + } + if (offset > data_size) { + offset = data_size; + } + res.data = res.data.slice(offset, offset + size); + return res; + } + + doDelete(info) { + this.unload(info); + return info.idb_key; + } + + async readFile(query) { + try { + if (!this.isInited) { + throw new Error('FileManager is not inited'); + } + const info = this.cache.get(query.file_id); + if (!info) { + throw new Error('File is not loaded'); + } + if (info.node) { + this.lru.onUsed(info.node); + } + query.offset = query.offset || 0; + query.size = query.count || query.size || 0; + const response = await this.doLoad(info, query.offset, query.size); + return { + '@type': 'filePart', + '@extra': query['@extra'], + data: response.data, + transaction_id: response.transaction_id + }; + } catch (e) { + return { + '@type': 'error', + '@extra': query['@extra'], + code: 400, + message: e + }; + } + } + + deleteFile(query) { + const res = { + '@type': 'ok', + '@extra': query['@extra'] + }; + try { + if (!this.isInited) { + throw new Error('FileManager is not inited'); + } + const info = this.cache.get(query.file_id); + if (!info) { + throw new Error('File is not loaded'); + } + const idb_key = this.doDelete(info); + if (idb_key) { + res.idb_key = idb_key; + } + } catch (e) {} + return res; + } +} + +export default TdClient; -- cgit v1.2.3