diff options
Diffstat (limited to 'protocols/Telegram/tdlib/td/example/web/tdweb/src')
4 files changed, 1897 insertions, 0 deletions
diff --git a/protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js b/protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js new file mode 100644 index 0000000000..2f159b9424 --- /dev/null +++ b/protocols/Telegram/tdlib/td/example/web/tdweb/src/index.js @@ -0,0 +1,680 @@ +import MyWorker from './worker.js'; +//import localforage from 'localforage'; +import BroadcastChannel from 'broadcast-channel'; +import uuid4 from 'uuid/v4'; +import log from './logger.js'; + +const sleep = ms => new Promise(res => setTimeout(res, ms)); + +/** + * TDLib in a browser + * + * TDLib can be compiled to WebAssembly or asm.js using Emscripten compiler and used in a browser from JavaScript. + * This is a convenient wrapper for TDLib in a browser which controls TDLib instance creation, handles interaction + * with TDLib and manages a filesystem for persistent TDLib data. + * TDLib instance is created in a Web Worker to run it in a separate thread. + * TdClient just sends queries to the Web Worker and receives updates and results from it. + * <br> + * <br> + * Differences from the TDLib JSON API:<br> + * 1. Added the update <code>updateFatalError error:string = Update;</code> which is sent whenever TDLib encounters a fatal error.<br> + * 2. Added the method <code>setJsLogVerbosityLevel new_verbosity_level:string = Ok;</code>, which allows to change the verbosity level of tdweb logging.<br> + * 3. Added the possibility to use blobs as input files via the constructor <code>inputFileBlob data:<JavaScript blob> = InputFile;</code>.<br> + * 4. The class <code>filePart</code> contains data as a JavaScript blob instead of a base64-encoded string.<br> + * 5. The methods <code>getStorageStatistics</code>, <code>getStorageStatisticsFast</code>, <code>optimizeStorage</code>, <code>addProxy</code> and <code>getFileDownloadedPrefixSize</code> are not supported.<br> + * <br> + */ +class TdClient { + /** + * @callback TdClient~updateCallback + * @param {Object} update The update. + */ + + /** + * Create TdClient. + * @param {Object} options - Options for TDLib instance creation. + * @param {TdClient~updateCallback} options.onUpdate - Callback for all incoming updates. + * @param {string} [options.instanceName=tdlib] - Name of the TDLib instance. Currently only one instance of TdClient with a given name is allowed. All but one instances with the same name will be automatically closed. Usually, the newest non-background instance is kept alive. Files will be stored in an IndexedDb table with the same name. + * @param {boolean} [options.isBackground=false] - Pass true if the instance is opened from the background. + * @param {string} [options.jsLogVerbosityLevel=info] - The initial verbosity level of the JavaScript part of the code (one of 'error', 'warning', 'info', 'log', 'debug'). + * @param {number} [options.logVerbosityLevel=2] - The initial verbosity level for the TDLib internal logging (0-1023). + * @param {boolean} [options.useDatabase=true] - Pass false to use TDLib without database and secret chats. It will significantly improve loading time, but some functionality will be unavailable. + * @param {boolean} [options.readOnly=false] - For debug only. Pass true to open TDLib database in read-only mode + * @param {string} [options.mode=auto] - For debug only. The type of the TDLib build to use. 'asmjs' for asm.js and 'wasm' for WebAssembly. If mode == 'auto' WebAbassembly will be used if supported by browser, asm.js otherwise. + */ + constructor(options) { + log.setVerbosity(options.jsLogVerbosityLevel); + this.worker = new MyWorker(); + this.worker.onmessage = e => { + this.onResponse(e.data); + }; + this.query_id = 0; + this.query_callbacks = new Map(); + if ('onUpdate' in options) { + this.onUpdate = options.onUpdate; + delete options.onUpdate; + } + options.instanceName = options.instanceName || 'tdlib'; + this.fileManager = new FileManager(options.instanceName, this); + this.worker.postMessage({ '@type': 'init', options: options }); + this.closeOtherClients(options); + } + + /** + * Send a query to TDLib. + * + * If the query contains the field '@extra', the same field will be added into the result. + * + * @param {Object} query - The query for TDLib. See the [td_api.tl]{@link https://github.com/tdlib/td/blob/master/td/generate/scheme/td_api.tl} scheme or + * the automatically generated [HTML documentation]{@link https://core.telegram.org/tdlib/docs/td__api_8h.html} + * for a list of all available TDLib [methods]{@link https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_function.html} and + * [classes]{@link https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_object.html}. + * @returns {Promise} Promise object represents the result of the query. + */ + send(query) { + return this.doSend(query, true); + } + + /** @private */ + sendInternal(query) { + return this.doSend(query, false); + } + /** @private */ + doSend(query, isExternal) { + this.query_id++; + if (query['@extra']) { + query['@extra'] = { + '@old_extra': JSON.parse(JSON.stringify(query['@extra'])), + query_id: this.query_id + }; + } else { + query['@extra'] = { + query_id: this.query_id + }; + } + if (query['@type'] === 'setJsLogVerbosityLevel') { + log.setVerbosity(query.new_verbosity_level); + } + + log.debug('send to worker: ', query); + const res = new Promise((resolve, reject) => { + this.query_callbacks.set(this.query_id, [resolve, reject]); + }); + if (isExternal) { + this.externalPostMessage(query); + } else { + this.worker.postMessage(query); + } + return res; + } + + /** @private */ + externalPostMessage(query) { + const unsupportedMethods = [ + 'getStorageStatistics', + 'getStorageStatisticsFast', + 'optimizeStorage', + 'addProxy', + 'init', + 'start' + ]; + if (unsupportedMethods.includes(query['@type'])) { + this.onResponse({ + '@type': 'error', + '@extra': query['@extra'], + code: 400, + message: "Method '" + query['@type'] + "' is not supported" + }); + return; + } + if (query['@type'] === 'readFile' || query['@type'] === 'readFilePart') { + this.readFile(query); + return; + } + if (query['@type'] === 'deleteFile') { + this.deleteFile(query); + return; + } + this.worker.postMessage(query); + } + + /** @private */ + async readFile(query) { + const response = await this.fileManager.readFile(query); + this.onResponse(response); + } + + /** @private */ + async deleteFile(query) { + const response = this.fileManager.deleteFile(query); + try { + if (response.idb_key) { + await this.sendInternal({ + '@type': 'deleteIdbKey', + idb_key: response.idb_key + }); + delete response.idb_key; + } + await this.sendInternal({ + '@type': 'deleteFile', + file_id: query.file_id + }); + } catch (e) {} + this.onResponse(response); + } + + /** @private */ + onResponse(response) { + log.debug( + 'receive from worker: ', + JSON.parse( + JSON.stringify(response, (key, value) => { + if (key === 'arr' || key === 'data') { + return undefined; + } + return value; + }) + ) + ); + + // for FileManager + response = this.prepareResponse(response); + + if ('@extra' in response) { + const query_id = response['@extra'].query_id; + const [resolve, reject] = this.query_callbacks.get(query_id); + this.query_callbacks.delete(query_id); + if ('@old_extra' in response['@extra']) { + response['@extra'] = response['@extra']['@old_extra']; + } + if (resolve) { + if (response['@type'] === 'error') { + reject(response); + } else { + resolve(response); + } + } + } else { + if (response['@type'] === 'inited') { + this.onInited(); + return; + } + if (response['@type'] === 'fsInited') { + this.onFsInited(); + return; + } + if ( + response['@type'] === 'updateAuthorizationState' && + response.authorization_state['@type'] === 'authorizationStateClosed' + ) { + this.onClosed(); + } + this.onUpdate(response); + } + } + + /** @private */ + prepareFile(file) { + return this.fileManager.registerFile(file); + } + + /** @private */ + prepareResponse(response) { + if (response['@type'] === 'file') { + if (false && Math.random() < 0.1) { + (async () => { + log.warn('DELETE FILE', response.id); + try { + await this.send({ '@type': 'deleteFile', file_id: response.id }); + } catch (e) {} + })(); + } + return this.prepareFile(response); + } + for (const key in response) { + const field = response[key]; + if ( + field && + typeof field === 'object' && + key !== 'data' && + key !== 'arr' + ) { + response[key] = this.prepareResponse(field); + } + } + return response; + } + + /** @private */ + onBroadcastMessage(e) { + //const message = e.data; + const message = e; + if (message.uid === this.uid) { + log.info('ignore self broadcast message: ', message); + return; + } + log.info('got broadcast message: ', message); + if (message.isBackground && !this.isBackground) { + // continue + } else if ( + (!message.isBackground && this.isBackground) || + message.timestamp > this.timestamp + ) { + this.close(); + return; + } + if (message.state === 'closed') { + this.waitSet.delete(message.uid); + if (this.waitSet.size === 0) { + log.info('onWaitSetEmpty'); + this.onWaitSetEmpty(); + this.onWaitSetEmpty = () => {}; + } + } else { + this.waitSet.add(message.uid); + if (message.state !== 'closing') { + this.postState(); + } + } + } + + /** @private */ + postState() { + const state = { + uid: this.uid, + state: this.state, + timestamp: this.timestamp, + isBackground: this.isBackground + }; + log.info('Post state: ', state); + this.channel.postMessage(state); + } + + /** @private */ + onWaitSetEmpty() { + // nop + } + + /** @private */ + onFsInited() { + this.fileManager.init(); + } + + /** @private */ + onInited() { + this.isInited = true; + this.doSendStart(); + } + + /** @private */ + sendStart() { + this.wantSendStart = true; + this.doSendStart(); + } + + /** @private */ + doSendStart() { + if (!this.isInited || !this.wantSendStart || this.state !== 'start') { + return; + } + this.wantSendStart = false; + this.state = 'active'; + const query = { '@type': 'start' }; + log.info('send to worker: ', query); + this.worker.postMessage(query); + } + + /** @private */ + onClosed() { + this.isClosing = true; + this.worker.terminate(); + log.info('worker is terminated'); + this.state = 'closed'; + this.postState(); + } + + /** @private */ + close() { + if (this.isClosing) { + return; + } + this.isClosing = true; + + log.info('close state: ', this.state); + + if (this.state === 'start') { + this.onClosed(); + this.onUpdate({ + '@type': 'updateAuthorizationState', + authorization_state: { + '@type': 'authorizationStateClosed' + } + }); + return; + } + + const query = { '@type': 'close' }; + log.info('send to worker: ', query); + this.worker.postMessage(query); + + this.state = 'closing'; + this.postState(); + } + + /** @private */ + async closeOtherClients(options) { + this.uid = uuid4(); + this.state = 'start'; + this.isBackground = !!options.isBackground; + this.timestamp = Date.now(); + this.waitSet = new Set(); + + log.info('close other clients'); + this.channel = new BroadcastChannel(options.instanceName, { + webWorkerSupport: false + }); + + this.postState(); + + this.channel.onmessage = message => { + this.onBroadcastMessage(message); + }; + + await sleep(300); + if (this.waitSet.size !== 0) { + await new Promise(resolve => { + this.onWaitSetEmpty = resolve; + }); + } + this.sendStart(); + } + + /** @private */ + onUpdate(update) { + log.info('ignore onUpdate'); + //nop + } +} + +/** @private */ +class ListNode { + constructor(value) { + this.value = value; + this.clear(); + } + + erase() { + this.prev.connect(this.next); + this.clear(); + } + clear() { + this.prev = this; + this.next = this; + } + + connect(other) { + this.next = other; + other.prev = this; + } + + onUsed(other) { + other.usedAt = Date.now(); + other.erase(); + other.connect(this.next); + log.debug('LRU: used file_id: ', other.value); + this.connect(other); + } + + getLru() { + if (this === this.next) { + throw new Error('popLru from empty list'); + } + return this.prev; + } +} + +/** @private */ +class FileManager { + constructor(instanceName, client) { + this.instanceName = instanceName; + this.cache = new Map(); + this.pending = []; + this.transaction_id = 0; + this.totalSize = 0; + this.lru = new ListNode(-1); + this.client = client; + } + + init() { + this.idb = new Promise((resolve, reject) => { + const request = indexedDB.open(this.instanceName); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + //this.store = localforage.createInstance({ + //name: instanceName + //}); + this.isInited = true; + } + + unload(info) { + if (info.arr) { + log.debug( + 'LRU: delete file_id: ', + info.node.value, + ' with arr.length: ', + info.arr.length + ); + this.totalSize -= info.arr.length; + delete info.arr; + } + if (info.node) { + info.node.erase(); + delete info.node; + } + } + + registerFile(file) { + if (file.idb_key || file.arr) { + file.local.is_downloading_completed = true; + } else { + file.local.is_downloading_completed = false; + } + let info = {}; + const cached_info = this.cache.get(file.id); + if (cached_info) { + info = cached_info; + } else { + this.cache.set(file.id, info); + } + if (file.idb_key) { + info.idb_key = file.idb_key; + delete file.idb_key; + } else { + delete info.idb_key; + } + if (file.arr) { + const now = Date.now(); + while (this.totalSize > 100000000) { + const node = this.lru.getLru(); + // immunity for 60 seconds + if (node.usedAt + 60 * 1000 > now) { + break; + } + const lru_info = this.cache.get(node.value); + this.unload(lru_info); + } + + if (info.arr) { + log.warn('Got file.arr at least twice for the same file'); + this.totalSize -= info.arr.length; + } + info.arr = file.arr; + delete file.arr; + this.totalSize += info.arr.length; + if (!info.node) { + log.debug( + 'LRU: create file_id: ', + file.id, + ' with arr.length: ', + info.arr.length + ); + info.node = new ListNode(file.id); + } + this.lru.onUsed(info.node); + log.info('Total file.arr size: ', this.totalSize); + } + info.file = file; + return file; + } + + async flushLoad() { + const pending = this.pending; + this.pending = []; + const idb = await this.idb; + const transaction_id = this.transaction_id++; + const read = idb + .transaction(['keyvaluepairs'], 'readonly') + .objectStore('keyvaluepairs'); + log.debug('Load group of files from idb', pending.length); + for (const query of pending) { + const request = read.get(query.key); + request.onsuccess = event => { + const blob = event.target.result; + if (blob) { + if (blob.size === 0) { + log.error('Got empty blob from db ', query.key); + } + query.resolve({ data: blob, transaction_id: transaction_id }); + } else { + query.reject(); + } + }; + request.onerror = () => query.reject(request.error); + } + } + + load(key, resolve, reject) { + if (this.pending.length === 0) { + setTimeout(() => { + this.flushLoad(); + }, 1); + } + this.pending.push({ key: key, resolve: resolve, reject: reject }); + } + + async doLoadFull(info) { + if (info.arr) { + return { data: new Blob([info.arr]), transaction_id: -1 }; + } + if (info.idb_key) { + const idb_key = info.idb_key; + //return this.store.getItem(idb_key); + return await new Promise((resolve, reject) => { + this.load(idb_key, resolve, reject); + }); + } + throw new Error('File is not loaded'); + } + async doLoad(info, offset, size) { + if (!info.arr && !info.idb_key && info.file.local.path) { + try { + const count = await this.client.sendInternal({ + '@type': 'getFileDownloadedPrefixSize', + file_id: info.file.id, + offset: offset + }); + //log.error(count, size); + if (!size) { + size = count.count; + } else if (size > count.count) { + throw new Error('File not loaded yet'); + } + const res = await this.client.sendInternal({ + '@type': 'readFilePart', + path: info.file.local.path, + offset: offset, + count: size + }); + res.data = new Blob([res.data]); + res.transaction_id = -2; + //log.error(res); + return res; + } catch (e) { + log.info('readFilePart failed', info, offset, size, e); + } + } + + const res = await this.doLoadFull(info); + + // return slice(size, offset + size) + const data_size = res.data.size; + if (!size) { + size = data_size; + } + if (offset > data_size) { + offset = data_size; + } + res.data = res.data.slice(offset, offset + size); + return res; + } + + doDelete(info) { + this.unload(info); + return info.idb_key; + } + + async readFile(query) { + try { + if (!this.isInited) { + throw new Error('FileManager is not inited'); + } + const info = this.cache.get(query.file_id); + if (!info) { + throw new Error('File is not loaded'); + } + if (info.node) { + this.lru.onUsed(info.node); + } + query.offset = query.offset || 0; + query.size = query.count || query.size || 0; + const response = await this.doLoad(info, query.offset, query.size); + return { + '@type': 'filePart', + '@extra': query['@extra'], + data: response.data, + transaction_id: response.transaction_id + }; + } catch (e) { + return { + '@type': 'error', + '@extra': query['@extra'], + code: 400, + message: e + }; + } + } + + deleteFile(query) { + const res = { + '@type': 'ok', + '@extra': query['@extra'] + }; + try { + if (!this.isInited) { + throw new Error('FileManager is not inited'); + } + const info = this.cache.get(query.file_id); + if (!info) { + throw new Error('File is not loaded'); + } + const idb_key = this.doDelete(info); + if (idb_key) { + res.idb_key = idb_key; + } + } catch (e) {} + return res; + } +} + +export default TdClient; diff --git a/protocols/Telegram/tdlib/td/example/web/tdweb/src/logger.js b/protocols/Telegram/tdlib/td/example/web/tdweb/src/logger.js new file mode 100644 index 0000000000..95baed0318 --- /dev/null +++ b/protocols/Telegram/tdlib/td/example/web/tdweb/src/logger.js @@ -0,0 +1,47 @@ +class Logger { + constructor() { + this.setVerbosity('WARNING'); + } + debug(...str) { + if (this.checkVerbosity(4)) { + console.log(...str); + } + } + log(...str) { + if (this.checkVerbosity(4)) { + console.log(...str); + } + } + info(...str) { + if (this.checkVerbosity(3)) { + console.info(...str); + } + } + warn(...str) { + if (this.checkVerbosity(2)) { + console.warn(...str); + } + } + error(...str) { + if (this.checkVerbosity(1)) { + console.error(...str); + } + } + setVerbosity(level, default_level = 'info') { + if (level === undefined) { + level = default_level; + } + if (typeof level === 'string') { + level = + { ERROR: 1, WARNING: 2, INFO: 3, LOG: 4, DEBUG: 4 }[ + level.toUpperCase() + ] || 2; + } + this.level = level; + } + checkVerbosity(level) { + return this.level >= level; + } +} +let log = new Logger(); +export default log; diff --git a/protocols/Telegram/tdlib/td/example/web/tdweb/src/wasm-utils.js b/protocols/Telegram/tdlib/td/example/web/tdweb/src/wasm-utils.js new file mode 100644 index 0000000000..50447d65b3 --- /dev/null +++ b/protocols/Telegram/tdlib/td/example/web/tdweb/src/wasm-utils.js @@ -0,0 +1,136 @@ +// 1. +++ fetchAndInstantiate() +++ // + +// This library function fetches the wasm module at 'url', instantiates it with +// the given 'importObject', and returns the instantiated object instance + +export async function instantiateStreaming(url, importObject) { + let result = await WebAssembly.instantiateStreaming(fetch(url), importObject); + return result.instance; +} +export function fetchAndInstantiate(url, importObject) { + return fetch(url) + .then(response => response.arrayBuffer()) + .then(bytes => WebAssembly.instantiate(bytes, importObject)) + .then(results => results.instance); +} + +// 2. +++ instantiateCachedURL() +++ // + +// This library function fetches the wasm Module at 'url', instantiates it with +// the given 'importObject', and returns a Promise resolving to the finished +// wasm Instance. Additionally, the function attempts to cache the compiled wasm +// Module in IndexedDB using 'url' as the key. The entire site's wasm cache (not +// just the given URL) is versioned by dbVersion and any change in dbVersion on +// any call to instantiateCachedURL() will conservatively clear out the entire +// cache to avoid stale modules. +export function instantiateCachedURL(dbVersion, url, importObject) { + const dbName = 'wasm-cache'; + const storeName = 'wasm-cache'; + + // This helper function Promise-ifies the operation of opening an IndexedDB + // database and clearing out the cache when the version changes. + function openDatabase() { + return new Promise((resolve, reject) => { + var request = indexedDB.open(dbName, dbVersion); + request.onerror = reject.bind(null, 'Error opening wasm cache database'); + request.onsuccess = () => { + resolve(request.result); + }; + request.onupgradeneeded = event => { + var db = request.result; + if (db.objectStoreNames.contains(storeName)) { + console.log(`Clearing out version ${event.oldVersion} wasm cache`); + db.deleteObjectStore(storeName); + } + console.log(`Creating version ${event.newVersion} wasm cache`); + db.createObjectStore(storeName); + }; + }); + } + + // This helper function Promise-ifies the operation of looking up 'url' in the + // given IDBDatabase. + function lookupInDatabase(db) { + return new Promise((resolve, reject) => { + var store = db.transaction([storeName]).objectStore(storeName); + var request = store.get(url); + request.onerror = reject.bind(null, `Error getting wasm module ${url}`); + request.onsuccess = event => { + if (request.result) resolve(request.result); + else reject(`Module ${url} was not found in wasm cache`); + }; + }); + } + + // This helper function fires off an async operation to store the given wasm + // Module in the given IDBDatabase. + function storeInDatabase(db, module) { + var store = db.transaction([storeName], 'readwrite').objectStore(storeName); + var request = store.put(module, url); + request.onerror = err => { + console.log(`Failed to store in wasm cache: ${err}`); + }; + request.onsuccess = err => { + console.log(`Successfully stored ${url} in wasm cache`); + }; + } + + // This helper function fetches 'url', compiles it into a Module, + // instantiates the Module with the given import object. + function fetchAndInstantiate() { + return fetch(url) + .then(response => response.arrayBuffer()) + .then(buffer => WebAssembly.instantiate(buffer, importObject)); + } + + // With all the Promise helper functions defined, we can now express the core + // logic of an IndexedDB cache lookup. We start by trying to open a database. + return openDatabase().then( + db => { + // Now see if we already have a compiled Module with key 'url' in 'db': + return lookupInDatabase(db).then( + module => { + // We do! Instantiate it with the given import object. + console.log(`Found ${url} in wasm cache`); + return WebAssembly.instantiate(module, importObject); + }, + errMsg => { + // Nope! Compile from scratch and then store the compiled Module in 'db' + // with key 'url' for next time. + console.log(errMsg); + return fetchAndInstantiate().then(results => { + try { + storeInDatabase(db, results.module); + } catch (e) { + console.log('Failed to store module into db'); + } + return results.instance; + }); + } + ); + }, + errMsg => { + // If opening the database failed (due to permissions or quota), fall back + // to simply fetching and compiling the module and don't try to store the + // results. + console.log(errMsg); + return fetchAndInstantiate().then(results => results.instance); + } + ); +} + +export async function instantiateAny(version, url, importObject) { + console.log("instantiate"); + try { + return await instantiateStreaming(url, importObject); + } catch (e) { + console.log("instantiateStreaming failed", e); + } + try { + return await instantiateCachedURL(version, url, importObject); + } catch (e) { + console.log("instantiateCachedURL failed", e); + } + throw new Error("can't instantiate wasm"); +} + diff --git a/protocols/Telegram/tdlib/td/example/web/tdweb/src/worker.js b/protocols/Telegram/tdlib/td/example/web/tdweb/src/worker.js new file mode 100644 index 0000000000..dff1845126 --- /dev/null +++ b/protocols/Telegram/tdlib/td/example/web/tdweb/src/worker.js @@ -0,0 +1,1034 @@ +import localforage from 'localforage'; +import log from './logger.js'; +import { instantiateAny } from './wasm-utils.js'; + +import td_wasm_release from './prebuilt/release/td_wasm.wasm'; +import td_asmjs_mem_release from './prebuilt/release/td_asmjs.js.mem'; + +const tdlibVersion = 6; +const localForageDrivers = [ + localforage.INDEXEDDB, + localforage.LOCALSTORAGE, + 'memoryDriver' +]; + +async function initLocalForage() { + // Implement the driver here. + const memoryDriver = { + _driver: 'memoryDriver', + _initStorage: function(options) { + const dbInfo = {}; + if (options) { + for (const i in options) { + dbInfo[i] = options[i]; + } + } + this._dbInfo = dbInfo; + this._map = new Map(); + }, + clear: async function() { + this._map.clear(); + }, + getItem: async function(key) { + const value = this._map.get(key); + console.log('getItem', this._map, key, value); + return value; + }, + iterate: async function(iteratorCallback) { + log.error('iterate is not supported'); + }, + key: async function(n) { + log.error('key n is not supported'); + }, + keys: async function() { + return this._map.keys(); + }, + length: async function() { + return this._map.size(); + }, + removeItem: async function(key) { + this._map.delete(key); + }, + setItem: async function(key, value) { + const originalValue = this._map.get(key); + console.log('setItem', this._map, key, value); + this._map.set(key, value); + return originalValue; + } + }; + + // Add the driver to localForage. + localforage.defineDriver(memoryDriver); +} + +async function loadTdlibWasm(onFS, wasmUrl) { + console.log('loadTdlibWasm'); + const td_module = await import('./prebuilt/release/td_wasm.js'); + const createTdwebModule = td_module.default; + log.info('got td_wasm.js', td_module, createTdwebModule); + let td_wasm = td_wasm_release; + if (wasmUrl) { + td_wasm = wasmUrl; + } + let module = createTdwebModule({ + onRuntimeInitialized: () => { + log.info('runtime intialized'); + onFS(module.FS); + }, + instantiateWasm: (imports, successCallback) => { + log.info('start instantiateWasm', td_wasm, imports); + const next = instance => { + log.info('finish instantiateWasm'); + successCallback(instance); + }; + instantiateAny(tdlibVersion, td_wasm, imports).then(next); + return {}; + }, + ENVIROMENT: 'WORKER' + }); + log.info('Wait module'); + module = await module; + log.info('Got module', module); + //onFS(module.FS); + return module; +} + +async function loadTdlibAsmjs(onFS) { + console.log('loadTdlibAsmjs'); + const createTdwebModule = (await import('./prebuilt/release/td_asmjs.js')) + .default; + console.log('got td_asm.js', createTdwebModule); + const fromFile = 'td_asmjs.js.mem'; + const toFile = td_asmjs_mem_release; + let module = createTdwebModule({ + onRuntimeInitialized: () => { + console.log('runtime intialized'); + onFS(module.FS); + }, + locateFile: name => { + if (name === fromFile) { + return toFile; + } + return name; + }, + ENVIROMENT: 'WORKER' + }); + log.info('Wait module'); + module = await module; + log.info('Got module', module); + //onFS(module.FS); + return module; +} + +async function loadTdlib(mode, onFS, wasmUrl) { + const wasmSupported = (() => { + try { + if ( + typeof WebAssembly === 'object' && + typeof WebAssembly.instantiate === 'function' + ) { + const module = new WebAssembly.Module( + Uint8Array.of(0x0, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00) + ); + if (module instanceof WebAssembly.Module) + return ( + new WebAssembly.Instance(module) instanceof WebAssembly.Instance + ); + } + } catch (e) {} + return false; + })(); + if (!wasmSupported) { + if (mode === 'wasm') { + log.error('WebAssembly is not supported, trying to use it anyway'); + } else { + log.warn('WebAssembly is not supported, trying to use asm.js'); + mode = 'asmjs'; + } + } + + if (mode === 'asmjs') { + return loadTdlibAsmjs(onFS); + } + return loadTdlibWasm(onFS, wasmUrl); +} + +class OutboundFileSystem { + constructor(root, FS) { + this.root = root; + this.nextFileId = 0; + this.FS = FS; + this.files = new Set(); + FS.mkdir(root); + } + blobToPath(blob, name) { + const dir = this.root + '/' + this.nextFileId; + if (!name) { + name = 'blob'; + } + this.nextFileId++; + this.FS.mkdir(dir); + this.FS.mount( + this.FS.filesystems.WORKERFS, + { + blobs: [{ name: name, data: blob }] + }, + dir + ); + const path = dir + '/' + name; + this.files.add(path); + return path; + } + + forgetPath(path) { + if (this.files.has(path)) { + this.FS.unmount(path); + this.files.delete(path); + } + } +} + +class InboundFileSystem { + static async create(dbName, root, FS_promise) { + const start = performance.now(); + try { + const ifs = new InboundFileSystem(); + ifs.pending = []; + ifs.pendingHasTimeout = false; + ifs.persistCount = 0; + ifs.persistSize = 0; + ifs.pendingI = 0; + ifs.inPersist = false; + ifs.totalCount = 0; + + ifs.root = root; + + //ifs.store = localforage.createInstance({ + //name: dbName, + //driver: localForageDrivers + //}); + log.debug('IDB name: ' + dbName); + ifs.idb = new Promise((resolve, reject) => { + const request = indexedDB.open(dbName); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + request.onupgradeneeded = () => { + request.result.createObjectStore('keyvaluepairs'); + }; + }); + + ifs.load_pids(); + + const FS = await FS_promise; + await ifs.idb; + ifs.FS = FS; + ifs.FS.mkdir(root); + const create_time = (performance.now() - start) / 1000; + log.debug('InboundFileSystem::create ' + create_time); + return ifs; + } catch (e) { + log.error('Failed to init Inbound FileSystem: ', e); + } + } + + async load_pids() { + const keys_start = performance.now(); + log.debug('InboundFileSystem::create::keys start'); + //const keys = await this.store.keys(); + + let idb = await this.idb; + let read = idb + .transaction(['keyvaluepairs'], 'readonly') + .objectStore('keyvaluepairs'); + const keys = await new Promise((resolve, reject) => { + const request = read.getAllKeys(); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + + const keys_time = (performance.now() - keys_start) / 1000; + log.debug( + 'InboundFileSystem::create::keys ' + keys_time + ' ' + keys.length + ); + this.pids = new Set(keys); + } + + has(pid) { + if (!this.pids) { + return true; + } + + return this.pids.has(pid); + } + + forget(pid) { + if (this.pids) { + this.pids.delete(pid); + } + } + + async doPersist(pid, path, arr, resolve, reject, write) { + this.persistCount++; + let size = arr.length; + this.persistSize += size; + try { + //log.debug('persist.do start', pid, path, arr.length); + //await this.store.setItem(pid, new Blob([arr])); + await new Promise((resolve, reject) => { + const request = write.put(new Blob([arr]), pid); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + if (this.pids) { + this.pids.add(pid); + } + this.FS.unlink(path); + } catch (e) { + log.error('Failed persist ' + path + ' ', e); + } + //log.debug('persist.do finish', pid, path, arr.length); + this.persistCount--; + this.persistSize -= size; + resolve(); + + this.tryFinishPersist(); + } + + async flushPersist() { + if (this.inPersist) { + return; + } + log.debug('persist.flush'); + this.inPersist = true; + let idb = await this.idb; + this.writeBegin = performance.now(); + let write = idb + .transaction(['keyvaluepairs'], 'readwrite') + .objectStore('keyvaluepairs'); + while ( + this.pendingI < this.pending.length && + this.persistCount < 20 && + this.persistSize < 50 << 20 + ) { + var q = this.pending[this.pendingI]; + this.pending[this.pendingI] = null; + // TODO: add to transaction + this.doPersist(q.pid, q.path, q.arr, q.resolve, q.reject, write); + this.pendingI++; + this.totalCount++; + } + log.debug( + 'persist.flush transaction cnt=' + + this.persistCount + + ', size=' + + this.persistSize + ); + this.inPersist = false; + this.tryFinishPersist(); + } + + async tryFinishPersist() { + if (this.inPersist) { + return; + } + if (this.persistCount !== 0) { + return; + } + log.debug('persist.finish ' + (performance.now() - this.writeBegin) / 1000); + if (this.pendingI === this.pending.length) { + this.pending = []; + this.pendingHasTimeout = false; + this.pendingI = 0; + log.debug('persist.finish done'); + return; + } + log.debug('persist.finish continue'); + this.flushPersist(); + } + + async persist(pid, path, arr) { + if (!this.pendingHasTimeout) { + this.pendingHasTimeout = true; + log.debug('persist set timeout'); + setTimeout(() => { + this.flushPersist(); + }, 1); + } + await new Promise((resolve, reject) => { + this.pending.push({ + pid: pid, + path: path, + arr: arr, + resolve: resolve, + reject: reject + }); + }); + } + + async unlink(pid) { + log.debug('Unlink ' + pid); + try { + this.forget(pid); + //await this.store.removeItem(pid); + let idb = await this.idb; + await new Promise((resolve, reject) => { + let write = idb + .transaction(['keyvaluepairs'], 'readwrite') + .objectStore('keyvaluepairs'); + const request = write.delete(pid); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + } catch (e) { + log.error('Failed unlink ' + pid + ' ', e); + } + } +} + +class DbFileSystem { + static async create(root, FS_promise, readOnly = false) { + const start = performance.now(); + try { + const dbfs = new DbFileSystem(); + dbfs.root = root; + const FS = await FS_promise; + dbfs.FS = FS; + dbfs.syncfs_total_time = 0; + dbfs.readOnly = readOnly; + dbfs.syncActive = 0; + FS.mkdir(root); + FS.mount(FS.filesystems.IDBFS, {}, root); + + await new Promise((resolve, reject) => { + FS.syncfs(true, err => { + resolve(); + }); + }); + + const rmrf = path => { + log.debug('rmrf ', path); + let info; + try { + info = FS.lookupPath(path); + } catch (e) { + return; + } + log.debug('rmrf ', path, info); + if (info.node.isFolder) { + for (const key in info.node.contents) { + rmrf(info.path + '/' + info.node.contents[key].name); + } + log.debug('rmdir ', path); + FS.rmdir(path); + } else { + log.debug('unlink ', path); + FS.unlink(path); + } + }; + //const dirs = ['thumbnails', 'profile_photos', 'secret', 'stickers', 'temp', 'wallpapers', 'secret_thumbnails', 'passport']; + const dirs = []; + const root_dir = FS.lookupPath(root); + for (const key in root_dir.node.contents) { + const value = root_dir.node.contents[key]; + log.debug('node ', key, value); + if (!value.isFolder) { + continue; + } + dirs.push(root_dir.path + '/' + value.name); + } + for (const i in dirs) { + const dir = dirs[i]; + rmrf(dir); + //FS.mkdir(dir); + //FS.mount(FS.filesystems.MEMFS, {}, dir); + } + dbfs.syncfsInterval = setInterval(() => { + dbfs.sync(); + }, 5000); + const create_time = (performance.now() - start) / 1000; + log.debug('DbFileSystem::create ' + create_time); + return dbfs; + } catch (e) { + log.error('Failed to init DbFileSystem: ', e); + } + } + async sync(force) { + if (this.readOnly) { + return; + } + if (this.syncActive > 0 && !force) { + log.debug('SYNC: skip'); + return; + } + this.syncActive++; + const start = performance.now(); + await new Promise((resolve, reject) => { + this.FS.syncfs(false, () => { + const syncfs_time = (performance.now() - start) / 1000; + this.syncfs_total_time += syncfs_time; + log.debug('SYNC: ' + syncfs_time); + log.debug('SYNC total: ' + this.syncfs_total_time); + resolve(); + }); + }); + this.syncActive--; + } + async close() { + clearInterval(this.syncfsInterval); + await this.sync(true); + } + async destroy() { + clearInterval(this.syncfsInterval); + if (this.readOnly) { + return; + } + this.FS.unmount(this.root); + const req = indexedDB.deleteDatabase(this.root); + await new Promise((resolve, reject) => { + req.onsuccess = function(e) { + log.info('SUCCESS'); + resolve(e.result); + }; + req.onerror = function(e) { + log.info('ONERROR'); + reject(e.error); + }; + req.onblocked = function(e) { + log.info('ONBLOCKED'); + reject('blocked'); + }; + }); + } +} + +class TdFileSystem { + static async init_fs(prefix, FS_promise) { + const FS = await FS_promise; + FS.mkdir(prefix); + return FS; + } + static async create(instanceName, FS_promise, readOnly = false) { + try { + const tdfs = new TdFileSystem(); + const prefix = '/' + instanceName; + tdfs.prefix = prefix; + FS_promise = TdFileSystem.init_fs(prefix, FS_promise); + + //MEMFS. Store to IDB and delete files as soon as possible + const inboundFileSystem = InboundFileSystem.create( + instanceName, + prefix + '/inboundfs', + FS_promise + ); + + //IDBFS. MEMFS which is flushed to IDB from time to time + const dbFileSystem = DbFileSystem.create( + prefix + '/dbfs', + FS_promise, + readOnly + ); + + const FS = await FS_promise; + tdfs.FS = FS; + + //WORKERFS. Temporary stores Blobs for outbound files + tdfs.outboundFileSystem = new OutboundFileSystem( + prefix + '/outboundfs', + tdfs.FS + ); + + tdfs.inboundFileSystem = await inboundFileSystem; + tdfs.dbFileSystem = await dbFileSystem; + return tdfs; + } catch (e) { + log.error('Failed to init TdFileSystem: ', e); + } + } + async destroy() { + await this.dbFileSystem.destroy(); + } +} + +class TdClient { + constructor(callback) { + log.info('Start worker'); + this.pendingQueries = []; + this.isPending = true; + this.callback = callback; + this.wasInit = false; + } + + async testLocalForage() { + await initLocalForage(); + const DRIVERS = [ + localforage.INDEXEDDB, + 'memoryDriver', + localforage.LOCALSTORAGE, + localforage.WEBSQL, + localForageDrivers + ]; + for (const driverName of DRIVERS) { + console.log('Test ', driverName); + try { + await localforage.setDriver(driverName); + console.log('A'); + await localforage.setItem('hello', 'world'); + console.log('B'); + const x = await localforage.getItem('hello'); + console.log('got ', x); + await localforage.clear(); + console.log('C'); + } catch (error) { + console.log('Error', error); + } + } + } + + async init(options) { + if (this.wasInit) { + return; + } + //await this.testLocalForage(); + log.setVerbosity(options.jsLogVerbosityLevel); + this.wasInit = true; + + options = options || {}; + const mode = options.mode || 'wasm'; + + const FS_promise = new Promise(resolve => { + this.onFS = resolve; + }); + + const tdfs_promise = TdFileSystem.create( + options.instanceName, + FS_promise, + options.readOnly + ); + + this.useDatabase = true; + if ('useDatabase' in options) { + this.useDatabase = options.useDatabase; + } + + log.info('load TdModule'); + this.TdModule = await loadTdlib(mode, this.onFS, options.wasmUrl); + log.info('got TdModule'); + this.td_functions = { + td_create: this.TdModule.cwrap( + 'td_emscripten_create_client_id', + 'number', + [] + ), + td_send: this.TdModule.cwrap('td_emscripten_send', null, [ + 'number', + 'string' + ]), + td_execute: this.TdModule.cwrap('td_emscripten_execute', 'string', [ + 'string' + ]), + td_receive: this.TdModule.cwrap('td_emscripten_receive', 'string', []), + td_set_verbosity: verbosity => { + this.td_functions.td_execute( + JSON.stringify({ + '@type': 'setLogVerbosityLevel', + new_verbosity_level: verbosity + }) + ); + }, + td_get_timeout: this.TdModule.cwrap( + 'td_emscripten_get_timeout', + 'number', + [] + ) + }; + //this.onFS(this.TdModule.FS); + this.FS = this.TdModule.FS; + this.TdModule['websocket']['on']('error', error => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('open', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('listen', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('connection', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('message', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('close', fd => { + this.scheduleReceiveSoon(); + }); + + // wait till it is allowed to start + this.callback({ '@type': 'inited' }); + await new Promise(resolve => { + this.onStart = resolve; + }); + this.isStarted = true; + + log.info('may start now'); + if (this.isClosing) { + return; + } + log.info('FS start init'); + this.tdfs = await tdfs_promise; + log.info('FS inited'); + this.callback({ '@type': 'fsInited' }); + + // no async initialization after this point + if (options.logVerbosityLevel === undefined) { + options.logVerbosityLevel = 2; + } + this.td_functions.td_set_verbosity(options.logVerbosityLevel); + this.client_id = this.td_functions.td_create(); + + this.savingFiles = new Map(); + this.send({ + '@type': 'setOption', + name: 'store_all_files_in_files_directory', + value: { + '@type': 'optionValueBoolean', + value: true + } + }); + this.send({ + '@type': 'setOption', + name: 'language_pack_database_path', + value: { + '@type': 'optionValueString', + value: this.tdfs.dbFileSystem.root + '/language' + } + }); + this.send({ + '@type': 'setOption', + name: 'ignore_background_updates', + value: { + '@type': 'optionValueBoolean', + value: !this.useDatabase + } + }); + + this.flushPendingQueries(); + + this.receive(); + } + + prepareQueryRecursive(query) { + if (query['@type'] === 'inputFileBlob') { + return { + '@type': 'inputFileLocal', + path: this.tdfs.outboundFileSystem.blobToPath(query.data, query.name) + }; + } + for (const key in query) { + const field = query[key]; + if (field && typeof field === 'object') { + query[key] = this.prepareQueryRecursive(field); + } + } + return query; + } + + prepareQuery(query) { + if (query['@type'] === 'setTdlibParameters') { + query.database_directory = this.tdfs.dbFileSystem.root; + query.files_directory = this.tdfs.inboundFileSystem.root; + + const useDb = this.useDatabase; + query.use_file_database = useDb; + query.use_chat_info_database = useDb; + query.use_message_database = useDb; + query.use_secret_chats = useDb; + } + if (query['@type'] === 'getLanguagePackString') { + query.language_pack_database_path = + this.tdfs.dbFileSystem.root + '/language'; + } + return this.prepareQueryRecursive(query); + } + + onStart() { + //nop + log.info('ignore on_start'); + } + + deleteIdbKey(query) { + try { + } catch (e) { + this.callback({ + '@type': 'error', + '@extra': query['@extra'], + code: 400, + message: e + }); + return; + } + this.callback({ + '@type': 'ok', + '@extra': query['@extra'] + }); + } + + readFilePart(query) { + let res; + try { + //const file_size = this.FS.stat(query.path).size; + const stream = this.FS.open(query.path, 'r'); + const buf = new Uint8Array(query.count); + this.FS.read(stream, buf, 0, query.count, query.offset); + this.FS.close(stream); + res = buf; + } catch (e) { + this.callback({ + '@type': 'error', + '@extra': query['@extra'], + code: 400, + message: e.toString() + }); + return; + } + this.callback( + { + '@type': 'filePart', + '@extra': query['@extra'], + data: res + }, + [res.buffer] + ); + } + + send(query) { + if (this.isClosing) { + return; + } + if (this.wasFatalError) { + if (query['@type'] === 'destroy') { + this.destroy({ '@type': 'ok', '@extra': query['@extra'] }); + } + return; + } + if (query['@type'] === 'init') { + this.init(query.options); + return; + } + if (query['@type'] === 'start') { + log.info('on_start'); + this.onStart(); + return; + } + if (query['@type'] === 'setJsLogVerbosityLevel') { + log.setVerbosity(query.new_verbosity_level); + return; + } + if (this.isPending) { + this.pendingQueries.push(query); + return; + } + if ( + query['@type'] === 'setLogVerbosityLevel' || + query['@type'] === 'getLogVerbosityLevel' || + query['@type'] === 'setLogTagVerbosityLevel' || + query['@type'] === 'getLogTagVerbosityLevel' || + query['@type'] === 'getLogTags' + ) { + this.execute(query); + return; + } + if (query['@type'] === 'readFilePart') { + this.readFilePart(query); + return; + } + if (query['@type'] === 'deleteIdbKey') { + this.deleteIdbKey(query); + return; + } + query = this.prepareQuery(query); + this.td_functions.td_send(this.client_id, JSON.stringify(query)); + this.scheduleReceiveSoon(); + } + + execute(query) { + try { + const res = this.td_functions.td_execute(JSON.stringify(query)); + const response = JSON.parse(res); + this.callback(response); + } catch (error) { + this.onFatalError(error); + } + } + receive() { + this.cancelReceive(); + if (this.wasFatalError) { + return; + } + try { + while (true) { + const msg = this.td_functions.td_receive(); + if (!msg) { + break; + } + const response = this.prepareResponse(JSON.parse(msg)); + if ( + response['@type'] === 'updateAuthorizationState' && + response.authorization_state['@type'] === 'authorizationStateClosed' + ) { + this.close(response); + break; + } + this.callback(response); + } + + this.scheduleReceive(); + } catch (error) { + this.onFatalError(error); + } + } + + cancelReceive() { + if (this.receiveTimeout) { + clearTimeout(this.receiveTimeout); + delete this.receiveTimeout; + } + delete this.receiveSoon; + } + scheduleReceiveSoon() { + if (this.receiveSoon) { + return; + } + this.cancelReceive(); + this.receiveSoon = true; + this.scheduleReceiveIn(0.001); + } + scheduleReceive() { + if (this.receiveSoon) { + return; + } + this.cancelReceive(); + const timeout = this.td_functions.td_get_timeout(); + this.scheduleReceiveIn(timeout); + } + scheduleReceiveIn(timeout) { + //return; + log.debug('Scheduler receive in ' + timeout + 's'); + this.receiveTimeout = setTimeout(() => this.receive(), timeout * 1000); + } + + onFatalError(error) { + this.wasFatalError = true; + this.asyncOnFatalError(error); + } + + async close(last_update) { + // close db and cancell all timers + this.isClosing = true; + if (this.isStarted) { + log.debug('close worker: start'); + await this.tdfs.dbFileSystem.close(); + this.cancelReceive(); + log.debug('close worker: finish'); + } + this.callback(last_update); + } + + async destroy(result) { + try { + log.info('destroy tdfs ...'); + await this.tdfs.destroy(); + log.info('destroy tdfs ok'); + } catch (e) { + log.error('Failed destroy', e); + } + this.callback(result); + this.callback({ + '@type': 'updateAuthorizationState', + authorization_state: { + '@type': 'authorizationStateClosed' + } + }); + } + + async asyncOnFatalError(error) { + await this.tdfs.dbFileSystem.sync(); + this.callback({ '@type': 'updateFatalError', error: error }); + } + + saveFile(pid, file) { + const isSaving = this.savingFiles.has(pid); + this.savingFiles.set(pid, file); + if (isSaving) { + return file; + } + try { + const arr = this.FS.readFile(file.local.path); + if (arr) { + file = Object.assign({}, file); + file.arr = arr; + this.doSaveFile(pid, file, arr); + } + } catch (e) { + log.error('Failed to readFile: ', e); + } + return file; + } + + async doSaveFile(pid, file, arr) { + await this.tdfs.inboundFileSystem.persist(pid, file.local.path, arr); + file = this.savingFiles.get(pid); + file.idb_key = pid; + this.callback({ '@type': 'updateFile', file: file }); + + this.savingFiles.delete(pid); + } + + prepareFile(file) { + const pid = file.remote.unique_id ? file.remote.unique_id : file.remote.id; + if (!pid) { + return file; + } + + if (file.local.is_downloading_active) { + this.tdfs.inboundFileSystem.forget(pid); + } else if (this.tdfs.inboundFileSystem.has(pid)) { + file.idb_key = pid; + return file; + } + + if (file.local.is_downloading_completed) { + file = this.saveFile(pid, file); + } + return file; + } + + prepareResponse(response) { + if (response['@type'] === 'file') { + return this.prepareFile(response); + } + for (const key in response) { + const field = response[key]; + if (field && typeof field === 'object') { + response[key] = this.prepareResponse(field); + } + } + return response; + } + + flushPendingQueries() { + this.isPending = false; + for (const query of this.pendingQueries) { + this.send(query); + } + } +} + +const client = new TdClient((e, t = []) => postMessage(e, t)); + +onmessage = function(e) { + try { + client.send(e.data); + } catch (error) { + client.onFatalError(error); + } +}; |