diff options
Diffstat (limited to 'libs/tdlib/td/td/telegram/files')
38 files changed, 8958 insertions, 0 deletions
diff --git a/libs/tdlib/td/td/telegram/files/FileDb.cpp b/libs/tdlib/td/td/telegram/files/FileDb.cpp new file mode 100644 index 0000000000..96795dcea1 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileDb.cpp @@ -0,0 +1,290 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileDb.h" + +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/Version.h" + +#include "td/actor/actor.h" + +#include "td/db/SqliteKeyValueSafe.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/Status.h" +#include "td/utils/tl_helpers.h" +#include "td/utils/tl_parsers.h" +#include "td/utils/tl_storers.h" + +namespace td { + +Status drop_file_db(SqliteDb &db, int32 version) { + LOG(WARNING) << "Drop file_db " << tag("version", version) << tag("current_db_version", current_db_version()); + TRY_STATUS(SqliteKeyValue::drop(db, "files")); + return Status::OK(); +} + +Status fix_file_remote_location_key_bug(SqliteDb &db); +Status init_file_db(SqliteDb &db, int32 version) { + LOG(INFO) << "Init file db " << tag("version", version); + + // Check if database exists + TRY_RESULT(has_table, db.has_table("files")); + + if (!has_table) { + version = 0; + } else if (version < static_cast<int32>(DbVersion::DialogDbCreated)) { + TRY_STATUS(drop_file_db(db, version)); + version = 0; + } else if (version < static_cast<int>(DbVersion::FixFileRemoteLocationKeyBug)) { + TRY_STATUS(fix_file_remote_location_key_bug(db)); + } + + if (version == 0) { + TRY_STATUS(SqliteKeyValue::init(db, "files")); + } + return Status::OK(); +} + +class FileDb : public FileDbInterface { + public: + class FileDbActor : public Actor { + public: + using Id = FileDbInterface::Id; + FileDbActor(Id current_pmc_id, std::shared_ptr<SqliteKeyValueSafe> file_kv_safe) + : current_pmc_id_(current_pmc_id), file_kv_safe_(std::move(file_kv_safe)) { + } + + void close(Promise<> promise) { + file_kv_safe_.reset(); + LOG(INFO) << "FileDb is closed"; + promise.set_value(Unit()); + stop(); + } + + void load_file_data(const string &key, Promise<FileData> promise) { + promise.set_result(load_file_data_impl(file_pmc(), key)); + } + + void clear_file_data(Id id, const string &remote_key, const string &local_key, const string &generate_key) { + auto &pmc = file_pmc(); + pmc.begin_transaction().ensure(); + SCOPE_EXIT { + pmc.commit_transaction().ensure(); + }; + + if (id > current_pmc_id_) { + pmc.set("file_id", to_string(id)); + current_pmc_id_ = id; + } + + pmc.erase("file" + to_string(id)); + LOG(DEBUG) << "ERASE " << format::as_hex_dump<4>(Slice(PSLICE() << "file" << to_string(id))); + + if (!remote_key.empty()) { + pmc.erase(remote_key); + LOG(DEBUG) << "ERASE remote " << format::as_hex_dump<4>(Slice(remote_key)); + } + if (!local_key.empty()) { + pmc.erase(local_key); + LOG(DEBUG) << "ERASE local " << format::as_hex_dump<4>(Slice(local_key)); + } + if (!generate_key.empty()) { + pmc.erase(generate_key); + } + } + void store_file_data(Id id, const string &file_data, const string &remote_key, const string &local_key, + const string &generate_key) { + auto &pmc = file_pmc(); + pmc.begin_transaction().ensure(); + SCOPE_EXIT { + pmc.commit_transaction().ensure(); + }; + + if (id > current_pmc_id_) { + pmc.set("file_id", to_string(id)); + current_pmc_id_ = id; + } + + pmc.set("file" + to_string(id), file_data); + + if (!remote_key.empty()) { + pmc.set(remote_key, to_string(id)); + } + if (!local_key.empty()) { + pmc.set(local_key, to_string(id)); + } + if (!generate_key.empty()) { + pmc.set(generate_key, to_string(id)); + } + } + void store_file_data_ref(Id id, Id new_id) { + auto &pmc = file_pmc(); + pmc.begin_transaction().ensure(); + SCOPE_EXIT { + pmc.commit_transaction().ensure(); + }; + + if (id > current_pmc_id_) { + pmc.set("file_id", to_string(id)); + current_pmc_id_ = id; + } + + pmc.set("file" + to_string(id), "@@" + to_string(new_id)); + } + + private: + Id current_pmc_id_; + std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_; + + SqliteKeyValue &file_pmc() { + return file_kv_safe_->get(); + } + }; + + explicit FileDb(std::shared_ptr<SqliteKeyValueSafe> kv_safe, int scheduler_id = -1) { + file_kv_safe_ = std::move(kv_safe); + CHECK(file_kv_safe_); + current_pmc_id_ = to_integer<int32>(file_kv_safe_->get().get("file_id")); + file_db_actor_ = + create_actor_on_scheduler<FileDbActor>("FileDbActor", scheduler_id, current_pmc_id_, file_kv_safe_); + } + + Id create_pmc_id() override { + return ++current_pmc_id_; + } + + void close(Promise<> promise) override { + send_closure(std::move(file_db_actor_), &FileDbActor::close, std::move(promise)); + } + + void get_file_data_impl(string key, Promise<FileData> promise) override { + send_closure(file_db_actor_, &FileDbActor::load_file_data, std::move(key), std::move(promise)); + } + + Result<FileData> get_file_data_sync_impl(string key) override { + return load_file_data_impl(file_kv_safe_->get(), key); + } + + void clear_file_data(Id id, const FileData &file_data) override { + string remote_key; + if (file_data.remote_.type() == RemoteFileLocation::Type::Full) { + remote_key = as_key(file_data.remote_.full()); + } + string local_key; + if (file_data.local_.type() == LocalFileLocation::Type::Full) { + local_key = as_key(file_data.local_.full()); + } + string generate_key; + if (file_data.generate_ != nullptr) { + generate_key = as_key(*file_data.generate_); + } + send_closure(file_db_actor_, &FileDbActor::clear_file_data, id, remote_key, local_key, generate_key); + } + void set_file_data(Id id, const FileData &file_data, bool new_remote, bool new_local, bool new_generate) override { + string remote_key; + if (file_data.remote_.type() == RemoteFileLocation::Type::Full && new_remote) { + remote_key = as_key(file_data.remote_.full()); + } + string local_key; + if (file_data.local_.type() == LocalFileLocation::Type::Full && new_local) { + local_key = as_key(file_data.local_.full()); + } + string generate_key; + if (file_data.generate_ != nullptr && new_generate) { + generate_key = as_key(*file_data.generate_); + } + LOG(DEBUG) << "SAVE " << id << " -> " << file_data << " " + << tag("remote", format::as_hex_dump<4>(Slice(remote_key))) + << tag("local", format::as_hex_dump<4>(Slice(local_key))); + send_closure(file_db_actor_, &FileDbActor::store_file_data, id, serialize(file_data), remote_key, local_key, + generate_key); + } + + void set_file_data_ref(Id id, Id new_id) override { + send_closure(file_db_actor_, &FileDbActor::store_file_data_ref, id, new_id); + } + SqliteKeyValue &pmc() override { + return file_kv_safe_->get(); + } + + private: + ActorOwn<FileDbActor> file_db_actor_; + Id current_pmc_id_; + std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_; + + static Result<FileData> load_file_data_impl(SqliteKeyValue &pmc, const string &key) { + //LOG(DEBUG) << "Load by key " << format::as_hex_dump<4>(Slice(key)); + TRY_RESULT(id, get_id(pmc, key)); + + string data_str; + int attempt_count = 0; + while (true) { + if (attempt_count > 5) { + LOG(FATAL) << "cycle in files db?"; + } + attempt_count++; + + data_str = pmc.get(PSTRING() << "file" << id); + auto data_slice = Slice(data_str); + + if (data_slice.substr(0, 2) == "@@") { + id = to_integer<Id>(data_slice.substr(2)); + } else { + break; + } + } + //LOG(DEBUG) << "By id " << id << " found data " << format::as_hex_dump<4>(Slice(data_str)); + + FileData data; + auto status = unserialize(data, data_str); + if (status.is_error()) { + return std::move(status); + } + return std::move(data); + } + + static Result<Id> get_id(SqliteKeyValue &pmc, const string &key) TD_WARN_UNUSED_RESULT { + auto id_str = pmc.get(key); + //LOG(DEBUG) << "Found id " << id_str << " by key " << format::as_hex_dump<4>(Slice(key)); + if (id_str.empty()) { + return Status::Error("There is no such a key in db"); + } + return to_integer<Id>(id_str); + } +}; + +std::shared_ptr<FileDbInterface> create_file_db(std::shared_ptr<SqliteConnectionSafe> connection, int scheduler_id) { + auto kv = std::make_shared<SqliteKeyValueSafe>("files", std::move(connection)); + return std::make_shared<FileDb>(std::move(kv), scheduler_id); +} + +Status fix_file_remote_location_key_bug(SqliteDb &db) { + static const int32 OLD_KEY_MAGIC = 0x64378433; + SqliteKeyValue kv; + kv.init_with_connection(db.clone(), "files").ensure(); + auto ptr = StackAllocator::alloc(4); + MutableSlice prefix = ptr.as_slice(); + TlStorerUnsafe(prefix.begin()).store_int(OLD_KEY_MAGIC); + kv.get_by_prefix(prefix, [&](Slice key, Slice value) { + CHECK(TlParser(key).fetch_int() == OLD_KEY_MAGIC); + auto remote_str = PSTRING() << key.substr(4, 4) << Slice("\0\0\0\0") << key.substr(8); + FullRemoteFileLocation remote; + if (unserialize(remote, remote_str).is_ok()) { + kv.set(as_key(remote), value); + } + LOG(DEBUG) << "ERASE " << format::as_hex_dump<4>(Slice(key)); + kv.erase(key); + }); + return Status::OK(); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileDb.h b/libs/tdlib/td/td/telegram/files/FileDb.h new file mode 100644 index 0000000000..d53fb32091 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileDb.h @@ -0,0 +1,75 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/files/FileLocation.h" + +#include "td/actor/PromiseFuture.h" + +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +#include <memory> + +namespace td { +class SqliteDb; +class SqliteConnectionSafe; +class SqliteKeyValue; +} // namespace td + +namespace td { +Status drop_file_db(SqliteDb &db, int32 version) TD_WARN_UNUSED_RESULT; +Status init_file_db(SqliteDb &db, int32 version) TD_WARN_UNUSED_RESULT; + +class FileDbInterface; +std::shared_ptr<FileDbInterface> create_file_db(std::shared_ptr<SqliteConnectionSafe> connection, + int32 scheduler_id = -1) TD_WARN_UNUSED_RESULT; + +using FileDbId = uint64; + +class FileDbInterface { + public: + using Id = FileDbId; + FileDbInterface() = default; + FileDbInterface(const FileDbInterface &) = delete; + FileDbInterface &operator=(const FileDbInterface &) = delete; + virtual ~FileDbInterface() = default; + + // non thread safe + virtual Id create_pmc_id() = 0; + + // thread safe + virtual void close(Promise<> promise) = 0; + template <class LocationT> + void get_file_data(const LocationT &location, Promise<FileData> promise) { + get_file_data(as_key(location), std::move(promise)); + } + + template <class LocationT> + Result<FileData> get_file_data_sync(const LocationT &location) { + auto res = get_file_data_sync_impl(as_key(location)); + if (res.is_ok()) { + LOG(DEBUG) << "GET " << location << " " << res.ok(); + } else { + LOG(DEBUG) << "GET " << location << " " << res.error(); + } + return res; + } + + virtual void clear_file_data(Id id, const FileData &file_data) = 0; + virtual void set_file_data(Id id, const FileData &file_data, bool new_remote, bool new_local, bool new_generate) = 0; + virtual void set_file_data_ref(Id id, Id new_id) = 0; + + // For FileStatsWorker. TODO: remove it + virtual SqliteKeyValue &pmc() = 0; + + private: + virtual void get_file_data_impl(string key, Promise<FileData> promise) = 0; + virtual Result<FileData> get_file_data_sync_impl(string key) = 0; +}; +; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileDownloader.cpp b/libs/tdlib/td/td/telegram/files/FileDownloader.cpp new file mode 100644 index 0000000000..29180dd701 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileDownloader.cpp @@ -0,0 +1,461 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileDownloader.h" + +#include "td/telegram/telegram_api.h" + +#include "td/telegram/files/FileLoaderUtils.h" +#include "td/telegram/Global.h" +#include "td/telegram/UniqueId.h" + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" + +#include <tuple> + +namespace td { + +FileDownloader::FileDownloader(const FullRemoteFileLocation &remote, const LocalFileLocation &local, int64 size, + string name, const FileEncryptionKey &encryption_key, bool is_small, bool search_file, + std::unique_ptr<Callback> callback) + : remote_(remote) + , local_(local) + , size_(size) + , name_(std::move(name)) + , encryption_key_(encryption_key) + , callback_(std::move(callback)) + , is_small_(is_small) + , search_file_(search_file) { + if (!encryption_key.empty()) { + set_ordered_flag(true); + } +} + +Result<FileLoader::FileInfo> FileDownloader::init() { + SCOPE_EXIT { + try_release_fd(); + }; + if (local_.type() == LocalFileLocation::Type::Full) { + return Status::Error("File is already downloaded"); + } + int ready_part_count = 0; + int32 part_size = 0; + if (local_.type() == LocalFileLocation::Type::Partial) { + const auto &partial = local_.partial(); + path_ = partial.path_; + auto result_fd = FileFd::open(path_, FileFd::Write | FileFd::Read); + // TODO: check timestamps.. + if (result_fd.is_ok()) { + if (!encryption_key_.empty()) { + CHECK(partial.iv_.size() == 32) << partial.iv_.size(); + encryption_key_.mutable_iv() = as<UInt256>(partial.iv_.data()); + next_part_ = partial.ready_part_count_; + } + fd_ = result_fd.move_as_ok(); + part_size = partial.part_size_; + ready_part_count = partial.ready_part_count_; + } + } + if (search_file_ && fd_.empty() && size_ > 0 && size_ < 1000 * (1 << 20) && encryption_key_.empty() && + !remote_.is_web()) { + [&] { + TRY_RESULT(path, search_file(get_files_dir(remote_.file_type_), name_, size_)); + TRY_RESULT(fd, FileFd::open(path, FileFd::Read)); + LOG(INFO) << "Check hash of local file " << path; + path_ = std::move(path); + fd_ = std::move(fd); + need_check_ = true; + only_check_ = true; + part_size = 32 * (1 << 10); + ready_part_count = narrow_cast<int>((size_ + part_size - 1) / part_size); + return Status::OK(); + }(); + } + + std::vector<int> parts(ready_part_count); + for (int i = 0; i < ready_part_count; i++) { + parts[i] = i; + } + + FileInfo res; + res.size = size_; + res.is_size_final = true; + res.part_size = part_size; + res.ready_parts = std::move(parts); + res.use_part_count_limit = false; + res.only_check = only_check_; + res.need_delay = !is_small_ && (remote_.file_type_ == FileType::VideoNote || + remote_.file_type_ == FileType::VoiceNote || remote_.file_type_ == FileType::Audio || + remote_.file_type_ == FileType::Video || remote_.file_type_ == FileType::Animation || + (remote_.file_type_ == FileType::Encrypted && size_ > (1 << 20))); + return res; +} +Status FileDownloader::on_ok(int64 size) { + auto dir = get_files_dir(remote_.file_type_); + + std::string path; + if (only_check_) { + path = path_; + } else { + TRY_RESULT(perm_path, create_from_temp(path_, dir, name_)); + path = std::move(perm_path); + } + fd_.close(); + callback_->on_ok(FullLocalFileLocation(remote_.file_type_, std::move(path), 0), size); + return Status::OK(); +} +void FileDownloader::on_error(Status status) { + fd_.close(); + callback_->on_error(std::move(status)); +} + +Result<bool> FileDownloader::should_restart_part(Part part, NetQueryPtr &net_query) { + // Check if we should use CDN or reupload file to CDN + + if (net_query->is_error()) { + if (net_query->error().message() == "FILE_TOKEN_INVALID") { + use_cdn_ = false; + return true; + } + if (net_query->error().message() == "REQUEST_TOKEN_INVALID") { + return true; + } + return false; + } + + switch (narrow_cast<QueryType>(UniqueId::extract_key(net_query->id()))) { + case QueryType::Default: { + if (net_query->ok_tl_constructor() == telegram_api::upload_fileCdnRedirect::ID) { + LOG(DEBUG) << part.id << " got REDIRECT"; + TRY_RESULT(file_base, fetch_result<telegram_api::upload_getFile>(net_query->ok())); + CHECK(file_base->get_id() == telegram_api::upload_fileCdnRedirect::ID); + auto file = move_tl_object_as<telegram_api::upload_fileCdnRedirect>(file_base); + + auto new_cdn_file_token = file->file_token_.as_slice(); + if (cdn_file_token_ == new_cdn_file_token) { + return true; + } + + use_cdn_ = true; + need_check_ = true; + cdn_file_token_generation_++; + cdn_file_token_ = new_cdn_file_token.str(); + cdn_dc_id_ = DcId::external(file->dc_id_); + cdn_encryption_key_ = file->encryption_key_.as_slice().str(); + cdn_encryption_iv_ = file->encryption_iv_.as_slice().str(); + add_hash_info(file->file_hashes_); + if (cdn_encryption_iv_.size() != 16 || cdn_encryption_key_.size() != 32) { + return Status::Error("Wrong ctr key or iv size"); + } + + return true; + } + return false; + } + case QueryType::ReuploadCDN: { + TRY_RESULT(file_hashes, fetch_result<telegram_api::upload_reuploadCdnFile>(net_query->ok())); + add_hash_info(file_hashes); + LOG(DEBUG) << part.id << " got REUPLOAD_OK"; + return true; + } + case QueryType::CDN: { + if (net_query->ok_tl_constructor() == telegram_api::upload_cdnFileReuploadNeeded::ID) { + LOG(DEBUG) << part.id << " got REUPLOAD"; + TRY_RESULT(file_base, fetch_result<telegram_api::upload_getCdnFile>(net_query->ok())); + CHECK(file_base->get_id() == telegram_api::upload_cdnFileReuploadNeeded::ID); + auto file = move_tl_object_as<telegram_api::upload_cdnFileReuploadNeeded>(file_base); + cdn_part_reupload_token_[part.id] = file->request_token_.as_slice().str(); + return true; + } + auto it = cdn_part_file_token_generation_.find(part.id); + CHECK(it != cdn_part_file_token_generation_.end()); + if (it->second != cdn_file_token_generation_) { + LOG(DEBUG) << part.id << " got part with old file_token"; + return true; + } + return false; + } + default: + UNREACHABLE(); + } + + return false; +} +Result<std::pair<NetQueryPtr, bool>> FileDownloader::start_part(Part part, int32 part_count) { + if (!encryption_key_.empty()) { + part.size = (part.size + 15) & ~15; // fix for last part + } + // auto size = part.size; + //// sometimes we can ask more than server has, just to check size + // if (size < get_part_size()) { + // size = min(size + 16, get_part_size()); + // LOG(INFO) << "Ask " << size << " instead of " << part.size; + //} + auto size = get_part_size(); + CHECK(part.size <= size); + + callback_->on_start_download(); + + NetQueryPtr net_query; + if (!use_cdn_) { + net_query = G()->net_query_creator().create( + UniqueId::next(UniqueId::Type::Default, static_cast<uint8>(QueryType::Default)), + remote_.is_web() + ? create_storer(telegram_api::upload_getWebFile(remote_.as_input_web_file_location(), + static_cast<int32>(part.offset), static_cast<int32>(size))) + : create_storer(telegram_api::upload_getFile(remote_.as_input_file_location(), + static_cast<int32>(part.offset), static_cast<int32>(size))), + remote_.get_dc_id(), is_small_ ? NetQuery::Type::DownloadSmall : NetQuery::Type::Download); + } else { + if (remote_.is_web()) { + return Status::Error("Can't download web file from CDN"); + } + auto it = cdn_part_reupload_token_.find(part.id); + if (it == cdn_part_reupload_token_.end()) { + auto query = telegram_api::upload_getCdnFile(BufferSlice(cdn_file_token_), static_cast<int32>(part.offset), + static_cast<int32>(size)); + cdn_part_file_token_generation_[part.id] = cdn_file_token_generation_; + LOG(DEBUG) << part.id << " " << to_string(query); + net_query = G()->net_query_creator().create( + UniqueId::next(UniqueId::Type::Default, static_cast<uint8>(QueryType::CDN)), create_storer(query), cdn_dc_id_, + is_small_ ? NetQuery::Type::DownloadSmall : NetQuery::Type::Download, NetQuery::AuthFlag::Off); + } else { + auto query = telegram_api::upload_reuploadCdnFile(BufferSlice(cdn_file_token_), BufferSlice(it->second)); + LOG(DEBUG) << part.id << " " << to_string(query); + net_query = G()->net_query_creator().create( + UniqueId::next(UniqueId::Type::Default, static_cast<uint8>(QueryType::ReuploadCDN)), create_storer(query), + remote_.get_dc_id(), is_small_ ? NetQuery::Type::DownloadSmall : NetQuery::Type::Download, + NetQuery::AuthFlag::On); + cdn_part_reupload_token_.erase(it); + } + } + net_query->file_type_ = narrow_cast<int32>(remote_.file_type_); + return std::make_pair(std::move(net_query), false); +} + +Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) { + if (net_query->is_error()) { + return std::move(net_query->error()); + } + + BufferSlice bytes; + bool need_cdn_decrypt = false; + auto query_type = narrow_cast<QueryType>(UniqueId::extract_key(net_query->id())); + switch (query_type) { + case QueryType::Default: { + if (remote_.is_web()) { + TRY_RESULT(file, fetch_result<telegram_api::upload_getWebFile>(net_query->ok())); + bytes = std::move(file->bytes_); + } else { + TRY_RESULT(file_base, fetch_result<telegram_api::upload_getFile>(net_query->ok())); + CHECK(file_base->get_id() == telegram_api::upload_file::ID); + auto file = move_tl_object_as<telegram_api::upload_file>(file_base); + LOG(DEBUG) << part.id << " upload_getFile result"; + bytes = std::move(file->bytes_); + } + break; + } + case QueryType::CDN: { + TRY_RESULT(file_base, fetch_result<telegram_api::upload_getCdnFile>(net_query->ok())); + CHECK(file_base->get_id() == telegram_api::upload_cdnFile::ID); + auto file = move_tl_object_as<telegram_api::upload_cdnFile>(file_base); + LOG(DEBUG) << part.id << " upload_getCdnFile result"; + bytes = std::move(file->bytes_); + need_cdn_decrypt = true; + break; + } + default: + UNREACHABLE(); + } + + auto padded_size = part.size; + if (!encryption_key_.empty()) { + padded_size = (part.size + 15) & ~15; + } + LOG(INFO) << "Got " << bytes.size() << " padded_size=" << padded_size; + if (bytes.size() > padded_size) { + return Status::Error("Part size is more than requested"); + } + if (bytes.empty()) { + return 0; + } + + // Encryption + if (need_cdn_decrypt) { + auto iv = as<UInt128>(cdn_encryption_iv_.c_str()); + CHECK(part.offset % 16 == 0); + auto offset = narrow_cast<uint32>(part.offset / 16); + offset = + ((offset & 0xff) << 24) | ((offset & 0xff00) << 8) | ((offset & 0xff0000) >> 8) | ((offset & 0xff000000) >> 24); + as<uint32>(iv.raw + 12) = offset; + auto key = as<UInt256>(cdn_encryption_key_.c_str()); + + AesCtrState ctr_state; + ctr_state.init(key, iv); + ctr_state.decrypt(bytes.as_slice(), bytes.as_slice()); + } + if (!encryption_key_.empty()) { + CHECK(next_part_ == part.id) << tag("expected part.id", next_part_) << "!=" << tag("part.id", part.id); + CHECK(!next_part_stop_); + next_part_++; + if (part.size % 16 != 0) { + next_part_stop_ = true; + } + aes_ige_decrypt(encryption_key_.key(), &encryption_key_.mutable_iv(), bytes.as_slice(), bytes.as_slice()); + } + + auto slice = bytes.as_slice().truncate(part.size); + TRY_STATUS(acquire_fd()); + TRY_RESULT(written, fd_.pwrite(slice, part.offset)); + // may write less than part.size, when size of downloadable file is unknown + if (written != slice.size()) { + return Status::Error("Failed to save file part to the file"); + } + return written; +} +void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, bool is_ready, + int64 ready_size) { + if (is_ready) { + // do not send partial location. will lead to wrong local_size + return; + } + if (ready_size == 0 || path_.empty()) { + return; + } + if (encryption_key_.empty()) { + callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, path_, part_size, ready_part_count, ""}, + ready_size); + } else { + UInt256 iv; + if (ready_part_count == next_part_) { + iv = encryption_key_.mutable_iv(); + } else { + LOG(FATAL) << tag("ready_part_count", ready_part_count) << tag("next_part", next_part_); + } + callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, path_, part_size, ready_part_count, + Slice(iv.raw, sizeof(iv)).str()}, + ready_size); + } +} + +FileLoader::Callback *FileDownloader::get_callback() { + return static_cast<FileLoader::Callback *>(callback_.get()); +} + +Status FileDownloader::process_check_query(NetQueryPtr net_query) { + has_hash_query_ = false; + TRY_RESULT(file_hashes, fetch_result<telegram_api::upload_getCdnFileHashes>(std::move(net_query))); + add_hash_info(file_hashes); + return Status::OK(); +} +Result<FileLoader::CheckInfo> FileDownloader::check_loop(int64 checked_prefix_size, int64 ready_prefix_size, + bool is_ready) { + if (!need_check_) { + return CheckInfo{}; + } + SCOPE_EXIT { + try_release_fd(); + }; + CheckInfo info; + while (checked_prefix_size < ready_prefix_size) { + //LOG(ERROR) << "NEED TO CHECK: " << checked_prefix_size << "->" << ready_prefix_size - checked_prefix_size; + HashInfo search_info; + search_info.offset = checked_prefix_size; + auto it = hash_info_.upper_bound(search_info); + if (it != hash_info_.begin()) { + it--; + } + if (it != hash_info_.end() && it->offset <= checked_prefix_size && + it->offset + narrow_cast<int64>(it->size) > checked_prefix_size) { + int64 begin_offset = it->offset; + int64 end_offset = it->offset + narrow_cast<int64>(it->size); + if (ready_prefix_size < end_offset) { + if (!is_ready) { + break; + } + end_offset = ready_prefix_size; + } + size_t size = narrow_cast<size_t>(end_offset - begin_offset); + auto slice = BufferSlice(size); + TRY_STATUS(acquire_fd()); + TRY_RESULT(read_size, fd_.pread(slice.as_slice(), begin_offset)); + if (size != read_size) { + return Status::Error("Failed to read file to check hash"); + } + string hash(32, ' '); + sha256(slice.as_slice(), hash); + + if (hash != it->hash) { + if (only_check_) { + return Status::Error("FILE_DOWNLOAD_RESTART"); + } + return Status::Error("Hash mismatch"); + } + + checked_prefix_size = end_offset; + info.changed = true; + continue; + } + if (!has_hash_query_) { + has_hash_query_ = true; + auto query = + telegram_api::upload_getFileHashes(remote_.as_input_file_location(), narrow_cast<int32>(checked_prefix_size)); + auto net_query = G()->net_query_creator().create( + create_storer(query), remote_.get_dc_id(), + is_small_ ? NetQuery::Type::DownloadSmall : NetQuery::Type::Download, NetQuery::AuthFlag::On); + info.queries.push_back(std::move(net_query)); + break; + } + // Should fail? + break; + } + info.need_check = need_check_; + info.checked_prefix_size = checked_prefix_size; + return std::move(info); +} +void FileDownloader::add_hash_info(const std::vector<telegram_api::object_ptr<telegram_api::fileHash>> &hashes) { + for (auto &hash : hashes) { + //LOG(ERROR) << "ADD HASH " << hash->offset_ << "->" << hash->limit_; + HashInfo hash_info; + hash_info.size = hash->limit_; + hash_info.offset = hash->offset_; + hash_info.hash = hash->hash_.as_slice().str(); + hash_info_.insert(std::move(hash_info)); + } +} + +void FileDownloader::keep_fd_flag(bool keep_fd) { + keep_fd_ = keep_fd; + try_release_fd(); +} + +void FileDownloader::try_release_fd() { + if (!keep_fd_ && !fd_.empty()) { + fd_.close(); + } +} + +Status FileDownloader::acquire_fd() { + if (fd_.empty()) { + if (path_.empty()) { + TRY_RESULT(file_path, open_temp_file(remote_.file_type_)); + std::tie(fd_, path_) = std::move(file_path); + } else { + TRY_RESULT(fd, FileFd::open(path_, (only_check_ ? 0 : FileFd::Write) | FileFd::Read)); + fd_ = std::move(fd); + } + } + return Status::OK(); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileDownloader.h b/libs/tdlib/td/td/telegram/files/FileDownloader.h new file mode 100644 index 0000000000..6a45ca567b --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileDownloader.h @@ -0,0 +1,100 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/telegram_api.h" + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileLoader.h" +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/net/NetQuery.h" + +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Status.h" + +#include <map> +#include <set> +#include <utility> + +namespace td { +class FileDownloader : public FileLoader { + public: + class Callback : public FileLoader::Callback { + public: + virtual void on_start_download() = 0; + virtual void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size) = 0; + virtual void on_ok(const FullLocalFileLocation &full_local, int64 size) = 0; + virtual void on_error(Status status) = 0; + }; + + FileDownloader(const FullRemoteFileLocation &remote, const LocalFileLocation &local, int64 size, string name, + const FileEncryptionKey &encryption_key, bool is_small, bool search_file, + std::unique_ptr<Callback> callback); + + // Should just implement all parent pure virtual methods. + // Must not call any of them... + private: + enum class QueryType : uint8 { Default = 1, CDN, ReuploadCDN }; + ResourceState resource_state_; + FullRemoteFileLocation remote_; + LocalFileLocation local_; + int64 size_; + string name_; + FileEncryptionKey encryption_key_; + std::unique_ptr<Callback> callback_; + bool only_check_{false}; + + string path_; + FileFd fd_; + + int32 next_part_ = 0; + bool next_part_stop_ = false; + bool is_small_; + bool search_file_{false}; + + bool use_cdn_ = false; + DcId cdn_dc_id_; + string cdn_encryption_key_; + string cdn_encryption_iv_; + string cdn_file_token_; + int32 cdn_file_token_generation_{0}; + std::map<int32, string> cdn_part_reupload_token_; + std::map<int32, int32> cdn_part_file_token_generation_; + + bool need_check_{false}; + struct HashInfo { + int64 offset; + size_t size; + string hash; + bool operator<(const HashInfo &other) const { + return offset < other.offset; + } + }; + std::set<HashInfo> hash_info_; + bool has_hash_query_ = false; + + Result<FileInfo> init() override TD_WARN_UNUSED_RESULT; + Status on_ok(int64 size) override TD_WARN_UNUSED_RESULT; + void on_error(Status status) override; + Result<bool> should_restart_part(Part part, NetQueryPtr &net_query) override TD_WARN_UNUSED_RESULT; + Result<std::pair<NetQueryPtr, bool>> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT; + Result<size_t> process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT; + void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, bool is_ready, int64 ready_size) override; + FileLoader::Callback *get_callback() override; + Status process_check_query(NetQueryPtr net_query) override; + Result<CheckInfo> check_loop(int64 checked_prefix_size, int64 ready_prefix_size, bool is_ready) override; + void add_hash_info(const std::vector<telegram_api::object_ptr<telegram_api::fileHash>> &hashes); + + bool keep_fd_ = false; + void keep_fd_flag(bool keep_fd) override; + void try_release_fd(); + Status acquire_fd() TD_WARN_UNUSED_RESULT; +}; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileFromBytes.cpp b/libs/tdlib/td/td/telegram/files/FileFromBytes.cpp new file mode 100644 index 0000000000..939803d8a8 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileFromBytes.cpp @@ -0,0 +1,50 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileFromBytes.h" + +#include "td/telegram/files/FileLoaderUtils.h" +#include "td/telegram/Global.h" + +#include "td/utils/common.h" +#include "td/utils/misc.h" + +#include <tuple> + +namespace td { + +FileFromBytes::FileFromBytes(FileType type, BufferSlice bytes, string name, std::unique_ptr<Callback> callback) + : type_(type), bytes_(std::move(bytes)), name_(std::move(name)), callback_(std::move(callback)) { +} + +void FileFromBytes::wakeup() { + auto r_fd_path = open_temp_file(type_); + if (r_fd_path.is_error()) { + return callback_->on_error(r_fd_path.move_as_error()); + } + FileFd fd; + string path; + std::tie(fd, path) = r_fd_path.move_as_ok(); + + auto r_size = fd.write(bytes_.as_slice()); + if (r_size.is_error()) { + return callback_->on_error(r_size.move_as_error()); + } + fd.close(); + auto size = r_size.ok(); + if (size != bytes_.size()) { + return callback_->on_error(Status::Error("Failed to write bytes to the file")); + } + + auto dir = get_files_dir(type_); + auto r_perm_path = create_from_temp(path, dir, name_); + if (r_perm_path.is_error()) { + return callback_->on_error(r_perm_path.move_as_error()); + } + callback_->on_ok(FullLocalFileLocation(type_, r_perm_path.move_as_ok(), 0), narrow_cast<int64>(bytes_.size())); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileFromBytes.h b/libs/tdlib/td/td/telegram/files/FileFromBytes.h new file mode 100644 index 0000000000..23f4b2ff4f --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileFromBytes.h @@ -0,0 +1,54 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" + +#include "td/telegram/files/FileLoader.h" +#include "td/telegram/files/FileLocation.h" + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Status.h" + +namespace td { +class FileFromBytes : public FileLoaderActor { + public: + class Callback { + public: + Callback() = default; + Callback(const Callback &) = delete; + Callback &operator=(const Callback &) = delete; + virtual ~Callback() = default; + virtual void on_ok(const FullLocalFileLocation &full_local, int64 size) = 0; + virtual void on_error(Status status) = 0; + }; + + FileFromBytes(FileType type, BufferSlice bytes, string name, std::unique_ptr<Callback> callback); + + // Should just implement all parent pure virtual methods. + // Must not call any of them... + private: + FileType type_; + BufferSlice bytes_; + string name_; + + std::unique_ptr<Callback> callback_; + + FileFd fd_; + string path_; + + void wakeup() override; + void set_resource_manager(ActorShared<ResourceManager>) override { + } + void update_priority(int8 priority) override { + } + void update_resources(const ResourceState &other) override { + } +}; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileGcParameters.cpp b/libs/tdlib/td/td/telegram/files/FileGcParameters.cpp new file mode 100644 index 0000000000..7b949d73a2 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileGcParameters.cpp @@ -0,0 +1,32 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileGcParameters.h" + +#include "td/telegram/ConfigShared.h" +#include "td/telegram/Global.h" + +namespace td { +FileGcParameters::FileGcParameters(int64 size, int32 ttl, int32 count, int32 immunity_delay, + vector<FileType> file_types, vector<DialogId> owner_dialog_ids, + vector<DialogId> exclude_owner_dialog_ids, int32 dialog_limit) + : file_types(std::move(file_types)) + , owner_dialog_ids(std::move(owner_dialog_ids)) + , exclude_owner_dialog_ids(std::move(exclude_owner_dialog_ids)) + , dialog_limit(dialog_limit) { + auto &config = G()->shared_config(); + this->max_files_size = + size >= 0 ? size : static_cast<int64>(config.get_option_integer("storage_max_files_size", 100 << 10)) << 10; + + this->max_time_from_last_access = + ttl >= 0 ? ttl : config.get_option_integer("storage_max_time_from_last_access", 60 * 60 * 23); + + this->max_file_count = count >= 0 ? count : config.get_option_integer("storage_max_file_count", 40000); + + this->immunity_delay = + immunity_delay >= 0 ? immunity_delay : config.get_option_integer("storage_immunity_delay", 60 * 60); +} +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileGcParameters.h b/libs/tdlib/td/td/telegram/files/FileGcParameters.h new file mode 100644 index 0000000000..a0c3c35524 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileGcParameters.h @@ -0,0 +1,34 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/DialogId.h" +#include "td/telegram/files/FileLocation.h" + +#include "td/utils/common.h" + +namespace td { + +struct FileGcParameters { + FileGcParameters() : FileGcParameters(-1, -1, -1, -1, {}, {}, {}, 0) { + } + FileGcParameters(int64 size, int32 ttl, int32 count, int32 immunity_delay, vector<FileType> file_types, + vector<DialogId> owner_dialog_ids, vector<DialogId> exclude_owner_dialog_ids, int32 dialog_limit); + + int64 max_files_size; + uint32 max_time_from_last_access; + uint32 max_file_count; + uint32 immunity_delay; + + vector<FileType> file_types; + vector<DialogId> owner_dialog_ids; + vector<DialogId> exclude_owner_dialog_ids; + + int32 dialog_limit; +}; + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileGcWorker.cpp b/libs/tdlib/td/td/telegram/files/FileGcWorker.cpp new file mode 100644 index 0000000000..35dce38e9c --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileGcWorker.cpp @@ -0,0 +1,174 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileGcWorker.h" + +#include "td/telegram/files/FileManager.h" +#include "td/telegram/Global.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/Clocks.h" +#include "td/utils/port/path.h" +#include "td/utils/Time.h" + +#include <algorithm> +#include <array> + +namespace td { +void FileGcWorker::do_remove_file(const FullFileInfo &info) { + // LOG(WARNING) << "Gc remove file: " << tag("path", file) << tag("mtime", stat.mtime_nsec_ / 1000000000) + // << tag("atime", stat.atime_nsec_ / 1000000000); + // TODO: remove file from db too. + auto status = unlink(info.path); + LOG_IF(WARNING, status.is_error()) << "Failed to unlink file during files gc: " << status; + send_closure(G()->file_manager(), &FileManager::on_file_unlink, + FullLocalFileLocation(info.file_type, info.path, info.mtime_nsec)); +} + +void FileGcWorker::run_gc(const FileGcParameters ¶meters, std::vector<FullFileInfo> files, + Promise<FileStats> promise) { + auto begin_time = Time::now(); + LOG(INFO) << "Start files gc"; + // quite stupid implementations + // needs a lot of memory + // may write something more clever, but i will need at least 2 passes over the files + // TODO update atime for all files in android (?) + + std::array<bool, file_type_size> immune_types{{false}}; + + if (G()->parameters().use_file_db) { + // immune by default + immune_types[narrow_cast<size_t>(FileType::Sticker)] = true; + immune_types[narrow_cast<size_t>(FileType::ProfilePhoto)] = true; + immune_types[narrow_cast<size_t>(FileType::Thumbnail)] = true; + immune_types[narrow_cast<size_t>(FileType::Wallpaper)] = true; + } + + if (!parameters.file_types.empty()) { + std::fill(immune_types.begin(), immune_types.end(), true); + for (auto file_type : parameters.file_types) { + immune_types[narrow_cast<size_t>(file_type)] = false; + } + } + + if (G()->parameters().use_file_db) { + immune_types[narrow_cast<size_t>(FileType::EncryptedThumbnail)] = true; + } + + auto file_cnt = files.size(); + int32 type_immunity_ignored_cnt = 0; + int32 time_immunity_ignored_cnt = 0; + int32 exclude_owner_dialog_id_ignored_cnt = 0; + int32 owner_dialog_id_ignored_cnt = 0; + int32 remove_by_atime_cnt = 0; + int32 remove_by_count_cnt = 0; + int32 remove_by_size_cnt = 0; + int64 total_removed_size = 0; + int64 total_size = 0; + for (auto &info : files) { + if (info.atime_nsec < info.mtime_nsec) { + info.atime_nsec = info.mtime_nsec; + } + total_size += info.size; + } + + FileStats new_stats; + new_stats.split_by_owner_dialog_id = parameters.dialog_limit != 0; + + // Remove all files with atime > now - max_time_from_last_access + double now = Clocks::system(); + files.erase( + std::remove_if( + files.begin(), files.end(), + [&](const FullFileInfo &info) { + if (immune_types[narrow_cast<size_t>(info.file_type)]) { + type_immunity_ignored_cnt++; + new_stats.add(FullFileInfo(info)); + return true; + } + if (std::find(parameters.exclude_owner_dialog_ids.begin(), parameters.exclude_owner_dialog_ids.end(), + info.owner_dialog_id) != parameters.exclude_owner_dialog_ids.end()) { + exclude_owner_dialog_id_ignored_cnt++; + new_stats.add(FullFileInfo(info)); + return true; + } + if (!parameters.owner_dialog_ids.empty() && + std::find(parameters.owner_dialog_ids.begin(), parameters.owner_dialog_ids.end(), + info.owner_dialog_id) == parameters.owner_dialog_ids.end()) { + owner_dialog_id_ignored_cnt++; + new_stats.add(FullFileInfo(info)); + return true; + } + if (static_cast<double>(info.mtime_nsec / 1000000000) > now - parameters.immunity_delay) { + // new files are immune to gc. + time_immunity_ignored_cnt++; + new_stats.add(FullFileInfo(info)); + return true; + } + + if (static_cast<double>(info.atime_nsec / 1000000000) < now - parameters.max_time_from_last_access) { + do_remove_file(info); + total_removed_size += info.size; + remove_by_atime_cnt++; + return true; + } + return false; + }), + files.end()); + + // sort by max(atime, mtime) + std::sort(files.begin(), files.end(), [](const auto &a, const auto &b) { return a.atime_nsec < b.atime_nsec; }); + + // 1. Total memory must be less than max_memory + // 2. Total file count must be less than MAX_FILE_COUNT + size_t remove_count = 0; + if (files.size() > parameters.max_file_count) { + remove_count = files.size() - parameters.max_file_count; + } + int64 remove_size = -parameters.max_files_size; + for (auto &file : files) { + remove_size += file.size; + } + + size_t pos = 0; + while (pos < files.size() && (remove_count > 0 || remove_size > 0)) { + if (remove_count > 0) { + remove_by_count_cnt++; + } else { + remove_by_size_cnt++; + } + + if (remove_count > 0) { + remove_count--; + } + remove_size -= files[pos].size; + + total_removed_size += files[pos].size; + do_remove_file(files[pos]); + pos++; + } + + while (pos < files.size()) { + new_stats.add(std::move(files[pos])); + pos++; + } + + auto end_time = Time::now(); + + LOG(INFO) << "Finish files gc: " << tag("time", end_time - begin_time) << tag("total", file_cnt) + << tag("removed", remove_by_atime_cnt + remove_by_count_cnt + remove_by_size_cnt) + << tag("total_size", format::as_size(total_size)) + << tag("total_removed_size", format::as_size(total_removed_size)) << tag("by_atime", remove_by_atime_cnt) + << tag("by_count", remove_by_count_cnt) << tag("by_size", remove_by_size_cnt) + << tag("type_immunity", type_immunity_ignored_cnt) << tag("time_immunity", time_immunity_ignored_cnt) + << tag("owner_dialog_id_immunity", owner_dialog_id_ignored_cnt) + << tag("exclude_owner_dialog_id_immunity", exclude_owner_dialog_id_ignored_cnt); + + promise.set_value(std::move(new_stats)); +} +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileGcWorker.h b/libs/tdlib/td/td/telegram/files/FileGcWorker.h new file mode 100644 index 0000000000..7e83d9fa98 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileGcWorker.h @@ -0,0 +1,28 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileGcParameters.h" +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/files/FileStats.h" + +namespace td { +class FileGcWorker : public Actor { + public: + explicit FileGcWorker(ActorShared<> parent) : parent_(std::move(parent)) { + } + void run_gc(const FileGcParameters ¶meters, std::vector<FullFileInfo> files, Promise<FileStats> promise); + + private: + ActorShared<> parent_; + void do_remove_file(const FullFileInfo &info); +}; + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileGenerateManager.cpp b/libs/tdlib/td/td/telegram/files/FileGenerateManager.cpp new file mode 100644 index 0000000000..22472a248f --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileGenerateManager.cpp @@ -0,0 +1,285 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileGenerateManager.h" + +#include "td/telegram/td_api.h" + +#include "td/telegram/files/FileId.h" +#include "td/telegram/files/FileLoaderUtils.h" +#include "td/telegram/files/FileManager.h" +#include "td/telegram/Global.h" +#include "td/telegram/Td.h" + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/path.h" +#include "td/utils/Slice.h" + +#include <utility> + +namespace td { + +class FileGenerateActor : public Actor { + public: + FileGenerateActor() = default; + FileGenerateActor(const FileGenerateActor &) = delete; + FileGenerateActor &operator=(const FileGenerateActor &) = delete; + FileGenerateActor(FileGenerateActor &&) = delete; + FileGenerateActor &operator=(FileGenerateActor &&) = delete; + ~FileGenerateActor() override = default; + virtual void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) = 0; + virtual void file_generate_finish(Status status, Promise<> promise) = 0; +}; + +class FileDownloadGenerateActor : public FileGenerateActor { + public: + FileDownloadGenerateActor(FileType file_type, FileId file_id, std::unique_ptr<FileGenerateCallback> callback, + ActorShared<> parent) + : file_type_(file_type), file_id_(file_id), callback_(std::move(callback)), parent_(std::move(parent)) { + } + void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) override { + UNREACHABLE(); + } + void file_generate_finish(Status status, Promise<> promise) override { + UNREACHABLE(); + } + + private: + FileType file_type_; + FileId file_id_; + std::unique_ptr<FileGenerateCallback> callback_; + ActorShared<> parent_; + + void start_up() override { + LOG(INFO) << "DOWNLOAD " << file_id_; + class Callback : public FileManager::DownloadCallback { + public: + explicit Callback(ActorId<FileDownloadGenerateActor> parent) : parent_(std::move(parent)) { + } + + // TODO: upload during download + + void on_download_ok(FileId file_id) override { + send_closure(parent_, &FileDownloadGenerateActor::on_download_ok); + } + void on_download_error(FileId file_id, Status error) override { + send_closure(parent_, &FileDownloadGenerateActor::on_download_error, std::move(error)); + } + + private: + ActorId<FileDownloadGenerateActor> parent_; + }; + + send_closure(G()->file_manager(), &FileManager::download, file_id_, std::make_unique<Callback>(actor_id(this)), 1); + } + void hangup() override { + send_closure(G()->file_manager(), &FileManager::download, file_id_, nullptr, 0); + stop(); + } + + void on_download_ok() { + send_lambda(G()->file_manager(), [file_type = file_type_, file_id = file_id_, callback = std::move(callback_)] { + auto file_view = G()->td().get_actor_unsafe()->file_manager_->get_file_view(file_id); + if (file_view.has_local_location()) { + auto location = file_view.local_location(); + location.file_type_ = file_type; + callback->on_ok(location); + } else { + LOG(ERROR) << "Expected to have local location"; + callback->on_error(Status::Error("Unknown")); + } + }); + stop(); + } + void on_download_error(Status error) { + callback_->on_error(std::move(error)); + stop(); + } +}; + +class FileExternalGenerateActor : public FileGenerateActor { + public: + FileExternalGenerateActor(uint64 query_id, const FullGenerateFileLocation &generate_location, + const LocalFileLocation &local_location, string name, + std::unique_ptr<FileGenerateCallback> callback, ActorShared<> parent) + : query_id_(query_id) + , generate_location_(generate_location) + , local_(local_location) + , name_(std::move(name)) + , callback_(std::move(callback)) + , parent_(std::move(parent)) { + } + + void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) override { + check_status(do_file_generate_progress(expected_size, local_prefix_size), std::move(promise)); + } + void file_generate_finish(Status status, Promise<> promise) override { + check_status(do_file_generate_finish(std::move(status)), std::move(promise)); + } + + private: + uint64 query_id_; + FullGenerateFileLocation generate_location_; + LocalFileLocation local_; + string name_; + string path_; + std::unique_ptr<FileGenerateCallback> callback_; + ActorShared<> parent_; + + void start_up() override { + if (local_.type() == LocalFileLocation::Type::Full) { + callback_->on_ok(local_.full()); + callback_.reset(); + return stop(); + } + + if (local_.type() == LocalFileLocation::Type::Partial) { + const auto &partial = local_.partial(); + path_ = partial.path_; + LOG(INFO) << "Unlink partially generated file at " << path_; + unlink(path_).ignore(); + } else { + auto r_file_path = open_temp_file(generate_location_.file_type_); + if (r_file_path.is_error()) { + return check_status(r_file_path.move_as_error()); + } + auto file_path = r_file_path.move_as_ok(); + file_path.first.close(); + path_ = file_path.second; + } + send_closure( + G()->td(), &Td::send_update, + make_tl_object<td_api::updateFileGenerationStart>( + static_cast<int64>(query_id_), generate_location_.original_path_, path_, generate_location_.conversion_)); + } + void hangup() override { + check_status(Status::Error(1, "Cancelled")); + } + + Status do_file_generate_progress(int32 expected_size, int32 local_prefix_size) { + if (local_prefix_size < 0) { + return Status::Error(1, "Invalid local prefix size"); + } + callback_->on_partial_generate( + PartialLocalFileLocation{generate_location_.file_type_, path_, 1, local_prefix_size, ""}, expected_size); + return Status::OK(); + } + + Status do_file_generate_finish(Status status) { + TRY_STATUS(std::move(status)); + + auto dir = get_files_dir(generate_location_.file_type_); + + TRY_RESULT(perm_path, create_from_temp(path_, dir, name_)); + callback_->on_ok(FullLocalFileLocation(generate_location_.file_type_, std::move(perm_path), 0)); + callback_.reset(); + stop(); + return Status::OK(); + } + + void check_status(Status status, Promise<> promise = Promise<>()) { + if (promise) { + if (status.is_ok() || status.code() == 1) { + promise.set_value(Unit()); + } else { + promise.set_error(Status::Error(400, status.message())); + } + } + + if (status.is_error()) { + LOG(INFO) << "Unlink partially generated file at " << path_ << " because of " << status; + unlink(path_).ignore(); + callback_->on_error(std::move(status)); + callback_.reset(); + stop(); + } + } + + void tear_down() override { + send_closure(G()->td(), &Td::send_update, + make_tl_object<td_api::updateFileGenerationStop>(static_cast<int64>(query_id_))); + } +}; + +FileGenerateManager::Query::~Query() = default; +FileGenerateManager::Query::Query(Query &&other) = default; +FileGenerateManager::Query &FileGenerateManager::Query::operator=(Query &&other) = default; + +void FileGenerateManager::generate_file(uint64 query_id, const FullGenerateFileLocation &generate_location, + const LocalFileLocation &local_location, string name, + std::unique_ptr<FileGenerateCallback> callback) { + CHECK(query_id != 0); + auto it_flag = query_id_to_query_.insert(std::make_pair(query_id, Query{})); + CHECK(it_flag.second) << "Query id must be unique"; + auto parent = actor_shared(this, query_id); + + Slice file_id_query = "#file_id#"; + Slice conversion = generate_location.conversion_; + + auto &query = it_flag.first->second; + if (conversion.copy().truncate(file_id_query.size()) == file_id_query) { + auto file_id = FileId(to_integer<int32>(conversion.substr(file_id_query.size())), 0); + query.worker_ = create_actor<FileDownloadGenerateActor>("FileDownloadGenerateActor", generate_location.file_type_, + file_id, std::move(callback), std::move(parent)); + } else { + query.worker_ = create_actor<FileExternalGenerateActor>("FileExternalGenerationActor", query_id, generate_location, + local_location, std::move(name), std::move(callback), + std::move(parent)); + } +} + +void FileGenerateManager::cancel(uint64 query_id) { + auto it = query_id_to_query_.find(query_id); + if (it == query_id_to_query_.end()) { + return; + } + it->second.worker_.reset(); +} + +void FileGenerateManager::external_file_generate_progress(uint64 query_id, int32 expected_size, int32 local_prefix_size, + Promise<> promise) { + auto it = query_id_to_query_.find(query_id); + if (it == query_id_to_query_.end()) { + return promise.set_error(Status::Error(400, "Unknown generation_id")); + } + send_closure(it->second.worker_, &FileGenerateActor::file_generate_progress, expected_size, local_prefix_size, + std::move(promise)); +} + +void FileGenerateManager::external_file_generate_finish(uint64 query_id, Status status, Promise<> promise) { + auto it = query_id_to_query_.find(query_id); + if (it == query_id_to_query_.end()) { + return promise.set_error(Status::Error(400, "Unknown generation_id")); + } + send_closure(it->second.worker_, &FileGenerateActor::file_generate_finish, std::move(status), std::move(promise)); +} + +void FileGenerateManager::do_cancel(uint64 query_id) { + query_id_to_query_.erase(query_id); +} + +void FileGenerateManager::hangup_shared() { + do_cancel(get_link_token()); + loop(); +} + +void FileGenerateManager::hangup() { + close_flag_ = true; + for (auto &it : query_id_to_query_) { + it.second.worker_.reset(); + } + loop(); +} + +void FileGenerateManager::loop() { + if (close_flag_ && query_id_to_query_.empty()) { + stop(); + } +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileGenerateManager.h b/libs/tdlib/td/td/telegram/files/FileGenerateManager.h new file mode 100644 index 0000000000..4acd495d72 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileGenerateManager.h @@ -0,0 +1,70 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileLocation.h" + +#include "td/utils/Status.h" + +#include <map> + +namespace td { +class FileGenerateActor; + +class FileGenerateCallback { + public: + FileGenerateCallback() = default; + FileGenerateCallback(const FileGenerateCallback &) = delete; + FileGenerateCallback &operator=(const FileGenerateCallback &) = delete; + virtual ~FileGenerateCallback() = default; + + virtual void on_partial_generate(const PartialLocalFileLocation &partial_local, int32 expected_size) = 0; + virtual void on_ok(const FullLocalFileLocation &local) = 0; + virtual void on_error(Status error) = 0; +}; + +class FileGenerateManager : public Actor { + public: + explicit FileGenerateManager(ActorShared<> parent) : parent_(std::move(parent)) { + } + + void generate_file(uint64 query_id, const FullGenerateFileLocation &generate_location, + const LocalFileLocation &local_location, string name, + std::unique_ptr<FileGenerateCallback> callback); + void cancel(uint64 query_id); + + // external updates about file generation state + void external_file_generate_progress(uint64 query_id, int32 expected_size, int32 local_prefix_size, + Promise<> promise); + void external_file_generate_finish(uint64 query_id, Status status, Promise<> promise); + + private: + struct Query { + Query() = default; + Query(const Query &other) = delete; + Query &operator=(const Query &other) = delete; + Query(Query &&other); + Query &operator=(Query &&other); + ~Query(); + + ActorOwn<FileGenerateActor> worker_; + }; + + ActorShared<> parent_; + std::map<uint64, Query> query_id_to_query_; + bool close_flag_ = false; + + void hangup() override; + void hangup_shared() override; + void loop() override; + void do_cancel(uint64 query_id); +}; + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileHashUploader.cpp b/libs/tdlib/td/td/telegram/files/FileHashUploader.cpp new file mode 100644 index 0000000000..2408e16df4 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileHashUploader.cpp @@ -0,0 +1,142 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileHashUploader.h" + +#include "td/telegram/telegram_api.h" + +#include "td/telegram/Global.h" +#include "td/telegram/net/NetQueryDispatcher.h" + +#include "td/utils/buffer.h" +#include "td/utils/crypto.h" +#include "td/utils/logging.h" +#include "td/utils/MimeType.h" +#include "td/utils/misc.h" +#include "td/utils/PathView.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Status.h" + +namespace td { +void FileHashUploader::start_up() { + auto status = init(); + if (status.is_error()) { + callback_->on_error(std::move(status)); + stop_flag_ = true; + return; + } +} +Status FileHashUploader::init() { + TRY_RESULT(fd, FileFd::open(local_.path_, FileFd::Read)); + if (fd.get_size() != size_) { + return Status::Error("size mismatch"); + } + fd_ = BufferedFd<FileFd>(std::move(fd)); + sha256_init(&sha256_state_); + + resource_state_.set_unit_size(1024); + resource_state_.update_estimated_limit(size_); + return Status::OK(); +} +void FileHashUploader::loop() { + if (stop_flag_) { + return; + } + + auto status = loop_impl(); + if (status.is_error()) { + callback_->on_error(std::move(status)); + stop_flag_ = true; + return; + } +} + +Status FileHashUploader::loop_impl() { + if (state_ == CalcSha) { + TRY_STATUS(loop_sha()); + } + if (state_ == NetRequest) { + // messages.getDocumentByHash#338e2464 sha256:bytes size:int mime_type:string = Document; + auto hash = BufferSlice(32); + sha256_final(&sha256_state_, hash.as_slice()); + auto mime_type = MimeType::from_extension(PathView(local_.path_).extension(), "image/gif"); + auto query = + telegram_api::messages_getDocumentByHash(std::move(hash), static_cast<int32>(size_), std::move(mime_type)); + LOG(INFO) << "Send getDocumentByHash request: " << to_string(query); + auto ptr = G()->net_query_creator().create(create_storer(query)); + G()->net_query_dispatcher().dispatch_with_callback(std::move(ptr), actor_shared(this)); + state_ = WaitNetResult; + } + return Status::OK(); +} + +Status FileHashUploader::loop_sha() { + auto limit = resource_state_.unused(); + if (limit == 0) { + return Status::OK(); + } + if (limit > size_left_) { + limit = size_left_; + } + resource_state_.start_use(limit); + + fd_.update_flags(Fd::Flag::Read); + TRY_RESULT(read_size, fd_.flush_read(static_cast<size_t>(limit))); + if (read_size != static_cast<size_t>(limit)) { + return Status::Error("unexpected end of file"); + } + while (true) { + auto ready = fd_.input_buffer().prepare_read(); + if (ready.empty()) { + break; + } + sha256_update(ready, &sha256_state_); + fd_.input_buffer().confirm_read(ready.size()); + } + resource_state_.stop_use(limit); + + size_left_ -= narrow_cast<int64>(read_size); + CHECK(size_left_ >= 0); + if (size_left_ == 0) { + state_ = NetRequest; + return Status::OK(); + } + return Status::OK(); +} + +void FileHashUploader::on_result(NetQueryPtr net_query) { + auto status = on_result_impl(std::move(net_query)); + if (status.is_error()) { + callback_->on_error(std::move(status)); + stop_flag_ = true; + return; + } +} + +Status FileHashUploader::on_result_impl(NetQueryPtr net_query) { + if (net_query->is_error()) { + return net_query->move_as_error(); + } + TRY_RESULT(res, fetch_result<telegram_api::messages_getDocumentByHash>(net_query->ok())); + + switch (res->get_id()) { + case telegram_api::documentEmpty::ID: + return Status::Error("Document is not found by hash"); + case telegram_api::document::ID: { + auto document = move_tl_object_as<telegram_api::document>(res); + callback_->on_ok(FullRemoteFileLocation(FileType::Document, document->id_, document->access_hash_, + DcId::internal(document->dc_id_))); + + stop_flag_ = true; + return Status::OK(); + } + default: + UNREACHABLE(); + return Status::Error("UNREACHABLE"); + } +} +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileHashUploader.h b/libs/tdlib/td/td/telegram/files/FileHashUploader.h new file mode 100644 index 0000000000..d836c521b7 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileHashUploader.h @@ -0,0 +1,80 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/files/FileLoaderActor.h" +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/files/ResourceManager.h" + +#include "td/utils/BufferedFd.h" +#include "td/utils/crypto.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Status.h" + +namespace td { + +class FileHashUploader : public FileLoaderActor { + public: + class Callback { + public: + Callback() = default; + Callback(const Callback &) = delete; + Callback &operator=(const Callback &) = delete; + virtual ~Callback() = default; + virtual void on_ok(const FullRemoteFileLocation &locatioin) = 0; + virtual void on_error(Status status) = 0; + }; + + FileHashUploader(const FullLocalFileLocation &local, int64 size, std::unique_ptr<Callback> callback) + : local_(local), size_(size), size_left_(size), callback_(std::move(callback)) { + } + + void set_resource_manager(ActorShared<ResourceManager> resource_manager) override { + resource_manager_ = std::move(resource_manager); + send_closure(resource_manager_, &ResourceManager::update_resources, resource_state_); + } + + void update_priority(int8 priority) override { + send_closure(resource_manager_, &ResourceManager::update_priority, priority); + } + void update_resources(const ResourceState &other) override { + if (stop_flag_) { + return; + } + resource_state_.update_slave(other); + loop(); + } + + private: + ResourceState resource_state_; + BufferedFd<FileFd> fd_; + + FullLocalFileLocation local_; + int64 size_; + int64 size_left_; + unique_ptr<Callback> callback_; + + ActorShared<ResourceManager> resource_manager_; + + enum { CalcSha, NetRequest, WaitNetResult } state_ = CalcSha; + bool stop_flag_ = false; + Sha256State sha256_state_; + + void start_up() override; + Status init(); + + void loop() override; + + Status loop_impl(); + + Status loop_sha(); + + void on_result(NetQueryPtr net_query) override; + + Status on_result_impl(NetQueryPtr net_query); +}; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileId.h b/libs/tdlib/td/td/telegram/files/FileId.h new file mode 100644 index 0000000000..ef68681ba0 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileId.h @@ -0,0 +1,67 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/StringBuilder.h" + +#include <functional> +#include <type_traits> + +namespace td { + +class FileId { + int32 id = 0; + int32 remote_id = 0; + + public: + FileId() = default; + + FileId(int32 file_id, int32 remote_id) : id(file_id), remote_id(remote_id) { + } + template <class T1, class T2, typename = std::enable_if_t<std::is_convertible<T1, int32>::value>, + typename = std::enable_if_t<std::is_convertible<T2, int32>::value>> + FileId(T1 file_id, T2 remote_id) = delete; + + bool empty() const { + return id <= 0; + } + bool is_valid() const { + return id > 0; + } + + int32 get() const { + return id; + } + + int32 get_remote() const { + return remote_id; + } + + bool operator<(const FileId &other) const { + return id < other.id; + } + + bool operator==(const FileId &other) const { + return id == other.id; + } + + bool operator!=(const FileId &other) const { + return id != other.id; + } +}; + +struct FileIdHash { + std::size_t operator()(FileId file_id) const { + return std::hash<int32>()(file_id.get()); + } +}; + +inline StringBuilder &operator<<(StringBuilder &string_builder, FileId file_id) { + return string_builder << file_id.get() << "(" << file_id.get_remote() << ")"; +} +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileId.hpp b/libs/tdlib/td/td/telegram/files/FileId.hpp new file mode 100644 index 0000000000..b4b5dbf429 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileId.hpp @@ -0,0 +1,26 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/files/FileId.h" + +#include "td/telegram/files/FileManager.hpp" +#include "td/telegram/Td.h" + +namespace td { + +template <class StorerT> +void store(const FileId &file_id, StorerT &storer) { + storer.context()->td().get_actor_unsafe()->file_manager_->store_file(file_id, storer); +} + +template <class ParserT> +void parse(FileId &file_id, ParserT &parser) { + file_id = parser.context()->td().get_actor_unsafe()->file_manager_->parse_file(parser); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLoadManager.cpp b/libs/tdlib/td/td/telegram/files/FileLoadManager.cpp new file mode 100644 index 0000000000..85ef83595d --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLoadManager.cpp @@ -0,0 +1,273 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileLoadManager.h" + +#include "td/telegram/Global.h" + +#include "td/utils/common.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" + +namespace td { +FileLoadManager::FileLoadManager(ActorShared<Callback> callback, ActorShared<> parent) + : callback_(std::move(callback)), parent_(std::move(parent)) { +} + +void FileLoadManager::start_up() { + upload_resource_manager_ = + create_actor<ResourceManager>("UploadResourceManager", !G()->parameters().use_file_db /*tdlib_engine*/ + ? ResourceManager::Mode::Greedy + : ResourceManager::Mode::Baseline); +} + +ActorOwn<ResourceManager> &FileLoadManager::get_download_resource_manager(bool is_small, DcId dc_id) { + auto &actor = is_small ? download_small_resource_manager_map_[dc_id] : download_resource_manager_map_[dc_id]; + if (actor.empty()) { + actor = create_actor<ResourceManager>( + PSLICE() << "DownloadResourceManager " << tag("is_small", is_small) << tag("dc_id", dc_id), + ResourceManager::Mode::Baseline); + } + return actor; +} + +void FileLoadManager::download(QueryId id, const FullRemoteFileLocation &remote_location, + const LocalFileLocation &local, int64 size, string name, + const FileEncryptionKey &encryption_key, bool search_file, int8 priority) { + if (stop_flag_) { + return; + } + CHECK(query_id_to_node_id_.find(id) == query_id_to_node_id_.end()); + NodeId node_id = nodes_container_.create(Node()); + Node *node = nodes_container_.get(node_id); + CHECK(node); + node->query_id_ = id; + auto callback = make_unique<FileDownloaderCallback>(actor_shared(this, node_id)); + bool is_small = size < 20 * 1024; + node->loader_ = create_actor<FileDownloader>("Downloader", remote_location, local, size, std::move(name), + encryption_key, is_small, search_file, std::move(callback)); + auto &resource_manager = get_download_resource_manager(is_small, remote_location.get_dc_id()); + send_closure(resource_manager, &ResourceManager::register_worker, + ActorShared<FileLoaderActor>(node->loader_.get(), static_cast<uint64>(-1)), priority); + query_id_to_node_id_[id] = node_id; +} + +void FileLoadManager::upload(QueryId id, const LocalFileLocation &local_location, + const RemoteFileLocation &remote_location, int64 size, + const FileEncryptionKey &encryption_key, int8 priority, vector<int> bad_parts) { + if (stop_flag_) { + return; + } + CHECK(query_id_to_node_id_.find(id) == query_id_to_node_id_.end()); + NodeId node_id = nodes_container_.create(Node()); + Node *node = nodes_container_.get(node_id); + CHECK(node); + node->query_id_ = id; + auto callback = make_unique<FileUploaderCallback>(actor_shared(this, node_id)); + node->loader_ = create_actor<FileUploader>("Uploader", local_location, remote_location, size, encryption_key, + std::move(bad_parts), std::move(callback)); + send_closure(upload_resource_manager_, &ResourceManager::register_worker, + ActorShared<FileLoaderActor>(node->loader_.get(), static_cast<uint64>(-1)), priority); + query_id_to_node_id_[id] = node_id; +} + +void FileLoadManager::upload_by_hash(QueryId id, const FullLocalFileLocation &local_location, int64 size, + int8 priority) { + if (stop_flag_) { + return; + } + CHECK(query_id_to_node_id_.find(id) == query_id_to_node_id_.end()); + NodeId node_id = nodes_container_.create(Node()); + Node *node = nodes_container_.get(node_id); + CHECK(node); + node->query_id_ = id; + auto callback = make_unique<FileHashUploaderCallback>(actor_shared(this, node_id)); + node->loader_ = create_actor<FileHashUploader>("HashUploader", local_location, size, std::move(callback)); + send_closure(upload_resource_manager_, &ResourceManager::register_worker, + ActorShared<FileLoaderActor>(node->loader_.get(), static_cast<uint64>(-1)), priority); + query_id_to_node_id_[id] = node_id; +} + +void FileLoadManager::update_priority(QueryId id, int8 priority) { + if (stop_flag_) { + return; + } + auto it = query_id_to_node_id_.find(id); + if (it == query_id_to_node_id_.end()) { + return; + } + auto node = nodes_container_.get(it->second); + if (node == nullptr) { + return; + } + send_closure(node->loader_, &FileLoaderActor::update_priority, priority); +} + +void FileLoadManager::from_bytes(QueryId id, FileType type, BufferSlice bytes, string name) { + if (stop_flag_) { + return; + } + CHECK(query_id_to_node_id_.find(id) == query_id_to_node_id_.end()); + NodeId node_id = nodes_container_.create(Node()); + Node *node = nodes_container_.get(node_id); + CHECK(node); + node->query_id_ = id; + auto callback = make_unique<FileFromBytesCallback>(actor_shared(this, node_id)); + node->loader_ = + create_actor<FileFromBytes>("FromBytes", type, std::move(bytes), std::move(name), std::move(callback)); + query_id_to_node_id_[id] = node_id; +} + +void FileLoadManager::get_content(const FullLocalFileLocation &local_location, Promise<BufferSlice> promise) { + // TODO: send query to other thread + promise.set_result(read_file(local_location.path_)); +} + +// void upload_reload_parts(QueryId id, vector<int32> parts); +// void upload_restart(QueryId id); +void FileLoadManager::cancel(QueryId id) { + if (stop_flag_) { + return; + } + auto it = query_id_to_node_id_.find(id); + if (it == query_id_to_node_id_.end()) { + return; + } + on_error_impl(it->second, Status::Error(1, "Cancelled")); +} +void FileLoadManager::update_local_file_location(QueryId id, const LocalFileLocation &local) { + if (stop_flag_) { + return; + } + auto it = query_id_to_node_id_.find(id); + if (it == query_id_to_node_id_.end()) { + return; + } + auto node = nodes_container_.get(it->second); + if (node == nullptr) { + return; + } + send_closure(node->loader_, &FileLoaderActor::update_local_file_location, local); +} +void FileLoadManager::close() { + nodes_container_.for_each([](auto id, auto &node) { node.loader_.reset(); }); + stop_flag_ = true; + loop(); +} + +void FileLoadManager::on_start_download() { + auto node_id = get_link_token(); + auto node = nodes_container_.get(node_id); + if (node == nullptr) { + return; + } + if (!stop_flag_) { + send_closure(callback_, &Callback::on_start_download, node->query_id_); + } +} + +void FileLoadManager::on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size) { + auto node_id = get_link_token(); + auto node = nodes_container_.get(node_id); + if (node == nullptr) { + return; + } + if (!stop_flag_) { + send_closure(callback_, &Callback::on_partial_download, node->query_id_, partial_local, ready_size); + } +} + +void FileLoadManager::on_partial_upload(const PartialRemoteFileLocation &partial_remote, int64 ready_size) { + auto node_id = get_link_token(); + auto node = nodes_container_.get(node_id); + if (node == nullptr) { + return; + } + if (!stop_flag_) { + send_closure(callback_, &Callback::on_partial_upload, node->query_id_, partial_remote, ready_size); + } +} + +void FileLoadManager::on_ok_download(const FullLocalFileLocation &local, int64 size) { + auto node_id = get_link_token(); + auto node = nodes_container_.get(node_id); + if (node == nullptr) { + return; + } + if (!stop_flag_) { + send_closure(callback_, &Callback::on_download_ok, node->query_id_, local, size); + } + close_node(node_id); + loop(); +} + +void FileLoadManager::on_ok_upload(FileType file_type, const PartialRemoteFileLocation &remote, int64 size) { + auto node_id = get_link_token(); + auto node = nodes_container_.get(node_id); + if (node == nullptr) { + return; + } + if (!stop_flag_) { + send_closure(callback_, &Callback::on_upload_ok, node->query_id_, file_type, remote, size); + } + close_node(node_id); + loop(); +} + +void FileLoadManager::on_ok_upload_full(const FullRemoteFileLocation &remote) { + auto node_id = get_link_token(); + auto node = nodes_container_.get(node_id); + if (node == nullptr) { + return; + } + if (!stop_flag_) { + send_closure(callback_, &Callback::on_upload_full_ok, node->query_id_, remote); + } + close_node(node_id); + loop(); +} + +void FileLoadManager::on_error(Status status) { + auto node_id = get_link_token(); + on_error_impl(node_id, std::move(status)); +} + +void FileLoadManager::on_error_impl(NodeId node_id, Status status) { + auto node = nodes_container_.get(node_id); + if (node == nullptr) { + status.ignore(); + return; + } + if (!stop_flag_) { + send_closure(callback_, &Callback::on_error, node->query_id_, std::move(status)); + } + close_node(node_id); + loop(); +} + +void FileLoadManager::hangup_shared() { + auto node_id = get_link_token(); + on_error_impl(node_id, Status::Error(1, "Cancelled")); +} + +void FileLoadManager::loop() { + if (stop_flag_) { + if (nodes_container_.empty()) { + stop(); + } + return; + } +} + +void FileLoadManager::close_node(NodeId node_id) { + auto node = nodes_container_.get(node_id); + CHECK(node); + query_id_to_node_id_.erase(node->query_id_); + nodes_container_.erase(node_id); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLoadManager.h b/libs/tdlib/td/td/telegram/files/FileLoadManager.h new file mode 100644 index 0000000000..d61df5d08e --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLoadManager.h @@ -0,0 +1,167 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileDownloader.h" +#include "td/telegram/files/FileFromBytes.h" +#include "td/telegram/files/FileHashUploader.h" +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/files/FileUploader.h" +#include "td/telegram/files/ResourceManager.h" + +#include "td/utils/buffer.h" +#include "td/utils/Container.h" +#include "td/utils/Status.h" + +#include <map> + +namespace td { + +class FileLoadManager final : public Actor { + public: + using QueryId = uint64; + class Callback : public Actor { + public: + Callback() = default; + Callback(const Callback &) = delete; + Callback &operator=(const Callback &) = delete; + ~Callback() override = default; + virtual void on_start_download(QueryId id) = 0; + virtual void on_partial_download(QueryId id, const PartialLocalFileLocation &partial_local, int64 ready_size) = 0; + virtual void on_partial_upload(QueryId id, const PartialRemoteFileLocation &partial_remote, int64 ready_size) = 0; + virtual void on_upload_ok(QueryId id, FileType file_type, const PartialRemoteFileLocation &remtoe, int64 size) = 0; + virtual void on_upload_full_ok(QueryId id, const FullRemoteFileLocation &remote) = 0; + virtual void on_download_ok(QueryId id, const FullLocalFileLocation &local, int64 size) = 0; + virtual void on_error(QueryId id, Status status) = 0; + }; + + explicit FileLoadManager(ActorShared<Callback> callback, ActorShared<> parent); + void download(QueryId id, const FullRemoteFileLocation &remote_location, const LocalFileLocation &local, int64 size, + string name, const FileEncryptionKey &encryption_key, bool search_file, int8 priority); + void upload(QueryId id, const LocalFileLocation &local_location, const RemoteFileLocation &remote_location, + int64 size, const FileEncryptionKey &encryption_key, int8 priority, vector<int> bad_parts); + void upload_by_hash(QueryId id, const FullLocalFileLocation &local_location, int64 size, int8 priority); + void update_priority(QueryId id, int8 priority); + void from_bytes(QueryId id, FileType type, BufferSlice bytes, string name); + void cancel(QueryId id); + void update_local_file_location(QueryId id, const LocalFileLocation &local); + void get_content(const FullLocalFileLocation &local_location, Promise<BufferSlice> promise); + + // just stops actor and all queries. no callbacks will be called + void close(); + + private: + struct Node { + QueryId query_id_; + ActorOwn<FileLoaderActor> loader_; + ResourceState resource_state_; + }; + using NodeId = uint64; + + std::map<DcId, ActorOwn<ResourceManager>> download_resource_manager_map_; + std::map<DcId, ActorOwn<ResourceManager>> download_small_resource_manager_map_; + ActorOwn<ResourceManager> upload_resource_manager_; + + Container<Node> nodes_container_; + ActorShared<Callback> callback_; + ActorShared<> parent_; + std::map<QueryId, NodeId> query_id_to_node_id_; + bool stop_flag_ = false; + + void start_up() override; + void loop() override; + void hangup_shared() override; + + void close_node(NodeId node_id); + ActorOwn<ResourceManager> &get_download_resource_manager(bool is_small, DcId dc_id); + + void on_start_download(); + void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size); + void on_partial_upload(const PartialRemoteFileLocation &partial_remote, int64 ready_size); + void on_ok_download(const FullLocalFileLocation &local, int64 size); + void on_ok_upload(FileType file_type, const PartialRemoteFileLocation &remote, int64 size); + void on_ok_upload_full(const FullRemoteFileLocation &remote); + void on_error(Status status); + void on_error_impl(NodeId node_id, Status status); + + class FileDownloaderCallback : public FileDownloader::Callback { + public: + explicit FileDownloaderCallback(ActorShared<FileLoadManager> actor_id) : actor_id_(std::move(actor_id)) { + } + + private: + ActorShared<FileLoadManager> actor_id_; + + void on_start_download() override { + send_closure(actor_id_, &FileLoadManager::on_start_download); + } + void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size) override { + send_closure(actor_id_, &FileLoadManager::on_partial_download, partial_local, ready_size); + } + void on_ok(const FullLocalFileLocation &full_local, int64 size) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_ok_download, full_local, size); + } + void on_error(Status status) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_error, std::move(status)); + } + }; + + class FileUploaderCallback : public FileUploader::Callback { + public: + explicit FileUploaderCallback(ActorShared<FileLoadManager> actor_id) : actor_id_(std::move(actor_id)) { + } + + private: + ActorShared<FileLoadManager> actor_id_; + + void on_partial_upload(const PartialRemoteFileLocation &partial_remote, int64 ready_size) override { + send_closure(actor_id_, &FileLoadManager::on_partial_upload, partial_remote, ready_size); + } + void on_ok(FileType file_type, const PartialRemoteFileLocation &partial_remote, int64 size) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_ok_upload, file_type, partial_remote, size); + } + void on_error(Status status) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_error, std::move(status)); + } + }; + class FileHashUploaderCallback : public FileHashUploader::Callback { + public: + explicit FileHashUploaderCallback(ActorShared<FileLoadManager> actor_id) : actor_id_(std::move(actor_id)) { + } + + private: + ActorShared<FileLoadManager> actor_id_; + + void on_ok(const FullRemoteFileLocation &remote) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_ok_upload_full, remote); + } + void on_error(Status status) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_error, std::move(status)); + } + }; + + class FileFromBytesCallback : public FileFromBytes::Callback { + public: + explicit FileFromBytesCallback(ActorShared<FileLoadManager> actor_id) : actor_id_(std::move(actor_id)) { + } + + private: + ActorShared<FileLoadManager> actor_id_; + + void on_ok(const FullLocalFileLocation &full_local, int64 size) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_ok_download, full_local, size); + } + void on_error(Status status) override { + send_closure(std::move(actor_id_), &FileLoadManager::on_error, std::move(status)); + } + }; +}; + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLoader.cpp b/libs/tdlib/td/td/telegram/files/FileLoader.cpp new file mode 100644 index 0000000000..8431aa8553 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLoader.cpp @@ -0,0 +1,292 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileLoader.h" + +#include "td/telegram/files/ResourceManager.h" +#include "td/telegram/Global.h" +#include "td/telegram/net/NetQueryDispatcher.h" +#include "td/telegram/Td.h" +#include "td/telegram/UniqueId.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/ScopeGuard.h" + +#include <tuple> + +namespace td { +void FileLoader::set_resource_manager(ActorShared<ResourceManager> resource_manager) { + resource_manager_ = std::move(resource_manager); + send_closure(resource_manager_, &ResourceManager::update_resources, resource_state_); +} +void FileLoader::update_priority(int8 priority) { + send_closure(resource_manager_, &ResourceManager::update_priority, priority); +} +void FileLoader::update_resources(const ResourceState &other) { + resource_state_.update_slave(other); + VLOG(files) << "update resources " << resource_state_; + loop(); +} +void FileLoader::set_ordered_flag(bool flag) { + ordered_flag_ = flag; +} +size_t FileLoader::get_part_size() const { + return parts_manager_.get_part_size(); +} +void FileLoader::hangup() { + // if (!stop_flag_) { + // stop_flag_ = true; + // on_error(Status::Error("Cancelled")); + //} + stop(); +} + +void FileLoader::update_local_file_location(const LocalFileLocation &local) { + auto r_prefix_info = on_update_local_location(local); + if (r_prefix_info.is_error()) { + on_error(r_prefix_info.move_as_error()); + stop_flag_ = true; + return; + } + auto prefix_info = r_prefix_info.move_as_ok(); + auto status = parts_manager_.set_known_prefix(narrow_cast<size_t>(prefix_info.size), prefix_info.is_ready); + if (status.is_error()) { + on_error(std::move(status)); + stop_flag_ = true; + return; + } + loop(); +} + +void FileLoader::start_up() { + auto r_file_info = init(); + if (r_file_info.is_error()) { + on_error(r_file_info.move_as_error()); + stop_flag_ = true; + return; + } + auto file_info = r_file_info.ok(); + auto size = file_info.size; + auto expected_size = max(size, file_info.expected_size); + bool is_size_final = file_info.is_size_final; + auto part_size = file_info.part_size; + auto &ready_parts = file_info.ready_parts; + auto use_part_count_limit = file_info.use_part_count_limit; + auto status = parts_manager_.init(size, expected_size, is_size_final, part_size, ready_parts, use_part_count_limit); + if (file_info.only_check) { + parts_manager_.set_checked_prefix_size(0); + } + if (status.is_error()) { + on_error(std::move(status)); + stop_flag_ = true; + return; + } + if (ordered_flag_) { + ordered_parts_ = OrderedEventsProcessor<std::pair<Part, NetQueryPtr>>(parts_manager_.get_ready_prefix_count()); + } + if (file_info.need_delay) { + delay_dispatcher_ = create_actor<DelayDispatcher>("DelayDispatcher", 0.003); + next_delay_ = 0.05; + } + resource_state_.set_unit_size(parts_manager_.get_part_size()); + update_estimated_limit(); + on_progress_impl(narrow_cast<size_t>(parts_manager_.get_ready_size())); + yield(); +} + +void FileLoader::loop() { + if (stop_flag_) { + return; + } + auto status = do_loop(); + if (status.is_error()) { + if (status.code() == 1) { + return; + } + on_error(std::move(status)); + stop_flag_ = true; + return; + } +} +Status FileLoader::do_loop() { + TRY_RESULT(check_info, + check_loop(parts_manager_.get_checked_prefix_size(), parts_manager_.get_unchecked_ready_prefix_size(), + parts_manager_.unchecked_ready())); + if (check_info.changed) { + on_progress_impl(narrow_cast<size_t>(parts_manager_.get_ready_size())); + } + for (auto &query : check_info.queries) { + G()->net_query_dispatcher().dispatch_with_callback( + std::move(query), actor_shared(this, UniqueId::next(UniqueId::Type::Default, CommonQueryKey))); + } + if (check_info.need_check) { + parts_manager_.set_need_check(); + parts_manager_.set_checked_prefix_size(check_info.checked_prefix_size); + } + + if (parts_manager_.ready()) { + TRY_STATUS(parts_manager_.finish()); + TRY_STATUS(on_ok(parts_manager_.get_size())); + LOG(INFO) << "Bad download order rate: " + << (debug_total_parts_ == 0 ? 0.0 : 100.0 * debug_bad_part_order_ / debug_total_parts_) << "% " + << debug_bad_part_order_ << "/" << debug_total_parts_ << " " << format::as_array(debug_bad_parts_); + stop_flag_ = true; + return Status::OK(); + } + + TRY_STATUS(before_start_parts()); + SCOPE_EXIT { + after_start_parts(); + }; + while (true) { + if (blocking_id_ != 0) { + break; + } + if (resource_state_.unused() < static_cast<int64>(parts_manager_.get_part_size())) { + VLOG(files) << "Got only " << resource_state_.unused() << " resource"; + break; + } + TRY_RESULT(part, parts_manager_.start_part()); + if (part.size == 0) { + break; + } + VLOG(files) << "Start part " << tag("id", part.id) << tag("size", part.size); + resource_state_.start_use(static_cast<int64>(part.size)); + + TRY_RESULT(query_flag, start_part(part, parts_manager_.get_part_count())); + NetQueryPtr query; + bool is_blocking; + std::tie(query, is_blocking) = std::move(query_flag); + uint64 id = UniqueId::next(); + if (is_blocking) { + CHECK(blocking_id_ == 0); + blocking_id_ = id; + } + part_map_[id] = std::make_pair(part, query->cancel_slot_.get_signal_new()); + // part_map_[id] = std::make_pair(part, query.get_weak()); + + auto callback = actor_shared(this, id); + if (delay_dispatcher_.empty()) { + G()->net_query_dispatcher().dispatch_with_callback(std::move(query), std::move(callback)); + } else { + send_closure(delay_dispatcher_, &DelayDispatcher::send_with_callback_and_delay, std::move(query), + std::move(callback), next_delay_); + next_delay_ = std::max(next_delay_ * 0.8, 0.003); + } + } + return Status::OK(); +} + +void FileLoader::tear_down() { + for (auto &it : part_map_) { + it.second.second.reset(); // cancel_query(it.second.second); + } +} +void FileLoader::update_estimated_limit() { + if (stop_flag_) { + return; + } + auto estimated_exta = parts_manager_.get_expected_size() - parts_manager_.get_ready_size(); + resource_state_.update_estimated_limit(estimated_exta); + VLOG(files) << "update estimated limit " << estimated_exta; + if (!resource_manager_.empty()) { + keep_fd_flag(narrow_cast<uint64>(resource_state_.active_limit()) >= parts_manager_.get_part_size()); + send_closure(resource_manager_, &ResourceManager::update_resources, resource_state_); + } +} + +void FileLoader::on_result(NetQueryPtr query) { + if (stop_flag_) { + return; + } + auto id = get_link_token(); + if (id == blocking_id_) { + blocking_id_ = 0; + } + if (UniqueId::extract_key(id) == CommonQueryKey) { + on_common_query(std::move(query)); + return loop(); + } + auto it = part_map_.find(id); + if (it == part_map_.end()) { + LOG(WARNING) << "Got result for unknown part"; + return; + } + + Part part = it->second.first; + it->second.second.release(); + CHECK(query->is_ready()); + + bool next = false; + auto status = [&] { + TRY_RESULT(should_restart, should_restart_part(part, query)); + if (should_restart) { + VLOG(files) << "Restart part " << tag("id", part.id) << tag("size", part.size); + resource_state_.stop_use(static_cast<int64>(part.size)); + parts_manager_.on_part_failed(part.id); + } else { + next = true; + } + return Status::OK(); + }(); + if (status.is_error()) { + on_error(std::move(status)); + stop_flag_ = true; + return; + } + + if (next) { + if (ordered_flag_) { + auto seq_no = part.id; + ordered_parts_.add(seq_no, std::make_pair(part, std::move(query)), + [this](auto seq_no, auto &&p) { this->on_part_query(p.first, std::move(p.second)); }); + } else { + on_part_query(part, std::move(query)); + } + } + update_estimated_limit(); + loop(); +} + +void FileLoader::on_part_query(Part part, NetQueryPtr query) { + auto status = try_on_part_query(part, std::move(query)); + if (status.is_error()) { + on_error(std::move(status)); + stop_flag_ = true; + } +} + +void FileLoader::on_common_query(NetQueryPtr query) { + auto status = process_check_query(std::move(query)); + if (status.is_error()) { + on_error(std::move(status)); + stop_flag_ = true; + } +} + +Status FileLoader::try_on_part_query(Part part, NetQueryPtr query) { + TRY_RESULT(size, process_part(part, std::move(query))); + VLOG(files) << "Ok part " << tag("id", part.id) << tag("size", part.size); + resource_state_.stop_use(static_cast<int64>(part.size)); + auto old_ready_prefix_count = parts_manager_.get_ready_prefix_count(); + TRY_STATUS(parts_manager_.on_part_ok(part.id, part.size, size)); + auto new_ready_prefix_count = parts_manager_.get_ready_prefix_count(); + debug_total_parts_++; + if (old_ready_prefix_count == new_ready_prefix_count) { + debug_bad_parts_.push_back(part.id); + debug_bad_part_order_++; + } + on_progress_impl(size); + return Status::OK(); +} + +void FileLoader::on_progress_impl(size_t size) { + on_progress(parts_manager_.get_part_count(), static_cast<int32>(parts_manager_.get_part_size()), + parts_manager_.get_ready_prefix_count(), parts_manager_.ready(), parts_manager_.get_ready_size()); +} +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLoader.h b/libs/tdlib/td/td/telegram/files/FileLoader.h new file mode 100644 index 0000000000..a97a5199e9 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLoader.h @@ -0,0 +1,129 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileLoaderActor.h" +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/files/PartsManager.h" +#include "td/telegram/files/ResourceManager.h" +#include "td/telegram/files/ResourceState.h" +#include "td/telegram/net/NetQuery.h" + +#include "td/telegram/DelayDispatcher.h" + +#include "td/utils/OrderedEventsProcessor.h" +#include "td/utils/Status.h" + +#include <map> +#include <utility> + +namespace td { +class FileLoader : public FileLoaderActor { + public: + class Callback { + public: + Callback() = default; + Callback(const Callback &) = delete; + Callback &operator=(const Callback &) = delete; + virtual ~Callback() = default; + }; + void set_resource_manager(ActorShared<ResourceManager> resource_manager) override; + void update_priority(int8 priority) override; + void update_resources(const ResourceState &other) override; + + void update_local_file_location(const LocalFileLocation &local) override; + + protected: + void set_ordered_flag(bool flag); + size_t get_part_size() const; + + struct PrefixInfo { + int64 size = -1; + bool is_ready = false; + }; + struct FileInfo { + int64 size; + int64 expected_size = 0; + bool is_size_final; + int32 part_size; + std::vector<int> ready_parts; + bool use_part_count_limit = true; + bool only_check = false; + bool need_delay = false; + }; + virtual Result<FileInfo> init() TD_WARN_UNUSED_RESULT = 0; + virtual Status on_ok(int64 size) TD_WARN_UNUSED_RESULT = 0; + virtual void on_error(Status status) = 0; + virtual Status before_start_parts() { + return Status::OK(); + } + virtual Result<std::pair<NetQueryPtr, bool>> start_part(Part part, int part_count) TD_WARN_UNUSED_RESULT = 0; + virtual void after_start_parts() { + } + virtual Result<size_t> process_part(Part part, NetQueryPtr net_query) TD_WARN_UNUSED_RESULT = 0; + virtual void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, bool is_ready, + int64 ready_size) = 0; + virtual Callback *get_callback() = 0; + virtual Result<PrefixInfo> on_update_local_location(const LocalFileLocation &location) TD_WARN_UNUSED_RESULT { + return Status::Error("unsupported"); + } + virtual Result<bool> should_restart_part(Part part, NetQueryPtr &net_query) TD_WARN_UNUSED_RESULT { + return false; + } + + virtual Status process_check_query(NetQueryPtr net_query) { + return Status::Error("unsupported"); + } + struct CheckInfo { + bool need_check{false}; + bool changed{false}; + int64 checked_prefix_size{0}; + std::vector<NetQueryPtr> queries; + }; + virtual Result<CheckInfo> check_loop(int64 checked_prefix_size, int64 ready_prefix_size, bool is_ready) { + return CheckInfo{}; + } + + virtual void keep_fd_flag(bool keep_fd) { + } + + private: + enum { CommonQueryKey = 2 }; + bool stop_flag_ = false; + ActorShared<ResourceManager> resource_manager_; + ResourceState resource_state_; + PartsManager parts_manager_; + uint64 blocking_id_{0}; + std::map<uint64, std::pair<Part, ActorShared<>>> part_map_; + // std::map<uint64, std::pair<Part, NetQueryRef>> part_map_; + bool ordered_flag_ = false; + OrderedEventsProcessor<std::pair<Part, NetQueryPtr>> ordered_parts_; + ActorOwn<DelayDispatcher> delay_dispatcher_; + double next_delay_ = 0; + + uint32 debug_total_parts_ = 0; + uint32 debug_bad_part_order_ = 0; + std::vector<int32> debug_bad_parts_; + + void start_up() override; + void loop() override; + Status do_loop(); + void hangup() override; + void tear_down() override; + + void update_estimated_limit(); + void on_progress_impl(size_t size); + + void on_result(NetQueryPtr query) override; + void on_part_query(Part part, NetQueryPtr query); + void on_common_query(NetQueryPtr query); + Status try_on_part_query(Part part, NetQueryPtr query); +}; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLoaderActor.h b/libs/tdlib/td/td/telegram/files/FileLoaderActor.h new file mode 100644 index 0000000000..d802f589ca --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLoaderActor.h @@ -0,0 +1,28 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/files/ResourceState.h" +#include "td/telegram/net/NetQuery.h" + +namespace td { + +class LocalFileLocation; +class ResourceManager; + +class FileLoaderActor : public NetQueryCallback { + public: + virtual void set_resource_manager(ActorShared<ResourceManager>) = 0; + virtual void update_priority(int8 priority) = 0; + virtual void update_resources(const ResourceState &other) = 0; + + // TODO: existence of this function is a dirty hack. Refactoring is highly appreciated + virtual void update_local_file_location(const LocalFileLocation &local) { + } +}; + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLoaderUtils.cpp b/libs/tdlib/td/td/telegram/files/FileLoaderUtils.cpp new file mode 100644 index 0000000000..1e89cb72ee --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLoaderUtils.cpp @@ -0,0 +1,168 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileLoaderUtils.h" + +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/Global.h" + +#include "td/utils/common.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/PathView.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/port/path.h" +#include "td/utils/Random.h" +#include "td/utils/StringBuilder.h" + +#include <tuple> + +namespace td { + +namespace { +Result<std::pair<FileFd, string>> try_create_new_file(Result<CSlice> result_name) { + TRY_RESULT(name, std::move(result_name)); + TRY_RESULT(fd, FileFd::open(name, FileFd::Read | FileFd::Write | FileFd::CreateNew, 0640)); + return std::make_pair(std::move(fd), name.str()); +} +Result<std::pair<FileFd, string>> try_open_file(Result<CSlice> result_name) { + TRY_RESULT(name, std::move(result_name)); + TRY_RESULT(fd, FileFd::open(name, FileFd::Read, 0640)); + return std::make_pair(std::move(fd), name.str()); +} + +struct RandSuff { + int len; +}; +StringBuilder &operator<<(StringBuilder &sb, const RandSuff &) { + for (int i = 0; i < 6; i++) { + sb << format::hex_digit(Random::fast(0, 15)); + } + return sb; +} +struct Ext { + Slice ext; +}; +StringBuilder &operator<<(StringBuilder &sb, const Ext &ext) { + if (ext.ext.empty()) { + return sb; + } + return sb << "." << ext.ext; +} +} // namespace + +Result<std::pair<FileFd, string>> open_temp_file(const FileType &file_type) { + auto pmc = G()->td_db()->get_binlog_pmc(); + // TODO: CAS? + int32 file_id = to_integer<int32>(pmc->get("tmp_file_id")); + pmc->set("tmp_file_id", to_string(file_id + 1)); + + auto temp_dir = get_files_temp_dir(file_type); + auto res = try_create_new_file(PSLICE_SAFE() << temp_dir << file_id); + if (res.is_error()) { + res = try_create_new_file(PSLICE_SAFE() << temp_dir << file_id << "_" << RandSuff{6}); + } + return res; +} + +template <class F> +bool for_suggested_file_name(CSlice name, bool use_pmc, bool use_random, F &&callback) { + auto try_callback = [&](Result<CSlice> r_path) { + if (r_path.is_error()) { + return true; + } + return callback(r_path.move_as_ok()); + }; + auto cleaned_name = clean_filename(name); + PathView path_view(cleaned_name); + auto stem = path_view.file_stem(); + auto ext = path_view.extension(); + bool active = true; + if (!stem.empty() && !G()->parameters().ignore_file_names) { + active = try_callback(PSLICE_SAFE() << stem << Ext{ext}); + for (int i = 0; active && i < 10; i++) { + active = try_callback(PSLICE_SAFE() << stem << "_(" << i << ")" << Ext{ext}); + } + for (int i = 2; active && i < 12 && use_random; i++) { + active = try_callback(PSLICE_SAFE() << stem << "_(" << RandSuff{i} << ")" << Ext{ext}); + } + } else if (use_pmc) { + auto pmc = G()->td_db()->get_binlog_pmc(); + int32 file_id = to_integer<int32>(pmc->get("perm_file_id")); + pmc->set("perm_file_id", to_string(file_id + 1)); + active = try_callback(PSLICE_SAFE() << "file_" << file_id << Ext{ext}); + if (active) { + active = try_callback(PSLICE_SAFE() << "file_" << file_id << "_" << RandSuff{6} << Ext{ext}); + } + } + return active; +} + +Result<string> create_from_temp(CSlice temp_path, CSlice dir, CSlice name) { + LOG(INFO) << "Create file in directory " << dir << " with suggested name " << name << " from temporary file " + << temp_path; + Result<std::pair<FileFd, string>> res = Status::Error(500, "Can't find suitable file name"); + for_suggested_file_name(name, true, true, [&](CSlice suggested_name) { + res = try_create_new_file(PSLICE_SAFE() << dir << suggested_name); + return res.is_error(); + }); + TRY_RESULT(tmp, std::move(res)); + tmp.first.close(); + auto perm_path = std::move(tmp.second); + TRY_STATUS(rename(temp_path, perm_path)); + return perm_path; +} + +Result<string> search_file(CSlice dir, CSlice name, int64 expected_size) { + Result<std::string> res = Status::Error(500, "Can't find suitable file name"); + for_suggested_file_name(name, false, false, [&](CSlice suggested_name) { + auto r_pair = try_open_file(PSLICE_SAFE() << dir << suggested_name); + if (r_pair.is_error()) { + return false; + } + FileFd fd; + std::string path; + std::tie(fd, path) = r_pair.move_as_ok(); + if (fd.stat().size_ != expected_size) { + return true; + } + fd.close(); + res = std::move(path); + return false; + }); + return res; +} + +const char *file_type_name[file_type_size] = {"thumbnails", "profile_photos", "photos", "voice", + "videos", "documents", "secret", "temp", + "stickers", "music", "animations", "secret_thumbnails", + "wallpapers", "video_notes"}; + +string get_file_base_dir(const FileDirType &file_dir_type) { + switch (file_dir_type) { + case FileDirType::Secure: + return G()->get_dir().str(); + case FileDirType::Common: + return G()->get_files_dir().str(); + default: + UNREACHABLE(); + return ""; + } +} + +string get_files_base_dir(const FileType &file_type) { + return get_file_base_dir(get_file_dir_type(file_type)); +} +string get_files_temp_dir(const FileType &file_type) { + return get_files_base_dir(file_type) + "temp" + TD_DIR_SLASH; +} +string get_files_dir(const FileType &file_type) { + return get_files_base_dir(file_type) + file_type_name[static_cast<int32>(file_type)] + TD_DIR_SLASH; +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLoaderUtils.h b/libs/tdlib/td/td/telegram/files/FileLoaderUtils.h new file mode 100644 index 0000000000..7eaf1e12e8 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLoaderUtils.h @@ -0,0 +1,25 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <utility> + +namespace td { +enum class FileType : int8; + +Result<std::pair<FileFd, string>> open_temp_file(const FileType &file_type) TD_WARN_UNUSED_RESULT; +Result<string> create_from_temp(CSlice temp_path, CSlice dir, CSlice name) TD_WARN_UNUSED_RESULT; +Result<string> search_file(CSlice dir, CSlice name, int64 expected_size) TD_WARN_UNUSED_RESULT; +string get_files_base_dir(const FileType &file_type); +string get_files_temp_dir(const FileType &file_type); +string get_files_dir(const FileType &file_type); +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileLocation.h b/libs/tdlib/td/td/telegram/files/FileLocation.h new file mode 100644 index 0000000000..7733b65d2a --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileLocation.h @@ -0,0 +1,1199 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/td_api.h" +#include "td/telegram/telegram_api.h" + +#include "td/telegram/DialogId.h" +#include "td/telegram/net/DcId.h" + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" +#include "td/utils/format.h" +#include "td/utils/int_types.h" +#include "td/utils/logging.h" +#include "td/utils/Random.h" +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/tl_helpers.h" +#include "td/utils/tl_storers.h" +#include "td/utils/Variant.h" + +#include <cstring> +#include <tuple> + +namespace td { + +enum class FileType : int8 { + Thumbnail, + ProfilePhoto, + Photo, + VoiceNote, + Video, + Document, + Encrypted, + Temp, + Sticker, + Audio, + Animation, + EncryptedThumbnail, + Wallpaper, + VideoNote, + Size, + None +}; + +inline FileType from_td_api(const td_api::FileType &file_type) { + switch (file_type.get_id()) { + case td_api::fileTypeThumbnail::ID: + return FileType::Thumbnail; + case td_api::fileTypeProfilePhoto::ID: + return FileType::ProfilePhoto; + case td_api::fileTypePhoto::ID: + return FileType::Photo; + case td_api::fileTypeVoiceNote::ID: + return FileType::VoiceNote; + case td_api::fileTypeVideo::ID: + return FileType::Video; + case td_api::fileTypeDocument::ID: + return FileType::Document; + case td_api::fileTypeSecret::ID: + return FileType::Encrypted; + case td_api::fileTypeUnknown::ID: + return FileType::Temp; + case td_api::fileTypeSticker::ID: + return FileType::Sticker; + case td_api::fileTypeAudio::ID: + return FileType::Audio; + case td_api::fileTypeAnimation::ID: + return FileType::Animation; + case td_api::fileTypeSecretThumbnail::ID: + return FileType::EncryptedThumbnail; + case td_api::fileTypeWallpaper::ID: + return FileType::Wallpaper; + case td_api::fileTypeVideoNote::ID: + return FileType::VideoNote; + case td_api::fileTypeNone::ID: + return FileType::None; + default: + UNREACHABLE(); + return FileType::None; + } +} + +inline tl_object_ptr<td_api::FileType> as_td_api(FileType file_type) { + switch (file_type) { + case FileType::Thumbnail: + return make_tl_object<td_api::fileTypeThumbnail>(); + case FileType::ProfilePhoto: + return make_tl_object<td_api::fileTypeProfilePhoto>(); + case FileType::Photo: + return make_tl_object<td_api::fileTypePhoto>(); + case FileType::VoiceNote: + return make_tl_object<td_api::fileTypeVoiceNote>(); + case FileType::Video: + return make_tl_object<td_api::fileTypeVideo>(); + case FileType::Document: + return make_tl_object<td_api::fileTypeDocument>(); + case FileType::Encrypted: + return make_tl_object<td_api::fileTypeSecret>(); + case FileType::Temp: + return make_tl_object<td_api::fileTypeUnknown>(); + case FileType::Sticker: + return make_tl_object<td_api::fileTypeSticker>(); + case FileType::Audio: + return make_tl_object<td_api::fileTypeAudio>(); + case FileType::Animation: + return make_tl_object<td_api::fileTypeAnimation>(); + case FileType::EncryptedThumbnail: + return make_tl_object<td_api::fileTypeSecretThumbnail>(); + case FileType::Wallpaper: + return make_tl_object<td_api::fileTypeWallpaper>(); + case FileType::VideoNote: + return make_tl_object<td_api::fileTypeVideoNote>(); + case FileType::None: + return make_tl_object<td_api::fileTypeNone>(); + default: + UNREACHABLE(); + return nullptr; + } +} + +enum class FileDirType : int8 { Secure, Common }; +inline FileDirType get_file_dir_type(FileType file_type) { + switch (file_type) { + case FileType::Thumbnail: + case FileType::ProfilePhoto: + case FileType::Encrypted: + case FileType::Sticker: + case FileType::Temp: + case FileType::Wallpaper: + case FileType::EncryptedThumbnail: + return FileDirType::Secure; + default: + return FileDirType::Common; + } +} + +constexpr int32 file_type_size = static_cast<int32>(FileType::Size); +extern const char *file_type_name[file_type_size]; + +struct FileEncryptionKey { + FileEncryptionKey() = default; + FileEncryptionKey(Slice key, Slice iv) : key_iv_(key.size() + iv.size(), '\0') { + if (key.size() != 32 || iv.size() != 32) { + LOG(ERROR) << "Wrong key/iv sizes: " << key.size() << " " << iv.size(); + return; + } + CHECK(key_iv_.size() == 64); + std::memcpy(&key_iv_[0], key.data(), key.size()); + std::memcpy(&key_iv_[key.size()], iv.data(), iv.size()); + } + static FileEncryptionKey create() { + FileEncryptionKey res; + res.key_iv_.resize(64); + Random::secure_bytes(res.key_iv_); + return res; + } + + const UInt256 &key() const { + CHECK(key_iv_.size() == 64); + return *reinterpret_cast<const UInt256 *>(key_iv_.data()); + } + Slice key_slice() const { + CHECK(key_iv_.size() == 64); + return Slice(key_iv_.data(), 32); + } + + UInt256 &mutable_iv() { + CHECK(key_iv_.size() == 64); + return *reinterpret_cast<UInt256 *>(&key_iv_[0] + 32); + } + Slice iv_slice() const { + CHECK(key_iv_.size() == 64); + return Slice(key_iv_.data() + 32, 32); + } + + int32 calc_fingerprint() const { + char buf[16]; + md5(key_iv_, {buf, sizeof(buf)}); + return as<int32>(buf) ^ as<int32>(buf + 4); + } + + bool empty() const { + return key_iv_.empty(); + } + + template <class StorerT> + void store(StorerT &storer) const { + td::store(key_iv_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + td::parse(key_iv_, parser); + } + + string key_iv_; // TODO wrong alignment is possible +}; + +inline bool operator==(const FileEncryptionKey &lhs, const FileEncryptionKey &rhs) { + return lhs.key_iv_ == rhs.key_iv_; +} + +inline bool operator!=(const FileEncryptionKey &lhs, const FileEncryptionKey &rhs) { + return !(lhs == rhs); +} + +struct EmptyRemoteFileLocation { + template <class StorerT> + void store(StorerT &storer) const { + } + template <class ParserT> + void parse(ParserT &parser) { + } +}; + +inline bool operator==(const EmptyRemoteFileLocation &lhs, const EmptyRemoteFileLocation &rhs) { + return true; +} + +inline bool operator!=(const EmptyRemoteFileLocation &lhs, const EmptyRemoteFileLocation &rhs) { + return !(lhs == rhs); +} + +struct PartialRemoteFileLocation { + int64 file_id_; + int32 part_count_; + int32 part_size_; + int32 ready_part_count_; + int32 is_big_; + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(file_id_, storer); + store(part_count_, storer); + store(part_size_, storer); + store(ready_part_count_, storer); + store(is_big_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + parse(file_id_, parser); + parse(part_count_, parser); + parse(part_size_, parser); + parse(ready_part_count_, parser); + parse(is_big_, parser); + } +}; + +inline bool operator==(const PartialRemoteFileLocation &lhs, const PartialRemoteFileLocation &rhs) { + return lhs.file_id_ == rhs.file_id_ && lhs.part_count_ == rhs.part_count_ && lhs.part_size_ == rhs.part_size_ && + lhs.ready_part_count_ == rhs.ready_part_count_ && lhs.is_big_ == rhs.is_big_; +} + +inline bool operator!=(const PartialRemoteFileLocation &lhs, const PartialRemoteFileLocation &rhs) { + return !(lhs == rhs); +} + +struct PhotoRemoteFileLocation { + int64 id_; + int64 access_hash_; + int64 volume_id_; + int64 secret_; + int32 local_id_; + + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(id_, storer); + store(access_hash_, storer); + store(volume_id_, storer); + store(secret_, storer); + store(local_id_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + parse(id_, parser); + parse(access_hash_, parser); + parse(volume_id_, parser); + parse(secret_, parser); + parse(local_id_, parser); + } + struct AsKey { + const PhotoRemoteFileLocation &key; + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(key.id_, storer); + store(key.volume_id_, storer); + store(key.local_id_, storer); + } + }; + AsKey as_key() const { + return AsKey{*this}; + } + + bool operator<(const PhotoRemoteFileLocation &other) const { + return std::tie(id_, volume_id_, local_id_) < std::tie(other.id_, other.volume_id_, other.local_id_); + } + bool operator==(const PhotoRemoteFileLocation &other) const { + return std::tie(id_, volume_id_, local_id_) == std::tie(other.id_, other.volume_id_, other.local_id_); + } +}; + +inline StringBuilder &operator<<(StringBuilder &string_builder, const PhotoRemoteFileLocation &location) { + return string_builder << "[id = " << location.id_ << ", access_hash = " << location.access_hash_ + << ", volume_id = " << location.volume_id_ << ", secret = " << location.secret_ + << ", local_id = " << location.local_id_ << "]"; +} + +struct WebRemoteFileLocation { + string url_; + int64 access_hash_; + + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(url_, storer); + store(access_hash_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + parse(url_, parser); + parse(access_hash_, parser); + } + struct AsKey { + const WebRemoteFileLocation &key; + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(key.url_, storer); + } + }; + AsKey as_key() const { + return AsKey{*this}; + } + bool operator<(const WebRemoteFileLocation &other) const { + return url_ < other.url_; + } + bool operator==(const WebRemoteFileLocation &other) const { + return url_ == other.url_; + } +}; + +inline StringBuilder &operator<<(StringBuilder &string_builder, const WebRemoteFileLocation &location) { + return string_builder << "[url = " << location.url_ << ", access_hash = " << location.access_hash_ << "]"; +} + +struct CommonRemoteFileLocation { + int64 id_; + int64 access_hash_; + + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(id_, storer); + store(access_hash_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + parse(id_, parser); + parse(access_hash_, parser); + } + struct AsKey { + const CommonRemoteFileLocation &key; + template <class StorerT> + void store(StorerT &storer) const { + td::store(key.id_, storer); + } + }; + AsKey as_key() const { + return AsKey{*this}; + } + bool operator<(const CommonRemoteFileLocation &other) const { + return std::tie(id_) < std::tie(other.id_); + } + bool operator==(const CommonRemoteFileLocation &other) const { + return std::tie(id_) == std::tie(other.id_); + } +}; + +inline StringBuilder &operator<<(StringBuilder &string_builder, const CommonRemoteFileLocation &location) { + return string_builder << "[id = " << location.id_ << ", access_hash = " << location.access_hash_ << "]"; +} + +class FullRemoteFileLocation { + public: + FileType file_type_{FileType::None}; + + private: + static constexpr int32 WEB_LOCATION_FLAG = 1 << 24; + bool web_location_flag_{false}; + DcId dc_id_; + enum class LocationType { Web, Photo, Common, None }; + Variant<WebRemoteFileLocation, PhotoRemoteFileLocation, CommonRemoteFileLocation> variant_; + + LocationType location_type() const { + if (is_web()) { + return LocationType::Web; + } + switch (file_type_) { + case FileType::Photo: + case FileType::ProfilePhoto: + case FileType::Thumbnail: + case FileType::EncryptedThumbnail: + case FileType::Wallpaper: + return LocationType::Photo; + case FileType::Video: + case FileType::VoiceNote: + case FileType::Document: + case FileType::Sticker: + case FileType::Audio: + case FileType::Animation: + case FileType::Encrypted: + case FileType::VideoNote: + return LocationType::Common; + case FileType::None: + case FileType::Size: + default: + UNREACHABLE(); + case FileType::Temp: + return LocationType::None; + } + } + + WebRemoteFileLocation &web() { + return variant_.get<WebRemoteFileLocation>(); + } + PhotoRemoteFileLocation &photo() { + return variant_.get<PhotoRemoteFileLocation>(); + } + CommonRemoteFileLocation &common() { + return variant_.get<CommonRemoteFileLocation>(); + } + const WebRemoteFileLocation &web() const { + return variant_.get<WebRemoteFileLocation>(); + } + const PhotoRemoteFileLocation &photo() const { + return variant_.get<PhotoRemoteFileLocation>(); + } + const CommonRemoteFileLocation &common() const { + return variant_.get<CommonRemoteFileLocation>(); + } + + friend StringBuilder &operator<<(StringBuilder &string_builder, + const FullRemoteFileLocation &full_remote_file_location); + + int32 full_type() const { + auto type = static_cast<int32>(file_type_); + if (is_web()) { + type |= WEB_LOCATION_FLAG; + } + return type; + } + + public: + template <class StorerT> + void store(StorerT &storer) const { + using ::td::store; + store(full_type(), storer); + store(dc_id_.get_value(), storer); + variant_.visit([&](auto &&value) { + using td::store; + store(value, storer); + }); + } + template <class ParserT> + void parse(ParserT &parser) { + using ::td::parse; + int32 raw_type; + parse(raw_type, parser); + web_location_flag_ = (raw_type & WEB_LOCATION_FLAG) != 0; + raw_type &= ~WEB_LOCATION_FLAG; + if (raw_type < 0 || raw_type >= static_cast<int32>(FileType::Size)) { + return parser.set_error("Invalid FileType in FullRemoteFileLocation"); + } + file_type_ = static_cast<FileType>(raw_type); + int32 dc_id_value; + parse(dc_id_value, parser); + dc_id_ = DcId::from_value(dc_id_value); + + switch (location_type()) { + case LocationType::Web: { + variant_ = WebRemoteFileLocation(); + return web().parse(parser); + } + case LocationType::Photo: { + variant_ = PhotoRemoteFileLocation(); + return photo().parse(parser); + } + case LocationType::Common: { + variant_ = CommonRemoteFileLocation(); + return common().parse(parser); + } + case LocationType::None: { + break; + } + } + parser.set_error("Invalid FileType in FullRemoteFileLocation"); + } + + struct AsKey { + const FullRemoteFileLocation &key; + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(key.full_type(), storer); + key.variant_.visit([&](auto &&value) { + using td::store; + store(value.as_key(), storer); + }); + } + }; + AsKey as_key() const { + return AsKey{*this}; + } + + DcId get_dc_id() const { + return dc_id_; + } + int64 get_access_hash() const { + switch (location_type()) { + case LocationType::Photo: + return photo().access_hash_; + case LocationType::Common: + return common().access_hash_; + case LocationType::Web: + return web().access_hash_; + case LocationType::None: + default: + UNREACHABLE(); + return 0; + } + } + int64 get_id() const { + switch (location_type()) { + case LocationType::Photo: + return photo().id_; + case LocationType::Common: + return common().id_; + case LocationType::Web: + case LocationType::None: + default: + UNREACHABLE(); + return 0; + } + } + string get_url() const { + if (is_web()) { + return web().url_; + } + + return string(); + } + + bool is_web() const { + return web_location_flag_; + } + bool is_photo() const { + return location_type() == LocationType::Photo; + } + bool is_common() const { + return location_type() == LocationType::Common; + } + bool is_encrypted() const { + return file_type_ == FileType::Encrypted; + } + + tl_object_ptr<telegram_api::inputWebFileLocation> as_input_web_file_location() const { + CHECK(is_web()); + return make_tl_object<telegram_api::inputWebFileLocation>(web().url_, web().access_hash_); + } + tl_object_ptr<telegram_api::InputFileLocation> as_input_file_location() const { + switch (location_type()) { + case LocationType::Photo: + return make_tl_object<telegram_api::inputFileLocation>(photo().volume_id_, photo().local_id_, photo().secret_); + case LocationType::Common: + if (is_encrypted()) { + return make_tl_object<telegram_api::inputEncryptedFileLocation>(common().id_, common().access_hash_); + } else { + return make_tl_object<telegram_api::inputDocumentFileLocation>(common().id_, common().access_hash_, 0); + } + case LocationType::Web: + case LocationType::None: + default: + UNREACHABLE(); + return nullptr; + } + } + + tl_object_ptr<telegram_api::InputDocument> as_input_document() const { + CHECK(is_common()); + LOG_IF(ERROR, is_encrypted()) << "Can't call as_input_document on an encrypted file"; + return make_tl_object<telegram_api::inputDocument>(common().id_, common().access_hash_); + } + + tl_object_ptr<telegram_api::InputPhoto> as_input_photo() const { + CHECK(is_photo()); + return make_tl_object<telegram_api::inputPhoto>(photo().id_, photo().access_hash_); + } + + tl_object_ptr<telegram_api::InputEncryptedFile> as_input_encrypted_file() const { + CHECK(is_encrypted()) << "Can't call as_input_encrypted_file on a non-encrypted file"; + return make_tl_object<telegram_api::inputEncryptedFile>(common().id_, common().access_hash_); + } + + // TODO: this constructor is just for immediate unserialize + FullRemoteFileLocation() = default; + FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret, + DcId dc_id) + : file_type_(file_type) + , dc_id_(dc_id) + , variant_(PhotoRemoteFileLocation{id, access_hash, volume_id, secret, local_id}) { + CHECK(is_photo()); + } + FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id) + : file_type_(file_type), dc_id_(dc_id), variant_(CommonRemoteFileLocation{id, access_hash}) { + CHECK(is_common()); + } + FullRemoteFileLocation(FileType file_type, string url, int64 access_hash, DcId dc_id) + : file_type_(file_type) + , web_location_flag_{true} + , dc_id_(dc_id) + , variant_(WebRemoteFileLocation{std::move(url), access_hash}) { + CHECK(is_web()); + CHECK(!web().url_.empty()); + } + + bool operator<(const FullRemoteFileLocation &other) const { + if (full_type() != other.full_type()) { + return full_type() < other.full_type(); + } + if (dc_id_ != other.dc_id_) { + return dc_id_ < other.dc_id_; + } + switch (location_type()) { + case LocationType::Photo: + return photo() < other.photo(); + case LocationType::Common: + return common() < other.common(); + case LocationType::Web: + return web() < other.web(); + case LocationType::None: + default: + UNREACHABLE(); + return false; + } + } + bool operator==(const FullRemoteFileLocation &other) const { + if (full_type() != other.full_type()) { + return false; + } + if (dc_id_ != other.dc_id_) { + return false; + } + switch (location_type()) { + case LocationType::Photo: + return photo() == other.photo(); + case LocationType::Common: + return common() == other.common(); + case LocationType::Web: + return web() == other.web(); + case LocationType::None: + default: + UNREACHABLE(); + return false; + } + } + + static const int32 KEY_MAGIC = 0x64374632; +}; + +inline StringBuilder &operator<<(StringBuilder &string_builder, + const FullRemoteFileLocation &full_remote_file_location) { + string_builder << "[" << file_type_name[static_cast<int32>(full_remote_file_location.file_type_)] << ", " + << full_remote_file_location.get_dc_id() << ", location = "; + + if (full_remote_file_location.is_web()) { + string_builder << full_remote_file_location.web(); + } else if (full_remote_file_location.is_photo()) { + string_builder << full_remote_file_location.photo(); + } else if (full_remote_file_location.is_common()) { + string_builder << full_remote_file_location.common(); + } + + return string_builder << "]"; +} + +class RemoteFileLocation { + public: + enum class Type : int32 { Empty, Partial, Full }; + + Type type() const { + return static_cast<Type>(variant_.get_offset()); + } + + template <class StorerT> + void store(StorerT &storer) const { + storer.store_int(variant_.get_offset()); + bool ok{false}; + variant_.visit([&](auto &&value) { + using td::store; + store(value, storer); + ok = true; + }); + CHECK(ok); + } + PartialRemoteFileLocation &partial() { + return variant_.get<1>(); + } + FullRemoteFileLocation &full() { + return variant_.get<2>(); + } + const PartialRemoteFileLocation &partial() const { + return variant_.get<1>(); + } + const FullRemoteFileLocation &full() const { + return variant_.get<2>(); + } + template <class ParserT> + void parse(ParserT &parser) { + auto type = static_cast<Type>(parser.fetch_int()); + switch (type) { + case Type::Empty: { + variant_ = EmptyRemoteFileLocation(); + return; + } + case Type::Partial: { + variant_ = PartialRemoteFileLocation(); + return partial().parse(parser); + } + case Type::Full: { + variant_ = FullRemoteFileLocation(); + return full().parse(parser); + } + } + parser.set_error("Invalid type in RemoteFileLocation"); + } + + RemoteFileLocation() : variant_{EmptyRemoteFileLocation{}} { + } + explicit RemoteFileLocation(const FullRemoteFileLocation &full) : variant_(full) { + } + explicit RemoteFileLocation(const PartialRemoteFileLocation &partial) : variant_(partial) { + } + RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret, + DcId dc_id) + : variant_(FullRemoteFileLocation{file_type, id, access_hash, local_id, volume_id, secret, dc_id}) { + } + RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id) + : variant_(FullRemoteFileLocation{file_type, id, access_hash, dc_id}) { + } + + private: + Variant<EmptyRemoteFileLocation, PartialRemoteFileLocation, FullRemoteFileLocation> variant_; + + friend bool operator==(const RemoteFileLocation &lhs, const RemoteFileLocation &rhs); +}; + +inline bool operator==(const RemoteFileLocation &lhs, const RemoteFileLocation &rhs) { + return lhs.variant_ == rhs.variant_; +} + +inline bool operator!=(const RemoteFileLocation &lhs, const RemoteFileLocation &rhs) { + return !(lhs == rhs); +} + +struct EmptyLocalFileLocation { + template <class StorerT> + void store(StorerT &storer) const { + } + template <class ParserT> + void parse(ParserT &parser) { + } +}; + +inline bool operator==(const EmptyLocalFileLocation &lhs, const EmptyLocalFileLocation &rhs) { + return true; +} + +inline bool operator!=(const EmptyLocalFileLocation &lhs, const EmptyLocalFileLocation &rhs) { + return !(lhs == rhs); +} + +struct PartialLocalFileLocation { + FileType file_type_; + string path_; + int32 part_size_; + int32 ready_part_count_; + string iv_; + + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(file_type_, storer); + store(path_, storer); + store(part_size_, storer); + store(ready_part_count_, storer); + store(iv_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + parse(file_type_, parser); + if (file_type_ < FileType::Thumbnail || file_type_ >= FileType::Size) { + return parser.set_error("Invalid type in PartialLocalFileLocation"); + } + parse(path_, parser); + parse(part_size_, parser); + parse(ready_part_count_, parser); + parse(iv_, parser); + } +}; + +inline bool operator==(const PartialLocalFileLocation &lhs, const PartialLocalFileLocation &rhs) { + return lhs.file_type_ == rhs.file_type_ && lhs.path_ == rhs.path_ && lhs.part_size_ == rhs.part_size_ && + lhs.ready_part_count_ == rhs.ready_part_count_ && lhs.iv_ == rhs.iv_; +} + +inline bool operator!=(const PartialLocalFileLocation &lhs, const PartialLocalFileLocation &rhs) { + return !(lhs == rhs); +} + +struct FullLocalFileLocation { + FileType file_type_; + string path_; + uint64 mtime_nsec_; + + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(file_type_, storer); + store(mtime_nsec_, storer); + store(path_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + parse(file_type_, parser); + if (file_type_ < FileType::Thumbnail || file_type_ >= FileType::Size) { + return parser.set_error("Invalid type in FullLocalFileLocation"); + } + parse(mtime_nsec_, parser); + parse(path_, parser); + } + const FullLocalFileLocation &as_key() const { + return *this; + } + + // TODO: remove this constructor + FullLocalFileLocation() : file_type_(FileType::Photo) { + } + FullLocalFileLocation(FileType file_type, string path, uint64 mtime_nsec) + : file_type_(file_type), path_(std::move(path)), mtime_nsec_(mtime_nsec) { + } + + static const int32 KEY_MAGIC = 0x84373817; +}; + +inline bool operator<(const FullLocalFileLocation &lhs, const FullLocalFileLocation &rhs) { + return std::tie(lhs.file_type_, lhs.mtime_nsec_, lhs.path_) < std::tie(rhs.file_type_, rhs.mtime_nsec_, rhs.path_); +} + +inline bool operator==(const FullLocalFileLocation &lhs, const FullLocalFileLocation &rhs) { + return std::tie(lhs.file_type_, lhs.mtime_nsec_, lhs.path_) == std::tie(rhs.file_type_, rhs.mtime_nsec_, rhs.path_); +} + +inline bool operator!=(const FullLocalFileLocation &lhs, const FullLocalFileLocation &rhs) { + return !(lhs == rhs); +} + +inline StringBuilder &operator<<(StringBuilder &sb, const FullLocalFileLocation &location) { + return sb << tag("path", location.path_); +} + +class LocalFileLocation { + public: + enum class Type : int32 { Empty, Partial, Full }; + + Type type() const { + return static_cast<Type>(variant_.get_offset()); + } + + PartialLocalFileLocation &partial() { + return variant_.get<1>(); + } + FullLocalFileLocation &full() { + return variant_.get<2>(); + } + const PartialLocalFileLocation &partial() const { + return variant_.get<1>(); + } + const FullLocalFileLocation &full() const { + return variant_.get<2>(); + } + + CSlice file_name() const { + switch (type()) { + case Type::Partial: + return partial().path_; + case Type::Full: + return full().path_; + case Type::Empty: + default: + return CSlice(); + } + } + + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(variant_.get_offset(), storer); + variant_.visit([&](auto &&value) { + using td::store; + store(value, storer); + }); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + auto type = static_cast<Type>(parser.fetch_int()); + switch (type) { + case Type::Empty: + variant_ = EmptyLocalFileLocation(); + return; + case Type::Partial: + variant_ = PartialLocalFileLocation(); + return parse(partial(), parser); + case Type::Full: + variant_ = FullLocalFileLocation(); + return parse(full(), parser); + } + return parser.set_error("Invalid type in LocalFileLocation"); + } + + LocalFileLocation() : variant_{EmptyLocalFileLocation()} { + } + explicit LocalFileLocation(const PartialLocalFileLocation &partial) : variant_(partial) { + } + explicit LocalFileLocation(const FullLocalFileLocation &full) : variant_(full) { + } + LocalFileLocation(FileType file_type, string path, uint64 mtime_nsec) + : variant_(FullLocalFileLocation{file_type, std::move(path), mtime_nsec}) { + } + + private: + Variant<EmptyLocalFileLocation, PartialLocalFileLocation, FullLocalFileLocation> variant_; + + friend bool operator==(const LocalFileLocation &lhs, const LocalFileLocation &rhs); +}; + +inline bool operator==(const LocalFileLocation &lhs, const LocalFileLocation &rhs) { + return lhs.variant_ == rhs.variant_; +} + +inline bool operator!=(const LocalFileLocation &lhs, const LocalFileLocation &rhs) { + return !(lhs == rhs); +} + +struct FullGenerateFileLocation { + FileType file_type_{FileType::None}; + string original_path_; + string conversion_; + static const int32 KEY_MAGIC = 0x8b60a1c8; + + template <class StorerT> + void store(StorerT &storer) const { + using td::store; + store(file_type_, storer); + store(original_path_, storer); + store(conversion_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using td::parse; + parse(file_type_, parser); + parse(original_path_, parser); + parse(conversion_, parser); + } + + const FullGenerateFileLocation &as_key() const { + return *this; + } + FullGenerateFileLocation() = default; + FullGenerateFileLocation(FileType file_type, string original_path, string conversion) + : file_type_(file_type), original_path_(std::move(original_path)), conversion_(std::move(conversion)) { + } +}; + +inline bool operator<(const FullGenerateFileLocation &lhs, const FullGenerateFileLocation &rhs) { + return std::tie(lhs.file_type_, lhs.original_path_, lhs.conversion_) < + std::tie(rhs.file_type_, rhs.original_path_, rhs.conversion_); +} + +inline bool operator==(const FullGenerateFileLocation &lhs, const FullGenerateFileLocation &rhs) { + return std::tie(lhs.file_type_, lhs.original_path_, lhs.conversion_) == + std::tie(rhs.file_type_, rhs.original_path_, rhs.conversion_); +} + +inline bool operator!=(const FullGenerateFileLocation &lhs, const FullGenerateFileLocation &rhs) { + return !(lhs == rhs); +} + +inline StringBuilder &operator<<(StringBuilder &string_builder, + const FullGenerateFileLocation &full_generated_file_location) { + return string_builder << "[" + << tag("file_type", file_type_name[static_cast<int32>(full_generated_file_location.file_type_)]) + << tag("original_path", full_generated_file_location.original_path_) + << tag("conversion", full_generated_file_location.conversion_) << "]"; +} + +class GenerateFileLocation { + public: + enum class Type : int32 { Empty, Full }; + + Type type() const { + return type_; + } + + FullGenerateFileLocation &full() { + CHECK(type_ == Type::Full); + return full_; + } + const FullGenerateFileLocation &full() const { + CHECK(type_ == Type::Full); + return full_; + } + + template <class StorerT> + void store(StorerT &storer) const { + td::store(type_, storer); + switch (type_) { + case Type::Empty: + return; + case Type::Full: + return td::store(full_, storer); + } + } + + template <class ParserT> + void parse(ParserT &parser) { + td::parse(type_, parser); + switch (type_) { + case Type::Empty: + return; + case Type::Full: + return td::parse(full_, parser); + } + return parser.set_error("Invalid type in GenerateFileLocation"); + } + + GenerateFileLocation() : type_(Type::Empty) { + } + + explicit GenerateFileLocation(const FullGenerateFileLocation &full) : type_(Type::Full), full_(full) { + } + + GenerateFileLocation(FileType file_type, string original_path, string conversion) + : type_(Type::Full), full_{file_type, std::move(original_path), std::move(conversion)} { + } + + private: + Type type_; + FullGenerateFileLocation full_; +}; + +inline bool operator==(const GenerateFileLocation &lhs, const GenerateFileLocation &rhs) { + if (lhs.type() != rhs.type()) { + return false; + } + switch (lhs.type()) { + case GenerateFileLocation::Type::Empty: + return true; + case GenerateFileLocation::Type::Full: + return lhs.full() == rhs.full(); + } + UNREACHABLE(); + return false; +} + +inline bool operator!=(const GenerateFileLocation &lhs, const GenerateFileLocation &rhs) { + return !(lhs == rhs); +} + +class FileData { + public: + DialogId owner_dialog_id_; + uint64 pmc_id_ = 0; + RemoteFileLocation remote_; + LocalFileLocation local_; + unique_ptr<FullGenerateFileLocation> generate_; + int64 size_ = 0; + int64 expected_size_ = 0; + string remote_name_; + string url_; + FileEncryptionKey encryption_key_; + + template <class StorerT> + void store(StorerT &storer) const { + using ::td::store; + bool has_owner_dialog_id = owner_dialog_id_.is_valid(); + bool has_expected_size = size_ == 0 && expected_size_ != 0; + BEGIN_STORE_FLAGS(); + STORE_FLAG(has_owner_dialog_id); + STORE_FLAG(has_expected_size); + END_STORE_FLAGS(); + + if (has_owner_dialog_id) { + store(owner_dialog_id_, storer); + } + store(pmc_id_, storer); + store(remote_, storer); + store(local_, storer); + auto generate = generate_ == nullptr ? GenerateFileLocation() : GenerateFileLocation(*generate_); + store(generate, storer); + if (has_expected_size) { + store(expected_size_, storer); + } else { + store(size_, storer); + } + store(remote_name_, storer); + store(url_, storer); + store(encryption_key_, storer); + } + template <class ParserT> + void parse(ParserT &parser) { + using ::td::parse; + bool has_owner_dialog_id; + bool has_expected_size; + BEGIN_PARSE_FLAGS(); + PARSE_FLAG(has_owner_dialog_id); + PARSE_FLAG(has_expected_size); + END_PARSE_FLAGS(); + + if (has_owner_dialog_id) { + parse(owner_dialog_id_, parser); + } + parse(pmc_id_, parser); + parse(remote_, parser); + parse(local_, parser); + GenerateFileLocation generate; + parse(generate, parser); + if (generate.type() == GenerateFileLocation::Type::Full) { + generate_ = std::make_unique<FullGenerateFileLocation>(generate.full()); + } else { + generate_ = nullptr; + } + if (has_expected_size) { + parse(expected_size_, parser); + } else { + parse(size_, parser); + } + parse(remote_name_, parser); + parse(url_, parser); + parse(encryption_key_, parser); + } +}; +inline StringBuilder &operator<<(StringBuilder &sb, const FileData &file_data) { + sb << "[" << tag("remote_name", file_data.remote_name_) << " " << file_data.owner_dialog_id_ << " " + << tag("size", file_data.size_) << tag("expected_size", file_data.expected_size_); + if (!file_data.url_.empty()) { + sb << tag("url", file_data.url_); + } + if (file_data.local_.type() == LocalFileLocation::Type::Full) { + sb << " local " << file_data.local_.full(); + } + if (file_data.generate_ != nullptr) { + sb << " generate " << *file_data.generate_; + } + if (file_data.remote_.type() == RemoteFileLocation::Type::Full) { + sb << " remote " << file_data.remote_.full(); + } + return sb << "]"; +} + +template <class T> +string as_key(const T &object) { + TlStorerCalcLength calc_length; + calc_length.store_int(0); + object.as_key().store(calc_length); + + BufferSlice key_buffer{calc_length.get_length()}; + auto key = key_buffer.as_slice(); + TlStorerUnsafe storer(key.begin()); + storer.store_int(T::KEY_MAGIC); + object.as_key().store(storer); + return key.str(); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileManager.cpp b/libs/tdlib/td/td/telegram/files/FileManager.cpp new file mode 100644 index 0000000000..dc5e2d1caf --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileManager.cpp @@ -0,0 +1,2356 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileManager.h" + +#include "td/telegram/telegram_api.h" + +#include "td/telegram/files/FileLoaderUtils.h" +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/files/FileUploader.h" +#include "td/telegram/Global.h" +#include "td/telegram/misc.h" +#include "td/telegram/Td.h" + +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/HttpUrl.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/PathView.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/port/path.h" +#include "td/utils/port/Stat.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/tl_helpers.h" + +#include <algorithm> +#include <limits> +#include <tuple> +#include <utility> + +namespace td { + +static int VERBOSITY_NAME(update_file) = VERBOSITY_NAME(INFO); + +FileNode *FileNodePtr::operator->() const { + return get(); +} + +FileNode &FileNodePtr::operator*() const { + return *get(); +} + +FileNode *FileNodePtr::get() const { + auto res = get_unsafe(); + CHECK(res); + return res; +} + +FullRemoteFileLocation *FileNodePtr::get_remote() const { + return file_manager_->get_remote(file_id_.get_remote()); +} + +FileNode *FileNodePtr::get_unsafe() const { + CHECK(file_manager_ != nullptr); + return file_manager_->get_file_node_raw(file_id_); +} + +FileNodePtr::operator bool() const { + return file_manager_ != nullptr && get_unsafe() != nullptr; +} + +void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size) { + if (local_ready_size_ != ready_size) { + local_ready_size_ = ready_size; + VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size"; + on_info_changed(); + } + if (local_ != local) { + VLOG(update_file) << "File " << main_file_id_ << " has changed local location"; + local_ = local; + on_changed(); + } +} + +void FileNode::set_remote_location(const RemoteFileLocation &remote, FileLocationSource source, int64 ready_size) { + if (remote_ready_size_ != ready_size) { + remote_ready_size_ = ready_size; + VLOG(update_file) << "File " << main_file_id_ << " has changed remote ready size"; + on_info_changed(); + } + if (remote_ == remote) { + if (remote_.type() == RemoteFileLocation::Type::Full) { + if (remote_.full().get_access_hash() == remote.full().get_access_hash()) { + return; + } + } else { + return; + } + } + + VLOG(update_file) << "File " << main_file_id_ << " has changed remote location"; + remote_ = remote; + remote_source_ = source; + on_changed(); +} + +void FileNode::set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate) { + bool is_changed = generate_ == nullptr ? generate != nullptr : generate == nullptr || *generate_ != *generate; + if (is_changed) { + generate_ = std::move(generate); + on_pmc_changed(); + } +} + +void FileNode::set_size(int64 size) { + if (size_ != size) { + VLOG(update_file) << "File " << main_file_id_ << " has changed size to " << size; + size_ = size; + on_changed(); + } +} + +void FileNode::set_expected_size(int64 expected_size) { + if (expected_size_ != expected_size) { + VLOG(update_file) << "File " << main_file_id_ << " has changed expected size to " << expected_size; + expected_size_ = expected_size; + on_changed(); + } +} + +void FileNode::set_remote_name(string remote_name) { + if (remote_name_ != remote_name) { + remote_name_ = std::move(remote_name); + on_pmc_changed(); + } +} + +void FileNode::set_url(string url) { + if (url_ != url) { + VLOG(update_file) << "File " << main_file_id_ << " has changed URL to " << url; + url_ = std::move(url); + on_changed(); + } +} + +void FileNode::set_owner_dialog_id(DialogId owner_id) { + if (owner_dialog_id_ != owner_id) { + owner_dialog_id_ = owner_id; + on_pmc_changed(); + } +} + +void FileNode::set_encryption_key(FileEncryptionKey key) { + if (encryption_key_ != key) { + encryption_key_ = std::move(key); + on_pmc_changed(); + } +} + +void FileNode::set_download_priority(int8 priority) { + if ((download_priority_ == 0) != (priority == 0)) { + VLOG(update_file) << "File " << main_file_id_ << " has changed download priority to " << priority; + on_info_changed(); + } + download_priority_ = priority; +} + +void FileNode::set_upload_priority(int8 priority) { + if ((upload_priority_ == 0) != (priority == 0)) { + VLOG(update_file) << "File " << main_file_id_ << " has changed upload priority to " << priority; + on_info_changed(); + } + upload_priority_ = priority; +} + +void FileNode::set_generate_priority(int8 download_priority, int8 upload_priority) { + if ((download_priority_ == 0) != (download_priority == 0) || (upload_priority_ == 0) != (upload_priority == 0)) { + VLOG(update_file) << "File " << main_file_id_ << " has changed generate priority to " << download_priority << "/" + << upload_priority; + on_info_changed(); + } + generate_priority_ = max(download_priority, upload_priority); + generate_download_priority_ = download_priority; + generate_upload_priority_ = upload_priority; +} + +void FileNode::on_changed() { + on_pmc_changed(); + on_info_changed(); +} +void FileNode::on_info_changed() { + info_changed_flag_ = true; +} +void FileNode::on_pmc_changed() { + pmc_changed_flag_ = true; +} + +bool FileNode::need_info_flush() const { + return info_changed_flag_; +} + +bool FileNode::need_pmc_flush() const { + if (!pmc_changed_flag_) { + return false; + } + + // already in pmc + if (pmc_id_ != 0) { + return true; + } + + // We must save encryption key + if (!encryption_key_.empty()) { + // && remote_.type() != RemoteFileLocation::Type::Empty + return true; + } + + bool has_generate_location = generate_ != nullptr; + // Do not save "#file_id#" conversion. + if (has_generate_location && begins_with(generate_->conversion_, "#file_id#")) { + has_generate_location = false; + } + + if (remote_.type() == RemoteFileLocation::Type::Full && + (has_generate_location || local_.type() != LocalFileLocation::Type::Empty)) { + return true; + } + if (local_.type() == LocalFileLocation::Type::Full && + (has_generate_location || remote_.type() != RemoteFileLocation::Type::Empty)) { + return true; + } + + // TODO: Generate location with constant conversion + + return false; +} + +void FileNode::on_pmc_flushed() { + pmc_changed_flag_ = false; +} +void FileNode::on_info_flushed() { + info_changed_flag_ = false; +} + +string FileNode::suggested_name() const { + if (!remote_name_.empty()) { + return remote_name_; + } + if (!url_.empty()) { + auto file_name = get_url_file_name(url_); + if (!file_name.empty()) { + return file_name; + } + } + if (generate_ != nullptr) { + if (!generate_->original_path_.empty()) { + return generate_->original_path_; + } + } + return local_.file_name().str(); +} + +/*** FileView ***/ +bool FileView::has_local_location() const { + return node_->local_.type() == LocalFileLocation::Type::Full; +} +const FullLocalFileLocation &FileView::local_location() const { + CHECK(has_local_location()); + return node_->local_.full(); +} +bool FileView::has_remote_location() const { + return node_->remote_.type() == RemoteFileLocation::Type::Full; +} +const FullRemoteFileLocation &FileView::remote_location() const { + CHECK(has_remote_location()); + auto *remote = node_.get_remote(); + if (remote) { + return *remote; + } + return node_->remote_.full(); +} +bool FileView::has_generate_location() const { + return node_->generate_ != nullptr; +} +const FullGenerateFileLocation &FileView::generate_location() const { + CHECK(has_generate_location()); + return *node_->generate_; +} + +int64 FileView::size() const { + return node_->size_; +} + +int64 FileView::expected_size() const { + if (node_->size_ != 0) { + return node_->size_; + } + return node_->expected_size_; +} + +bool FileView::is_downloading() const { + return node_->download_priority_ != 0 || node_->generate_download_priority_ != 0; +} + +int64 FileView::local_size() const { + switch (node_->local_.type()) { + case LocalFileLocation::Type::Full: + return node_->size_; + case LocalFileLocation::Type::Partial: + return node_->local_.partial().part_size_ * node_->local_.partial().ready_part_count_; + default: + return 0; + } +} +int64 FileView::local_total_size() const { + switch (node_->local_.type()) { + case LocalFileLocation::Type::Empty: + return 0; + case LocalFileLocation::Type::Full: + return node_->size_; + case LocalFileLocation::Type::Partial: + return max(static_cast<int64>(node_->local_.partial().part_size_) * node_->local_.partial().ready_part_count_, + node_->local_ready_size_); + default: + UNREACHABLE(); + return 0; + } +} + +bool FileView::is_uploading() const { + return node_->upload_priority_ != 0 || node_->generate_upload_priority_ != 0; +} + +int64 FileView::remote_size() const { + switch (node_->remote_.type()) { + case RemoteFileLocation::Type::Full: + return node_->size_; + case RemoteFileLocation::Type::Partial: { + auto res = + max(static_cast<int64>(node_->remote_.partial().part_size_) * node_->remote_.partial().ready_part_count_, + node_->remote_ready_size_); + if (size() != 0 && size() < res) { + res = size(); + } + return res; + } + default: + return 0; + } +} + +string FileView::path() const { + switch (node_->local_.type()) { + case LocalFileLocation::Type::Full: + return node_->local_.full().path_; + case LocalFileLocation::Type::Partial: + return node_->local_.partial().path_; + default: + return ""; + } +} + +bool FileView::has_url() const { + return !node_->url_.empty(); +} + +const string &FileView::url() const { + return node_->url_; +} + +const string &FileView::remote_name() const { + return node_->remote_name_; +} + +string FileView::suggested_name() const { + return node_->suggested_name(); +} + +DialogId FileView::owner_dialog_id() const { + return node_->owner_dialog_id_; +} + +bool FileView::get_by_hash() const { + return node_->get_by_hash_; +} + +FileView::FileView(ConstFileNodePtr node) : node_(node) { +} + +bool FileView::empty() const { + return !node_; +} + +bool FileView::can_download_from_server() const { + if (!has_remote_location()) { + return false; + } + if (remote_location().file_type_ == FileType::Encrypted && encryption_key().empty()) { + return false; + } + if (remote_location().get_dc_id().is_empty()) { + return false; + } + return true; +} +bool FileView::can_generate() const { + return has_generate_location(); +} + +bool FileView::can_delete() const { + if (has_local_location()) { + return begins_with(local_location().path_, get_files_dir(get_type())); + } + return node_->local_.type() == LocalFileLocation::Type::Partial; +} + +/*** FileManager ***/ +namespace { +void prepare_path_for_pmc(FileType file_type, string &path) { + path = PathView::relative(path, get_files_base_dir(file_type)).str(); +} +} // namespace + +FileManager::FileManager(std::unique_ptr<Context> context) : context_(std::move(context)) { + if (G()->parameters().use_file_db) { + file_db_ = G()->td_db()->get_file_db_shared(); + } + + parent_ = context_->create_reference(); + next_file_id(); + next_file_node_id(); + + std::vector<string> dirs; + auto create_dir = [&](CSlice path) { + dirs.push_back(path.str()); + auto status = mkdir(path, 0750); + if (status.is_error()) { + auto r_stat = stat(path); + if (r_stat.is_ok() && r_stat.ok().is_dir_) { + LOG(ERROR) << "mkdir " << tag("path", path) << " failed " << status << ", but directory exists"; + } else { + LOG(ERROR) << "mkdir " << tag("path", path) << " failed " << status; + } + } +#if TD_ANDROID + FileFd::open(dirs.back() + ".nomedia", FileFd::Create | FileFd::Read).ignore(); +#endif + }; + for (int i = 0; i < file_type_size; i++) { + auto path = get_files_dir(FileType(i)); + create_dir(path); + } + + // Create both temp dirs. + create_dir(get_files_temp_dir(FileType::Encrypted)); + create_dir(get_files_temp_dir(FileType::Video)); + + G()->td_db()->with_db_path([this](CSlice path) { this->bad_paths_.insert(path.str()); }); +} + +void FileManager::init_actor() { + file_load_manager_ = create_actor_on_scheduler<FileLoadManager>("FileLoadManager", G()->get_slow_net_scheduler_id(), + actor_shared(this), context_->create_reference()); + file_generate_manager_ = create_actor_on_scheduler<FileGenerateManager>( + "FileGenerateManager", G()->get_slow_net_scheduler_id(), context_->create_reference()); +} +FileManager::~FileManager() { + // NB: As FileLoadManager callback is just "this" pointer, this event must be processed immediately. + send_closure(std::move(file_load_manager_), &FileLoadManager::close); +} + +string FileManager::fix_file_extension(Slice file_name, Slice file_type, Slice file_extension) { + return (file_name.empty() ? file_type : file_name).str() + "." + file_extension.str(); +} + +string FileManager::get_file_name(FileType file_type, Slice path) { + PathView path_view(path); + auto file_name = path_view.file_name(); + auto extension = path_view.extension(); + switch (file_type) { + case FileType::Thumbnail: + if (extension != "jpg" && extension != "jpeg" && extension != "webp") { + return fix_file_extension(file_name, "thumbnail", "jpg"); + } + break; + case FileType::ProfilePhoto: + case FileType::Photo: + if (extension != "jpg" && extension != "jpeg" && extension != "gif" && extension != "png" && extension != "tif" && + extension != "bmp") { + return fix_file_extension(file_name, "photo", "jpg"); + } + break; + case FileType::VoiceNote: + if (extension != "ogg" && extension != "oga" && extension != "mp3" && extension != "mpeg3" && + extension != "m4a") { + return fix_file_extension(file_name, "voice", "oga"); + } + break; + case FileType::Video: + case FileType::VideoNote: + if (extension != "mov" && extension != "3gp" && extension != "mpeg4" && extension != "mp4") { + return fix_file_extension(file_name, "video", "mp4"); + } + break; + case FileType::Audio: + if (extension != "ogg" && extension != "oga" && extension != "mp3" && extension != "mpeg3" && + extension != "m4a") { + return fix_file_extension(file_name, "audio", "mp3"); + } + break; + case FileType::Document: + case FileType::Sticker: + case FileType::Animation: + case FileType::Encrypted: + case FileType::Temp: + case FileType::EncryptedThumbnail: + case FileType::Wallpaper: + break; + default: + UNREACHABLE(); + } + return file_name.str(); +} + +Status FileManager::check_local_location(FullLocalFileLocation &location, int64 &size) { + constexpr int64 MAX_THUMBNAIL_SIZE = 200 * (1 << 10) /* 200KB */; + constexpr int64 MAX_FILE_SIZE = 1500 * (1 << 20) /* 1500MB */; + + if (location.path_.empty()) { + return Status::Error("File must have non-empty path"); + } + TRY_RESULT(path, realpath(location.path_, true)); + if (bad_paths_.count(path) != 0) { + return Status::Error("Sending of internal database files is forbidden"); + } + location.path_ = std::move(path); + TRY_RESULT(stat, stat(location.path_)); + if (!stat.is_reg_) { + return Status::Error("File must be a regular file"); + } + if (stat.size_ < 0) { + // TODO is it possible? + return Status::Error("File is too big"); + } + if (stat.size_ == 0) { + return Status::Error("File must be non-empty"); + } + + if (size == 0) { + size = stat.size_; + } + if (location.mtime_nsec_ == 0) { + LOG(INFO) << "Set file \"" << location.path_ << "\" modification time to " << stat.mtime_nsec_; + location.mtime_nsec_ = stat.mtime_nsec_; + } else if (location.mtime_nsec_ != stat.mtime_nsec_) { + LOG(INFO) << "File \"" << location.path_ << "\" was nodified: old mtime = " << location.mtime_nsec_ + << ", new mtime = " << stat.mtime_nsec_; + return Status::Error(PSLICE() << "File \"" << location.path_ << "\" was modified"); + } + if ((location.file_type_ == FileType::Thumbnail || location.file_type_ == FileType::EncryptedThumbnail) && + size >= MAX_THUMBNAIL_SIZE) { + return Status::Error(PSLICE() << "File \"" << location.path_ << "\" is too big for thumbnail " + << tag("size", format::as_size(size))); + } + if (size >= MAX_FILE_SIZE) { + return Status::Error(PSLICE() << "File \"" << location.path_ << "\" is too big " + << tag("size", format::as_size(size))); + } + return Status::OK(); +} + +static Status check_partial_local_location(const PartialLocalFileLocation &location) { + TRY_RESULT(stat, stat(location.path_)); + if (!stat.is_reg_) { + if (stat.is_dir_) { + return Status::Error(PSLICE() << "Can't use directory \"" << location.path_ << "\" as a file path"); + } + return Status::Error("File must be a regular file"); + } + // can't check mtime. Hope nobody will mess with this file in our temporary dir. + return Status::OK(); +} + +Status FileManager::check_local_location(FileNodePtr node) { + Status status; + if (node->local_.type() == LocalFileLocation::Type::Full) { + status = check_local_location(node->local_.full(), node->size_); + } else if (node->local_.type() == LocalFileLocation::Type::Partial) { + status = check_partial_local_location(node->local_.partial()); + } + if (status.is_error()) { + node->set_local_location(LocalFileLocation(), 0); + try_flush_node(node); + } + return status; +} + +FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) { + CHECK(0 <= file_id.get() && file_id.get() < static_cast<int32>(file_id_info_.size())) + << file_id << " " << file_id_info_.size(); + return &file_id_info_[file_id.get()]; +} + +FileId FileManager::dup_file_id(FileId file_id) { + int32 file_node_id; + auto *file_node = get_file_node_raw(file_id, &file_node_id); + if (!file_node) { + return FileId(); + } + auto result = create_file_id(file_node_id, file_node); + LOG(INFO) << "Dup file " << file_id << " to " << result; + return result; +} + +FileId FileManager::create_file_id(int32 file_node_id, FileNode *file_node) { + auto file_id = next_file_id(); + get_file_id_info(file_id)->node_id_ = file_node_id; + file_node->file_ids_.push_back(file_id); + return file_id; +} +void FileManager::try_forget_file_id(FileId file_id) { + auto *info = get_file_id_info(file_id); + if (info->send_updates_flag_ || info->pin_flag_) { + return; + } + auto file_node = get_file_node(file_id); + if (file_node->main_file_id_ == file_id) { + return; + } + + LOG(INFO) << "Forget file " << file_id; + auto it = std::find(file_node->file_ids_.begin(), file_node->file_ids_.end(), file_id); + CHECK(it != file_node->file_ids_.end()); + file_node->file_ids_.erase(it); + *info = FileIdInfo(); + empty_file_ids_.push_back(file_id.get()); +} + +FileId FileManager::register_empty(FileType type) { + return register_local(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok(); +} + +void FileManager::on_file_unlink(const FullLocalFileLocation &location) { + auto it = local_location_to_file_id_.find(location); + if (it == local_location_to_file_id_.end()) { + return; + } + auto file_id = it->second; + auto file_node = get_sync_file_node(file_id); + CHECK(file_node); + file_node->set_local_location(LocalFileLocation(), 0); + try_flush_node_info(file_node); +} + +Result<FileId> FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, + bool get_by_hash, bool force) { + // TODO: use get_by_hash + FileData data; + data.local_ = LocalFileLocation(std::move(location)); + data.owner_dialog_id_ = owner_dialog_id; + data.size_ = size; + return register_file(std::move(data), FileLocationSource::None /*won't be used*/, "register_local", force); +} + +FileId FileManager::register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source, + DialogId owner_dialog_id, int64 size, int64 expected_size, string name) { + FileData data; + data.remote_ = RemoteFileLocation(location); + data.owner_dialog_id_ = owner_dialog_id; + data.size_ = size; + data.expected_size_ = expected_size; + data.remote_name_ = std::move(name); + return register_file(std::move(data), file_location_source, "register_remote", false).move_as_ok(); +} + +FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source, + DialogId owner_dialog_id) { + auto file_id = register_generate(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok(); + auto file_node = get_file_node(file_id); + CHECK(file_node); + file_node->set_url(url); + return file_id; +} + +Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSource file_location_source, + string original_path, string conversion, DialogId owner_dialog_id, + int64 expected_size) { + FileData data; + data.generate_ = make_unique<FullGenerateFileLocation>(file_type, std::move(original_path), std::move(conversion)); + data.owner_dialog_id_ = owner_dialog_id; + data.expected_size_ = expected_size; + return register_file(std::move(data), file_location_source, "register_generate", false); +} + +Result<FileId> FileManager::register_file(FileData data, FileLocationSource file_location_source, const char *source, + bool force) { + bool has_remote = data.remote_.type() == RemoteFileLocation::Type::Full; + bool has_generate = data.generate_ != nullptr; + if (data.local_.type() == LocalFileLocation::Type::Full && !force) { + if (file_location_source == FileLocationSource::FromDb) { + PathView path_view(data.local_.full().path_); + if (path_view.is_relative()) { + data.local_.full().path_ = get_files_base_dir(data.local_.full().file_type_) + data.local_.full().path_; + } + } + + auto status = check_local_location(data.local_.full(), data.size_); + if (status.is_error()) { + LOG(WARNING) << "Invalid local location: " << status << " from " << source; + data.local_ = LocalFileLocation(); + if (data.remote_.type() == RemoteFileLocation::Type::Partial) { + data.remote_ = {}; + } + + if (!has_remote && !has_generate) { + return std::move(status); + } + } + } + bool has_local = data.local_.type() == LocalFileLocation::Type::Full; + bool has_location = has_local || has_remote || has_generate; + if (!has_location) { + return Status::Error("No location"); + } + + FileId file_id = next_file_id(); + + LOG(INFO) << "Register file data " << data << " as " << file_id << " from " << source; + // create FileNode + auto file_node_id = next_file_node_id(); + auto &node = file_nodes_[file_node_id]; + node = std::make_unique<FileNode>(std::move(data.local_), std::move(data.remote_), std::move(data.generate_), + data.size_, data.expected_size_, std::move(data.remote_name_), std::move(data.url_), + data.owner_dialog_id_, std::move(data.encryption_key_), file_id, + static_cast<int8>(has_remote)); + node->remote_source_ = file_location_source; + node->pmc_id_ = data.pmc_id_; + get_file_id_info(file_id)->node_id_ = file_node_id; + node->file_ids_.push_back(file_id); + + FileView file_view(get_file_node(file_id)); + + std::vector<FileId> to_merge; + auto register_location = [&](const auto &location, auto &mp) { + auto &other_id = mp[location]; + if (other_id.empty()) { + other_id = file_id; + get_file_id_info(file_id)->pin_flag_ = true; + return true; + } else { + to_merge.push_back(other_id); + return false; + } + }; + bool new_remote = false; + int32 remote_key = 0; + if (file_view.has_remote_location()) { + RemoteInfo info{file_view.remote_location(), file_id}; + remote_key = remote_location_info_.add(info); + auto &stored_info = remote_location_info_.get(remote_key); + if (stored_info.file_id_ == file_id) { + get_file_id_info(file_id)->pin_flag_ = true; + new_remote = true; + } else { + to_merge.push_back(stored_info.file_id_); + if (stored_info.remote_ == file_view.remote_location() && + stored_info.remote_.get_access_hash() != file_view.remote_location().get_access_hash() && + file_location_source == FileLocationSource::FromServer) { + stored_info.remote_ = file_view.remote_location(); + } + } + } + bool new_local = false; + if (file_view.has_local_location()) { + new_local = register_location(file_view.local_location(), local_location_to_file_id_); + } + bool new_generate = false; + if (file_view.has_generate_location()) { + new_generate = register_location(file_view.generate_location(), generate_location_to_file_id_); + } + std::sort(to_merge.begin(), to_merge.end()); + to_merge.erase(std::unique(to_merge.begin(), to_merge.end()), to_merge.end()); + + int new_cnt = new_remote + new_local + new_generate; + if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) { + node->need_load_from_pmc_ = true; + } + bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0; + for (auto id : to_merge) { + // may invalidate node + merge(file_id, id, no_sync_merge).ignore(); + } + + try_flush_node(get_file_node(file_id)); + auto main_file_id = get_file_node(file_id)->main_file_id_; + try_forget_file_id(file_id); + return FileId(main_file_id.get(), remote_key); +} + +// 0 -- choose x +// 1 -- choose y +// 2 -- choose any +static int merge_choose(const LocalFileLocation &x, const LocalFileLocation &y) { + int32 x_type = static_cast<int32>(x.type()); + int32 y_type = static_cast<int32>(y.type()); + if (x_type != y_type) { + return x_type < y_type; + } + return 2; +} + +static int merge_choose(const RemoteFileLocation &x, int8 x_source, const RemoteFileLocation &y, int8 y_source) { + int32 x_type = static_cast<int32>(x.type()); + int32 y_type = static_cast<int32>(y.type()); + if (x_type != y_type) { + return x_type < y_type; + } + // If access_hash changed use a newer one + if (x.type() == RemoteFileLocation::Type::Full) { + if (x.full().get_access_hash() != y.full().get_access_hash()) { + return x_source < y_source; + } + } + return 2; +} +static int merge_choose(const unique_ptr<FullGenerateFileLocation> &x, const unique_ptr<FullGenerateFileLocation> &y) { + int x_type = static_cast<int>(x != nullptr); + int y_type = static_cast<int>(y != nullptr); + if (x_type != y_type) { + return x_type < y_type; + } + return 2; +} + +// -1 -- error +static int merge_choose_size(int64 x, int64 y) { + if (x == 0) { + return 1; + } + if (y == 0) { + return 0; + } + if (x != y) { + return -1; + } + return 2; +} +static int merge_choose_expected_size(int64 x, int64 y) { + if (x == 0) { + return 1; + } + if (y == 0) { + return 0; + } + return 2; +} + +static int merge_choose_name(Slice x, Slice y) { + if (x.empty() != y.empty()) { + return x.empty() > y.empty(); + } + return 2; +} + +static int merge_choose_owner(DialogId x, DialogId y) { + if (x.is_valid() != y.is_valid()) { + return x.is_valid() < y.is_valid(); + } + return 2; +} + +static int merge_choose_main_file_id(FileId a, int8 a_priority, FileId b, int8 b_priority) { + if (a_priority != b_priority) { + return a_priority < b_priority; + } + return a.get() > b.get(); +} + +static int merge_choose_encryption_key(const FileEncryptionKey &a, const FileEncryptionKey &b) { + if (a.empty() != b.empty()) { + return a.empty() > b.empty(); + } + if (a.key_iv_ != b.key_iv_) { + return -1; + } + return 2; +} + +void FileManager::cancel_download(FileNodePtr node) { + if (node->download_id_ == 0) { + return; + } + send_closure(file_load_manager_, &FileLoadManager::cancel, node->download_id_); + node->download_id_ = 0; + node->is_download_started_ = false; + node->set_download_priority(0); +} + +void FileManager::cancel_upload(FileNodePtr node) { + if (node->upload_id_ == 0) { + return; + } + send_closure(file_load_manager_, &FileLoadManager::cancel, node->upload_id_); + node->upload_id_ = 0; + node->set_upload_priority(0); +} + +void FileManager::cancel_generate(FileNodePtr node) { + if (node->generate_id_ == 0) { + return; + } + send_closure(file_generate_manager_, &FileGenerateManager::cancel, node->generate_id_); + node->generate_id_ = 0; + node->generate_was_update_ = false; + node->set_generate_priority(0, 0); +} + +Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { + LOG(INFO) << x_file_id << " VS " << y_file_id; + + if (!x_file_id.is_valid()) { + return Status::Error("First file_id is invalid"); + } + FileNodePtr x_node = no_sync ? get_file_node(x_file_id) : get_sync_file_node(x_file_id); + if (!x_node) { + return Status::Error(PSLICE() << "Can't merge files. First id is invalid: " << x_file_id << " and " << y_file_id); + } + + if (!y_file_id.is_valid()) { + return x_node->main_file_id_; + } + FileNodePtr y_node = get_file_node(y_file_id); + if (!y_node) { + return Status::Error(PSLICE() << "Can't merge files. Second id is invalid: " << x_file_id << " and " << y_file_id); + } + + if (x_file_id == x_node->upload_pause_) { + x_node->upload_pause_ = FileId(); + } + if (x_node.get() == y_node.get()) { + return x_node->main_file_id_; + } + if (y_file_id == y_node->upload_pause_) { + y_node->upload_pause_ = FileId(); + } + + if (x_node->remote_.type() == RemoteFileLocation::Type::Full && + y_node->remote_.type() == RemoteFileLocation::Type::Full && + x_node->remote_.full().get_dc_id() != y_node->remote_.full().get_dc_id()) { + LOG(ERROR) << "File remote location was changed from " << y_node->remote_.full() << " to " + << x_node->remote_.full(); + } + + FileNodePtr nodes[] = {x_node, y_node, x_node}; + FileNodeId node_ids[] = {get_file_id_info(x_file_id)->node_id_, get_file_id_info(y_file_id)->node_id_}; + + int local_i = merge_choose(x_node->local_, y_node->local_); + int remote_i = merge_choose(x_node->remote_, static_cast<int8>(x_node->remote_source_), y_node->remote_, + static_cast<int8>(y_node->remote_source_)); + int generate_i = merge_choose(x_node->generate_, y_node->generate_); + int size_i = merge_choose_size(x_node->size_, y_node->size_); + int expected_size_i = merge_choose_expected_size(x_node->expected_size_, y_node->expected_size_); + int remote_name_i = merge_choose_name(x_node->remote_name_, y_node->remote_name_); + int url_i = merge_choose_name(x_node->url_, y_node->url_); + int owner_i = merge_choose_owner(x_node->owner_dialog_id_, y_node->owner_dialog_id_); + int encryption_key_i = merge_choose_encryption_key(x_node->encryption_key_, y_node->encryption_key_); + int main_file_id_i = merge_choose_main_file_id(x_node->main_file_id_, x_node->main_file_id_priority_, + y_node->main_file_id_, y_node->main_file_id_priority_); + + if (size_i == -1) { + return Status::Error(PSLICE() << "Can't merge files. Different size: " << x_node->size_ << " and " + << y_node->size_); + } + if (encryption_key_i == -1) { + if (nodes[remote_i]->remote_.type() == RemoteFileLocation::Type::Full && + nodes[local_i]->local_.type() != LocalFileLocation::Type::Partial) { + //??? + LOG(ERROR) << "Different encryption key in files, but go Choose same key as remote location"; + encryption_key_i = remote_i; + } else { + return Status::Error("Can't merge files. Different encryption keys"); + } + } + + int node_i = std::make_tuple(y_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) > + std::make_tuple(x_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0); + + auto other_node_i = 1 - node_i; + FileNodePtr node = nodes[node_i]; + FileNodePtr other_node = nodes[other_node_i]; + auto file_view = FileView(node); + + LOG(INFO) << "x_node->pmc_id_ = " << x_node->pmc_id_ << ", y_node->pmc_id_ = " << y_node->pmc_id_ + << ", x_node_size = " << x_node->file_ids_.size() << ", y_node_size = " << y_node->file_ids_.size() + << ", node_i = " << node_i << ", local_i = " << local_i << ", remote_i = " << remote_i + << ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i + << ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i + << ", main_file_id_i = " << main_file_id_i; + if (local_i == other_node_i) { + cancel_download(node); + node->set_local_location(other_node->local_, other_node->local_ready_size_); + node->download_id_ = other_node->download_id_; + node->is_download_started_ |= other_node->is_download_started_; + node->set_download_priority(other_node->download_priority_); + other_node->download_id_ = 0; + other_node->is_download_started_ = false; + other_node->download_priority_ = 0; + + //cancel_generate(node); + //node->set_generate_location(std::move(other_node->generate_)); + //node->generate_id_ = other_node->generate_id_; + //node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_); + //other_node->generate_id_ = 0; + //other_node->generate_was_update_ = false; + //other_node->generate_priority_ = 0; + //other_node->generate_download_priority_ = 0; + //other_node->generate_upload_priority_ = 0; + } else { + cancel_download(other_node); + //cancel_generate(other_node); + } + + if (remote_i == other_node_i) { + cancel_upload(node); + node->set_remote_location(other_node->remote_, other_node->remote_source_, other_node->remote_ready_size_); + node->upload_id_ = other_node->upload_id_; + node->set_upload_priority(other_node->upload_priority_); + node->upload_pause_ = other_node->upload_pause_; + other_node->upload_id_ = 0; + other_node->upload_priority_ = 0; + other_node->upload_pause_ = FileId(); + } else { + cancel_upload(other_node); + } + + if (generate_i == other_node_i) { + cancel_generate(node); + node->set_generate_location(std::move(other_node->generate_)); + node->generate_id_ = other_node->generate_id_; + node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_); + other_node->generate_id_ = 0; + other_node->generate_priority_ = 0; + other_node->generate_download_priority_ = 0; + other_node->generate_upload_priority_ = 0; + } else { + cancel_generate(other_node); + } + + if (size_i == other_node_i) { + node->set_size(other_node->size_); + } + + if (expected_size_i == other_node_i) { + node->set_expected_size(other_node->expected_size_); + } + + if (remote_name_i == other_node_i) { + node->set_remote_name(other_node->remote_name_); + } + + if (url_i == other_node_i) { + node->set_url(other_node->url_); + } + + if (owner_i == other_node_i) { + node->set_owner_dialog_id(other_node->owner_dialog_id_); + } + + if (encryption_key_i == other_node_i) { + node->set_encryption_key(other_node->encryption_key_); + nodes[node_i]->set_encryption_key(nodes[encryption_key_i]->encryption_key_); + } + node->need_load_from_pmc_ |= other_node->need_load_from_pmc_; + node->can_search_locally_ &= other_node->can_search_locally_; + + if (main_file_id_i == other_node_i) { + node->main_file_id_ = other_node->main_file_id_; + node->main_file_id_priority_ = other_node->main_file_id_priority_; + } + + bool send_updates_flag = false; + auto other_pmc_id = other_node->pmc_id_; + node->file_ids_.insert(node->file_ids_.end(), other_node->file_ids_.begin(), other_node->file_ids_.end()); + + for (auto file_id : other_node->file_ids_) { + auto file_id_info = get_file_id_info(file_id); + CHECK(file_id_info->node_id_ == node_ids[other_node_i]) + << node_ids[node_i] << " " << node_ids[other_node_i] << " " << file_id << " " << file_id_info->node_id_; + file_id_info->node_id_ = node_ids[node_i]; + send_updates_flag |= file_id_info->send_updates_flag_; + } + other_node = {}; + + if (send_updates_flag) { + // node might not changed, but other_node might changed, so we need to send update anyway + VLOG(update_file) << "File " << node->main_file_id_ << " has been merged"; + node->on_info_changed(); + } + + // Check is some download/upload queries are ready + for (auto file_id : vector<FileId>(node->file_ids_)) { + auto *info = get_file_id_info(file_id); + if (info->download_priority_ != 0 && file_view.has_local_location()) { + info->download_priority_ = 0; + if (info->download_callback_) { + info->download_callback_->on_download_ok(file_id); + info->download_callback_.reset(); + } + } + if (info->upload_priority_ != 0 && file_view.has_remote_location()) { + info->upload_priority_ = 0; + if (info->upload_callback_) { + info->upload_callback_->on_upload_ok(file_id, nullptr); + info->upload_callback_.reset(); + } + } + } + + file_nodes_[node_ids[other_node_i]] = nullptr; + + run_generate(node); + run_download(node); + run_upload(node, {}); + + if (other_pmc_id != 0) { + // node might not changed, but we need to merge nodes in pmc anyway + node->on_pmc_changed(); + } + try_flush_node(node, node_i != remote_i, node_i != local_i, node_i != generate_i, other_pmc_id); + + return node->main_file_id_; +} + +void FileManager::try_flush_node(FileNodePtr node, bool new_remote, bool new_local, bool new_generate, + FileDbId other_pmc_id) { + if (node->need_pmc_flush()) { + if (file_db_) { + load_from_pmc(node, true, true, true); + flush_to_pmc(node, new_remote, new_local, new_generate); + if (other_pmc_id != 0 && node->pmc_id_ != other_pmc_id) { + file_db_->set_file_data_ref(other_pmc_id, node->pmc_id_); + } + } + node->on_pmc_flushed(); + } + + try_flush_node_info(node); +} + +void FileManager::try_flush_node_info(FileNodePtr node) { + if (node->need_info_flush()) { + for (auto file_id : vector<FileId>(node->file_ids_)) { + auto *info = get_file_id_info(file_id); + if (info->send_updates_flag_) { + VLOG(update_file) << "Send UpdateFile about file " << file_id; + context_->on_file_updated(file_id); + } + } + node->on_info_flushed(); + } +} + +void FileManager::clear_from_pmc(FileNodePtr node) { + if (!file_db_) { + return; + } + if (node->pmc_id_ == 0) { + return; + } + + LOG(INFO) << "Delete files " << format::as_array(node->file_ids_) << " from pmc"; + FileData data; + auto file_view = FileView(node); + if (file_view.has_local_location()) { + data.local_ = node->local_; + } + if (file_view.has_remote_location()) { + data.remote_ = node->remote_; + } + if (file_view.has_generate_location()) { + data.generate_ = std::make_unique<FullGenerateFileLocation>(*node->generate_); + } + file_db_->clear_file_data(node->pmc_id_, data); + node->pmc_id_ = 0; +} + +void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate) { + if (!file_db_) { + return; + } + FileView view(node); + bool create_flag = false; + if (node->pmc_id_ == 0) { + create_flag = true; + node->pmc_id_ = file_db_->create_pmc_id(); + } + + FileData data; + data.pmc_id_ = node->pmc_id_; + data.local_ = node->local_; + if (data.local_.type() == LocalFileLocation::Type::Full) { + prepare_path_for_pmc(data.local_.full().file_type_, data.local_.full().path_); + } + data.remote_ = node->remote_; + if (node->generate_ != nullptr && !begins_with(node->generate_->conversion_, "#file_id#")) { + data.generate_ = std::make_unique<FullGenerateFileLocation>(*node->generate_); + } + + // TODO: not needed when GenerateLocation has constant convertion + if (data.remote_.type() != RemoteFileLocation::Type::Full && data.local_.type() != LocalFileLocation::Type::Full) { + data.local_ = LocalFileLocation(); + data.remote_ = RemoteFileLocation(); + } + + data.size_ = node->size_; + data.expected_size_ = node->expected_size_; + data.remote_name_ = node->remote_name_; + data.encryption_key_ = node->encryption_key_; + data.url_ = node->url_; + data.owner_dialog_id_ = node->owner_dialog_id_; + + file_db_->set_file_data(node->pmc_id_, data, (create_flag || new_remote), (create_flag || new_local), + (create_flag || new_generate)); +} + +FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) { + if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) { + return nullptr; + } + FileNodeId node_id = file_id_info_[file_id.get()].node_id_; + if (node_id == 0) { + return nullptr; + } + if (file_node_id != nullptr) { + *file_node_id = node_id; + } + return file_nodes_[node_id].get(); +} + +FileNodePtr FileManager::get_sync_file_node(FileId file_id) { + auto file_node = get_file_node(file_id); + if (!file_node) { + return {}; + } + load_from_pmc(file_node, true, true, true); + return file_node; +} + +void FileManager::load_from_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate) { + if (!node->need_load_from_pmc_) { + return; + } + auto file_id = node->main_file_id_; + node->need_load_from_pmc_ = false; + if (!file_db_) { + return; + } + auto file_view = get_file_view(file_id); + + FullRemoteFileLocation remote; + FullLocalFileLocation local; + FullGenerateFileLocation generate; + new_remote &= file_view.has_remote_location(); + if (new_remote) { + remote = file_view.remote_location(); + } + new_local &= file_view.has_local_location(); + if (new_local) { + local = get_file_view(file_id).local_location(); + prepare_path_for_pmc(local.file_type_, local.path_); + } + new_generate &= file_view.has_generate_location(); + if (new_generate) { + generate = file_view.generate_location(); + } + + LOG(INFO) << "Load from pmc " << file_id << "/" << file_view.file_id() << ", new_remote = " << new_remote + << ", new_local = " << new_local << ", new_generate = " << new_generate; + auto load = [&](auto location) { + TRY_RESULT(file_data, file_db_->get_file_data_sync(location)); + TRY_RESULT(new_file_id, register_file(std::move(file_data), FileLocationSource::FromDb, "load_from_pmc", false)); + TRY_RESULT(main_file_id, merge(file_id, new_file_id)); + file_id = main_file_id; + return Status::OK(); + }; + if (new_remote) { + load(remote); + } + if (new_local) { + load(local); + } + if (new_generate) { + load(generate); + } + return; +} + +bool FileManager::set_encryption_key(FileId file_id, FileEncryptionKey key) { + auto node = get_sync_file_node(file_id); + if (!node) { + return false; + } + auto view = FileView(node); + if (view.has_local_location() && view.has_remote_location()) { + return false; + } + if (!node->encryption_key_.empty()) { + return false; + } + node->set_encryption_key(std::move(key)); + try_flush_node(node); + return true; +} + +bool FileManager::set_content(FileId file_id, BufferSlice bytes) { + auto node = get_sync_file_node(file_id); + if (!node) { + return false; + } + + if (node->local_.type() == LocalFileLocation::Type::Full) { + // There was no download so we don't need an update + return true; + } + + if (node->download_priority_ == FROM_BYTES_PRIORITY) { + return true; + } + + cancel_download(node); + + auto *file_info = get_file_id_info(file_id); + file_info->download_priority_ = FROM_BYTES_PRIORITY; + + node->set_download_priority(FROM_BYTES_PRIORITY); + + QueryId id = queries_container_.create(Query{file_id, Query::SetContent}); + node->download_id_ = id; + node->is_download_started_ = true; + send_closure(file_load_manager_, &FileLoadManager::from_bytes, id, node->remote_.full().file_type_, std::move(bytes), + node->suggested_name()); + return true; +} + +void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) { + auto node = get_sync_file_node(file_id); + if (!node) { + return promise.set_error(Status::Error("Unknown file_id")); + } + auto status = check_local_location(node); + status.ignore(); + + auto file_view = FileView(node); + if (!file_view.has_local_location()) { + return promise.set_error(Status::Error("No local location")); + } + + send_closure(file_load_manager_, &FileLoadManager::get_content, node->local_.full(), std::move(promise)); +} + +void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char *source) { + LOG(INFO) << "Trying to delete file " << file_id << " from " << source; + auto node = get_sync_file_node(file_id); + if (!node) { + return promise.set_value(Unit()); + } + + auto file_view = FileView(node); + + // TODO: review delete condition + if (file_view.has_local_location()) { + if (begins_with(file_view.local_location().path_, get_files_dir(file_view.get_type()))) { + LOG(INFO) << "Unlink file " << file_id << " at " << file_view.local_location().path_; + clear_from_pmc(node); + + unlink(file_view.local_location().path_).ignore(); + context_->on_new_file(-file_view.size()); + node->set_local_location(LocalFileLocation(), 0); + try_flush_node(node); + } + } else { + if (file_view.get_type() == FileType::Encrypted) { + clear_from_pmc(node); + } + if (node->local_.type() == LocalFileLocation::Type::Partial) { + LOG(INFO) << "Unlink partial file " << file_id << " at " << node->local_.partial().path_; + unlink(node->local_.partial().path_).ignore(); + node->set_local_location(LocalFileLocation(), 0); + try_flush_node(node); + } + } + + promise.set_value(Unit()); +} + +void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> callback, int32 new_priority) { + auto node = get_sync_file_node(file_id); + if (!node) { + if (callback) { + callback->on_download_error(file_id, Status::Error("File not found")); + } + return; + } + + if (node->local_.type() == LocalFileLocation::Type::Full) { + auto status = check_local_location(node); + if (status.is_error()) { + LOG(WARNING) << "Need to redownload file " << file_id << ": " << status.error(); + } else { + if (callback) { + callback->on_download_ok(file_id); + } + return; + } + } else if (node->local_.type() == LocalFileLocation::Type::Partial) { + auto status = check_local_location(node); + if (status.is_error()) { + LOG(WARNING) << "Need to download file " << file_id << " from beginning: " << status.error(); + } + } + + FileView file_view(node); + if (!file_view.can_download_from_server() && !file_view.can_generate()) { + if (callback) { + callback->on_download_error(file_id, Status::Error("Can't download or generate file")); + } + return; + } + + if (new_priority == -1) { + if (node->is_download_started_) { + return; + } + new_priority = 0; + } + + auto *file_info = get_file_id_info(file_id); + CHECK(new_priority == 0 || callback); + file_info->download_priority_ = narrow_cast<int8>(new_priority); + file_info->download_callback_ = std::move(callback); + // TODO: send current progress? + + run_generate(node); + run_download(node); + + try_flush_node(node); +} + +void FileManager::run_download(FileNodePtr node) { + if (node->need_load_from_pmc_) { + return; + } + if (node->generate_id_) { + return; + } + auto file_view = FileView(node); + if (!file_view.can_download_from_server()) { + return; + } + int8 priority = 0; + for (auto id : node->file_ids_) { + auto *info = get_file_id_info(id); + if (info->download_priority_ > priority) { + priority = info->download_priority_; + } + } + + auto old_priority = node->download_priority_; + node->set_download_priority(priority); + + if (priority == 0) { + if (old_priority != 0) { + cancel_download(node); + } + return; + } + + if (old_priority != 0) { + CHECK(node->download_id_ != 0); + send_closure(file_load_manager_, &FileLoadManager::update_priority, node->download_id_, priority); + return; + } + + CHECK(node->download_id_ == 0); + CHECK(!node->file_ids_.empty()); + auto file_id = node->file_ids_.back(); + QueryId id = queries_container_.create(Query{file_id, Query::Download}); + node->download_id_ = id; + node->is_download_started_ = false; + send_closure(file_load_manager_, &FileLoadManager::download, id, node->remote_.full(), node->local_, node->size_, + node->suggested_name(), node->encryption_key_, node->can_search_locally_, priority); +} + +void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback, + int32 new_priority, uint64 upload_order) { + LOG(INFO) << "Resume upload of file " << file_id << " with priority " << new_priority; + + auto node = get_sync_file_node(file_id); + if (!node) { + if (callback) { + callback->on_upload_error(file_id, Status::Error("Wrong file id to upload")); + } + return; + } + if (node->upload_pause_ == file_id) { + node->upload_pause_ = FileId(); + } + FileView file_view(node); + if (file_view.has_remote_location() && file_view.get_type() != FileType::Thumbnail && + file_view.get_type() != FileType::EncryptedThumbnail) { + if (callback) { + callback->on_upload_ok(file_id, nullptr); + } + return; + } + + if (file_view.has_local_location()) { + auto status = check_local_location(node); + if (status.is_error()) { + LOG(INFO) << "Full local location of file " << file_id << " for upload is invalid: " << status; + } + } + + if (!file_view.has_local_location() && !file_view.has_generate_location()) { + if (callback) { + callback->on_upload_error(file_id, Status::Error("Need full local (or generate) location for upload")); + } + return; + } + + auto *file_info = get_file_id_info(file_id); + CHECK(new_priority == 0 || callback); + file_info->upload_order_ = upload_order; + file_info->upload_priority_ = narrow_cast<int8>(new_priority); + file_info->upload_callback_ = std::move(callback); + // TODO: send current progress? + + run_generate(node); + run_upload(node, std::move(bad_parts)); + try_flush_node(node); +} + +bool FileManager::delete_partial_remote_location(FileId file_id) { + auto node = get_sync_file_node(file_id); + if (!node) { + LOG(INFO) << "Wrong file id " << file_id; + return false; + } + if (node->upload_pause_ == file_id) { + node->upload_pause_ = FileId(); + } + if (node->remote_.type() == RemoteFileLocation::Type::Full) { + LOG(INFO) << "File " << file_id << " is already uploaded"; + return true; + } + + node->set_remote_location(RemoteFileLocation(), FileLocationSource::None, 0); + auto *file_info = get_file_id_info(file_id); + file_info->upload_priority_ = 0; + + if (node->local_.type() != LocalFileLocation::Type::Full) { + LOG(INFO) << "Need full local location to upload file " << file_id; + return false; + } + + auto status = check_local_location(node); + if (status.is_error()) { + LOG(INFO) << "Need full local location to upload file " << file_id << ": " << status; + return false; + } + + run_upload(node, std::vector<int>()); + try_flush_node(node); + return true; +} + +void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, + Promise<> promise) { + send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size, + local_prefix_size, std::move(promise)); +} +void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) { + send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status), + std::move(promise)); +} + +void FileManager::run_generate(FileNodePtr node) { + if (node->need_load_from_pmc_) { + return; + } + FileView file_view(node); + if (file_view.has_local_location() || file_view.can_download_from_server() || !file_view.can_generate()) { + return; + } + + int8 download_priority = 0; + int8 upload_priority = 0; + FileId file_id = node->main_file_id_; + for (auto id : node->file_ids_) { + auto *info = get_file_id_info(id); + if (info->download_priority_ > download_priority) { + download_priority = info->download_priority_; + if (download_priority > upload_priority) { + file_id = id; + } + } + if (info->upload_priority_ > upload_priority) { + upload_priority = info->upload_priority_; + if (upload_priority > download_priority) { + file_id = id; + } + } + } + + auto old_priority = node->generate_priority_; + node->set_generate_priority(download_priority, upload_priority); + + if (node->generate_priority_ == 0) { + if (old_priority != 0) { + LOG(INFO) << "Cancel file " << file_id << " generation"; + cancel_generate(node); + } + return; + } + + if (old_priority != 0) { + LOG(INFO) << "TODO: change file " << file_id << " generation priority"; + return; + } + + QueryId id = queries_container_.create(Query{file_id, Query::Generate}); + node->generate_id_ = id; + send_closure(file_generate_manager_, &FileGenerateManager::generate_file, id, *node->generate_, node->local_, + node->suggested_name(), [file_manager = this, id] { + class Callback : public FileGenerateCallback { + ActorId<FileManager> actor_; + uint64 query_id_; + + public: + Callback(ActorId<FileManager> actor, QueryId id) : actor_(std::move(actor)), query_id_(id) { + } + void on_partial_generate(const PartialLocalFileLocation &partial_local, + int32 expected_size) override { + send_closure(actor_, &FileManager::on_partial_generate, query_id_, partial_local, expected_size); + } + void on_ok(const FullLocalFileLocation &local) override { + send_closure(actor_, &FileManager::on_generate_ok, query_id_, local); + } + void on_error(Status error) override { + send_closure(actor_, &FileManager::on_error, query_id_, std::move(error)); + } + }; + return std::make_unique<Callback>(file_manager->actor_id(file_manager), id); + }()); + + LOG(INFO) << "File " << file_id << " generate request has sent to FileGenerateManager"; +} + +void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) { + if (node->need_load_from_pmc_) { + return; + } + if (node->upload_pause_.is_valid()) { + return; + } + FileView file_view(node); + if (!file_view.has_local_location()) { + if (node->get_by_hash_ || node->generate_id_ == 0 || !node->generate_was_update_) { + return; + } + } + int8 priority = 0; + FileId file_id = node->main_file_id_; + for (auto id : node->file_ids_) { + auto *info = get_file_id_info(id); + if (info->upload_priority_ > priority) { + priority = info->upload_priority_; + file_id = id; + } + } + + auto old_priority = node->upload_priority_; + node->set_upload_priority(priority); + + if (priority == 0) { + if (old_priority != 0) { + LOG(INFO) << "Cancel file " << file_id << " uploading"; + cancel_upload(node); + } + return; + } + + // create encryption key if necessary + if (((file_view.has_generate_location() && file_view.generate_location().file_type_ == FileType::Encrypted) || + (file_view.has_local_location() && file_view.local_location().file_type_ == FileType::Encrypted)) && + file_view.encryption_key().empty()) { + CHECK(!node->file_ids_.empty()); + bool success = set_encryption_key(node->file_ids_[0], FileEncryptionKey::create()); + LOG_IF(FATAL, !success) << "Failed to set encryption key for file " << file_id; + } + + if (old_priority != 0) { + LOG(INFO) << "File " << file_id << " is already uploading"; + CHECK(node->upload_id_ != 0); + send_closure(file_load_manager_, &FileLoadManager::update_priority, node->upload_id_, narrow_cast<int8>(-priority)); + return; + } + + CHECK(node->upload_id_ == 0); + if (node->remote_.type() != RemoteFileLocation::Type::Partial && node->get_by_hash_) { + QueryId id = queries_container_.create(Query{file_id, Query::UploadByHash}); + node->upload_id_ = id; + + send_closure(file_load_manager_, &FileLoadManager::upload_by_hash, id, node->local_.full(), node->size_, + narrow_cast<int8>(-priority)); + return; + } + + QueryId id = queries_container_.create(Query{file_id, Query::Upload}); + node->upload_id_ = id; + send_closure(file_load_manager_, &FileLoadManager::upload, id, node->local_, node->remote_, node->size_, + node->encryption_key_, narrow_cast<int8>(bad_parts.empty() ? -priority : priority), + std::move(bad_parts)); + + LOG(INFO) << "File " << file_id << " upload request has sent to FileLoadManager"; +} + +void FileManager::upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority, + uint64 upload_order) { + return resume_upload(file_id, std::vector<int>(), std::move(callback), new_priority, upload_order); +} + +// is't quite stupid, yep +// 0x00 <count of zeroes> +static string zero_decode(Slice s) { + string res; + for (size_t n = s.size(), i = 0; i < n; i++) { + if (i + 1 < n && s[i] == 0) { + res.append(static_cast<unsigned char>(s[i + 1]), 0); + i++; + continue; + } + res.push_back(s[i]); + } + return res; +} + +static string zero_encode(Slice s) { + string res; + for (size_t n = s.size(), i = 0; i < n; i++) { + res.push_back(s[i]); + if (s[i] == 0) { + unsigned char cnt = 1; + while (cnt < 250 && i + cnt < n && s[i + cnt] == 0) { + cnt++; + } + res.push_back(cnt); + i += cnt - 1; + } + } + return res; +} + +static bool is_document_type(FileType type) { + return type == FileType::Document || type == FileType::Sticker || type == FileType::Audio || + type == FileType::Animation; +} + +string FileManager::get_persistent_id(const FullRemoteFileLocation &location) { + auto binary = serialize(location); + + binary = zero_encode(binary); + binary.push_back(PERSISTENT_ID_VERSION); + return base64url_encode(binary); +} + +Result<string> FileManager::to_persistent_id(FileId file_id) { + auto view = get_file_view(file_id); + if (view.empty()) { + return Status::Error(10, "Unknown file id"); + } + if (!view.has_remote_location()) { + return Status::Error(10, "File has no persistent id"); + } + return get_persistent_id(view.remote_location()); +} + +Result<FileId> FileManager::from_persistent_id(CSlice persistent_id, FileType file_type) { + if (persistent_id.find('.') != string::npos) { + string input_url = persistent_id.str(); // TODO do not copy persistent_id + TRY_RESULT(http_url, parse_url(input_url)); + auto url = http_url.get_url(); + if (!clean_input_string(url)) { + return Status::Error(400, "URL must be in UTF-8"); + } + return register_url(std::move(url), file_type, FileLocationSource::FromUser, DialogId()); + } + + auto r_binary = base64url_decode(persistent_id); + if (r_binary.is_error()) { + return Status::Error(10, "Wrong remote file id specified: " + r_binary.error().message().str()); + } + auto binary = r_binary.move_as_ok(); + if (binary.empty()) { + return Status::Error(10, "Remote file id can't be empty"); + } + if (binary.back() != PERSISTENT_ID_VERSION) { + return Status::Error(10, "Wrong remote file id specified: can't unserialize it. Wrong last symbol"); + } + binary.pop_back(); + binary = zero_decode(binary); + FullRemoteFileLocation remote_location; + auto status = unserialize(remote_location, binary); + if (status.is_error()) { + return Status::Error(10, "Wrong remote file id specified: can't unserialize it"); + } + auto &real_file_type = remote_location.file_type_; + if (is_document_type(real_file_type) && is_document_type(file_type)) { + real_file_type = file_type; + } else if (real_file_type != file_type && file_type != FileType::Temp) { + return Status::Error(10, "Type of file mismatch"); + } + FileData data; + data.remote_ = RemoteFileLocation(std::move(remote_location)); + return register_file(std::move(data), FileLocationSource::FromUser, "from_persistent_id", false).move_as_ok(); +} + +FileView FileManager::get_file_view(FileId file_id) const { + auto file_node = get_file_node(file_id); + if (!file_node) { + return FileView(); + } + return FileView(file_node); +} +FileView FileManager::get_sync_file_view(FileId file_id) { + auto file_node = get_sync_file_node(file_id); + if (!file_node) { + return FileView(); + } + return FileView(file_node); +} + +tl_object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool with_main_file_id) { + auto file_view = get_sync_file_view(file_id); + + if (file_view.empty()) { + return td_api::make_object<td_api::file>(0, 0, 0, td_api::make_object<td_api::localFile>(), + td_api::make_object<td_api::remoteFile>()); + } + + string persistent_file_id; + if (file_view.has_remote_location()) { + persistent_file_id = get_persistent_id(file_view.remote_location()); + } else if (file_view.has_url()) { + persistent_file_id = file_view.url(); + } + + int32 size = narrow_cast<int32>(file_view.size()); + int32 expected_size = narrow_cast<int32>(file_view.expected_size()); + int32 local_size = narrow_cast<int32>(file_view.local_size()); + int32 local_total_size = narrow_cast<int32>(file_view.local_total_size()); + int32 remote_size = narrow_cast<int32>(file_view.remote_size()); + string path = file_view.path(); + bool can_be_downloaded = file_view.can_download_from_server() || file_view.can_generate(); + bool can_be_deleted = file_view.can_delete(); + + auto result_file_id = file_id; + auto *file_info = get_file_id_info(result_file_id); + if (with_main_file_id) { + if (!file_info->send_updates_flag_) { + result_file_id = file_view.file_id(); + } + file_info = get_file_id_info(file_view.file_id()); + } + file_info->send_updates_flag_ = true; + VLOG(update_file) << "Send file " << file_id << " as " << result_file_id << " and update send_updates_flag_ for file " + << (with_main_file_id ? file_view.file_id() : result_file_id); + + return td_api::make_object<td_api::file>( + result_file_id.get(), size, expected_size, + td_api::make_object<td_api::localFile>(std::move(path), can_be_downloaded, can_be_deleted, + file_view.is_downloading(), file_view.has_local_location(), local_size, + local_total_size), + td_api::make_object<td_api::remoteFile>(std::move(persistent_file_id), file_view.is_uploading(), + file_view.has_remote_location(), remote_size)); +} + +Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> result, bool is_encrypted, + bool allow_zero) { + TRY_RESULT(file_id, std::move(result)); + if (allow_zero && !file_id.is_valid()) { + return FileId(); + } + + auto file_node = get_file_node(file_id); + if (!file_node) { + return Status::Error(6, "File not found"); + } + auto file_view = FileView(file_node); + FileType real_type = file_view.get_type(); + if (!is_encrypted) { + if (real_type != type && !(real_type == FileType::Temp && file_view.has_url()) && + !(is_document_type(real_type) && is_document_type(type))) { + // TODO: send encrypted file to unencrypted chat + return Status::Error(6, "Type of file mismatch"); + } + } + + if (!file_view.has_remote_location()) { + // TODO why not return file_id here? We will dup it anyway + // But it will not be duped if has_input_media(), so for now we can't return main_file_id + return dup_file_id(file_id); + } + return file_node->main_file_id_; +} + +Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> &thumbnail_input_file, + DialogId owner_dialog_id, bool is_encrypted) { + if (thumbnail_input_file == nullptr) { + return Status::Error(6, "inputThumbnail not specified"); + } + + switch (thumbnail_input_file->get_id()) { + case td_api::inputFileLocal::ID: { + const string &path = static_cast<const td_api::inputFileLocal *>(thumbnail_input_file.get())->path_; + return register_local( + FullLocalFileLocation(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, path, 0), + owner_dialog_id, 0, false); + } + case td_api::inputFileId::ID: + return Status::Error(6, "InputFileId is not supported for thumbnails"); + case td_api::inputFileRemote::ID: + return Status::Error(6, "InputFileRemote is not supported for thumbnails"); + case td_api::inputFileGenerated::ID: { + auto *generated_thumbnail = static_cast<const td_api::inputFileGenerated *>(thumbnail_input_file.get()); + return register_generate(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, + FileLocationSource::FromUser, generated_thumbnail->original_path_, + generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_); + } + default: + UNREACHABLE(); + return Status::Error(500, "Unreachable"); + } +} + +Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr<td_api::InputFile> &file, + DialogId owner_dialog_id, bool allow_zero, bool is_encrypted, + bool get_by_hash) { + if (is_encrypted) { + get_by_hash = false; + } + if (!file) { + if (allow_zero) { + return FileId(); + } + return Status::Error(6, "InputFile not specified"); + } + + auto r_file_id = [&]() -> Result<FileId> { + switch (file->get_id()) { + case td_api::inputFileLocal::ID: { + const string &path = static_cast<const td_api::inputFileLocal *>(file.get())->path_; + if (allow_zero && path.empty()) { + return FileId(); + } + return register_local(FullLocalFileLocation(is_encrypted ? FileType::Encrypted : type, path, 0), + owner_dialog_id, 0, get_by_hash); + } + case td_api::inputFileId::ID: { + FileId file_id(static_cast<const td_api::inputFileId *>(file.get())->id_, 0); + if (!file_id.is_valid()) { + return FileId(); + } + return file_id; + } + case td_api::inputFileRemote::ID: { + const string &file_persistent_id = static_cast<const td_api::inputFileRemote *>(file.get())->id_; + if (allow_zero && file_persistent_id.empty()) { + return FileId(); + } + return from_persistent_id(file_persistent_id, type); + } + case td_api::inputFileGenerated::ID: { + auto *generated_file = static_cast<const td_api::inputFileGenerated *>(file.get()); + return register_generate(is_encrypted ? FileType::Encrypted : type, FileLocationSource::FromUser, + generated_file->original_path_, generated_file->conversion_, owner_dialog_id, + generated_file->expected_size_); + } + default: + UNREACHABLE(); + return Status::Error(500, "Unreachable"); + } + }(); + + return check_input_file_id(type, std::move(r_file_id), is_encrypted, allow_zero); +} + +vector<tl_object_ptr<telegram_api::InputDocument>> FileManager::get_input_documents(const vector<FileId> &file_ids) { + vector<tl_object_ptr<telegram_api::InputDocument>> result; + result.reserve(file_ids.size()); + for (auto file_id : file_ids) { + auto file_view = get_file_view(file_id); + CHECK(!file_view.empty()); + CHECK(file_view.has_remote_location()); + CHECK(!file_view.remote_location().is_web()); + result.push_back(file_view.remote_location().as_input_document()); + } + return result; +} + +FileId FileManager::next_file_id() { + if (!empty_file_ids_.empty()) { + auto res = empty_file_ids_.back(); + empty_file_ids_.pop_back(); + return FileId{res, 0}; + } + FileId res(static_cast<int32>(file_id_info_.size()), 0); + // LOG(ERROR) << "NEXT file_id " << res; + file_id_info_.push_back({}); + return res; +} + +FileManager::FileNodeId FileManager::next_file_node_id() { + FileNodeId res = static_cast<FileNodeId>(file_nodes_.size()); + file_nodes_.emplace_back(nullptr); + return res; +} + +void FileManager::on_start_download(QueryId query_id) { + auto query = queries_container_.get(query_id); + CHECK(query != nullptr); + + auto file_id = query->file_id_; + auto file_node = get_file_node(file_id); + if (!file_node) { + return; + } + if (file_node->download_id_ != query_id) { + return; + } + + LOG(DEBUG) << "Start to download part of file " << file_id; + file_node->is_download_started_ = true; +} + +void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, + int64 ready_size) { + auto query = queries_container_.get(query_id); + CHECK(query != nullptr); + + auto file_id = query->file_id_; + auto file_node = get_file_node(file_id); + if (!file_node) { + return; + } + if (file_node->download_id_ != query_id) { + return; + } + + file_node->set_local_location(LocalFileLocation(partial_local), ready_size); + try_flush_node(file_node); +} + +void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLocation &partial_remote, + int64 ready_size) { + auto query = queries_container_.get(query_id); + CHECK(query != nullptr); + + auto file_id = query->file_id_; + auto file_node = get_file_node(file_id); + if (!file_node) { + return; + } + if (file_node->upload_id_ != query_id) { + return; + } + + file_node->set_remote_location(RemoteFileLocation(partial_remote), FileLocationSource::None, ready_size); + try_flush_node(file_node); +} +void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &local, int64 size) { + auto file_id = finish_query(query_id).first.file_id_; + LOG(INFO) << "ON DOWNLOAD OK file " << file_id << " of size " << size; + auto r_new_file_id = register_local(local, DialogId(), size); + if (r_new_file_id.is_error()) { + LOG(ERROR) << "Can't register local file after download: " << r_new_file_id.error(); + } else { + context_->on_new_file(get_file_view(r_new_file_id.ok()).size()); + LOG_STATUS(merge(r_new_file_id.ok(), file_id)); + } +} +void FileManager::on_upload_ok(QueryId query_id, FileType file_type, const PartialRemoteFileLocation &partial_remote, + int64 size) { + CHECK(partial_remote.ready_part_count_ == partial_remote.part_count_); + auto some_file_id = finish_query(query_id).first.file_id_; + LOG(INFO) << "ON UPLOAD OK file " << some_file_id << " of size " << size; + + auto file_node = get_file_node(some_file_id); + if (!file_node) { + return; + } + + FileId file_id; + uint64 file_id_upload_order{std::numeric_limits<uint64>::max()}; + for (auto id : file_node->file_ids_) { + auto *info = get_file_id_info(id); + if (info->upload_priority_ != 0 && info->upload_order_ < file_id_upload_order) { + file_id = id; + file_id_upload_order = info->upload_order_; + } + } + if (!file_id.is_valid()) { + return; + } + + auto *file_info = get_file_id_info(file_id); + file_info->upload_priority_ = 0; + file_info->download_priority_ = 0; + + FileView file_view(file_node); + string file_name = get_file_name(file_type, file_view.suggested_name()); + + if (file_view.is_encrypted()) { + tl_object_ptr<telegram_api::InputEncryptedFile> input_file; + if (partial_remote.is_big_) { + input_file = make_tl_object<telegram_api::inputEncryptedFileBigUploaded>( + partial_remote.file_id_, partial_remote.part_count_, file_view.encryption_key().calc_fingerprint()); + } else { + input_file = make_tl_object<telegram_api::inputEncryptedFileUploaded>( + partial_remote.file_id_, partial_remote.part_count_, "", file_view.encryption_key().calc_fingerprint()); + } + if (file_info->upload_callback_) { + file_info->upload_callback_->on_upload_encrypted_ok(file_id, std::move(input_file)); + file_node->upload_pause_ = file_id; + file_info->upload_callback_.reset(); + } + } else { + tl_object_ptr<telegram_api::InputFile> input_file; + if (partial_remote.is_big_) { + input_file = make_tl_object<telegram_api::inputFileBig>(partial_remote.file_id_, partial_remote.part_count_, + std::move(file_name)); + } else { + input_file = make_tl_object<telegram_api::inputFile>(partial_remote.file_id_, partial_remote.part_count_, + std::move(file_name), ""); + } + if (file_info->upload_callback_) { + file_info->upload_callback_->on_upload_ok(file_id, std::move(input_file)); + file_node->upload_pause_ = file_id; + file_info->upload_callback_.reset(); + } + } +} + +void FileManager::on_upload_full_ok(QueryId query_id, const FullRemoteFileLocation &remote) { + LOG(INFO) << "ON UPLOAD OK"; + auto file_id = finish_query(query_id).first.file_id_; + auto new_file_id = register_remote(remote, FileLocationSource::FromServer, DialogId(), 0, 0, ""); + LOG_STATUS(merge(new_file_id, file_id)); +} + +void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLocation &partial_local, + int32 expected_size) { + LOG(INFO) << "on_parital_generate: " << partial_local.path_ << " " << partial_local.ready_part_count_; + auto query = queries_container_.get(query_id); + CHECK(query != nullptr); + + auto file_id = query->file_id_; + auto file_node = get_file_node(file_id); + if (!file_node) { + return; + } + if (file_node->generate_id_ != query_id) { + return; + } + file_node->set_local_location(LocalFileLocation(partial_local), 0); + // TODO check for size and local_size, abort generation if needed + if (expected_size != 0) { + file_node->set_expected_size(expected_size); + } + if (!file_node->generate_was_update_) { + file_node->generate_was_update_ = true; + run_upload(file_node, {}); + } + if (file_node->upload_id_ != 0) { + send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_, + LocalFileLocation(partial_local)); + } + + try_flush_node(file_node); +} +void FileManager::on_generate_ok(QueryId query_id, const FullLocalFileLocation &local) { + LOG(INFO) << "on_ok_generate: " << local; + Query query; + bool was_active; + std::tie(query, was_active) = finish_query(query_id); + auto generate_file_id = query.file_id_; + + auto file_node = get_file_node(generate_file_id); + if (!file_node) { + return; + } + + auto old_upload_id = file_node->upload_id_; + + auto r_new_file_id = register_local(local, DialogId(), 0); + Status status; + if (r_new_file_id.is_error()) { + status = Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error()); + } else { + auto result = merge(r_new_file_id.ok(), generate_file_id); + if (result.is_error()) { + status = result.move_as_error(); + } + } + file_node = get_file_node(generate_file_id); + if (status.is_error()) { + return on_error_impl(file_node, query.type_, was_active, std::move(status)); + } + + CHECK(file_node); + context_->on_new_file(FileView(file_node).size()); + + run_upload(file_node, {}); + + if (was_active) { + if (old_upload_id != 0 && old_upload_id == file_node->upload_id_) { + send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_, + LocalFileLocation(local)); + } + } +} + +void FileManager::on_error(QueryId query_id, Status status) { + Query query; + bool was_active; + std::tie(query, was_active) = finish_query(query_id); + auto node = get_file_node(query.file_id_); + if (!node) { + LOG(ERROR) << "Can't find file node for " << query.file_id_ << " " << status; + return; + } + + if (query.type_ == Query::UploadByHash) { + LOG(INFO) << "Upload By Hash failed: " << status << ", restart upload"; + node->get_by_hash_ = false; + run_upload(node, {}); + return; + } + on_error_impl(node, query.type_, was_active, std::move(status)); +} + +void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type, bool was_active, Status status) { + SCOPE_EXIT { + try_flush_node(node); + }; + if (status.code() != 1) { + LOG(WARNING) << "Failed to upload/download/generate file: " << status << ". Query type = " << type + << ". File type is " << file_type_name[static_cast<int32>(FileView(node).get_type())]; + if (status.code() == 0) { + // Remove partial locations + if (node->local_.type() == LocalFileLocation::Type::Partial && status.message() != "FILE_UPLOAD_RESTART") { + LOG(INFO) << "Unlink file " << node->local_.partial().path_; + unlink(node->local_.partial().path_).ignore(); + node->set_local_location(LocalFileLocation(), 0); + } + if (node->remote_.type() == RemoteFileLocation::Type::Partial) { + node->set_remote_location(RemoteFileLocation(), FileLocationSource::None, 0); + } + status = Status::Error(400, status.message()); + } + } + + if (status.message() == "FILE_UPLOAD_RESTART") { + run_upload(node, {}); + return; + } + if (status.message() == "FILE_DOWNLOAD_RESTART") { + node->can_search_locally_ = false; + run_download(node); + return; + } + + if (!was_active) { + return; + } + + // Stop everything on error + cancel_generate(node); + cancel_download(node); + cancel_upload(node); + + for (auto file_id : vector<FileId>(node->file_ids_)) { + auto *info = get_file_id_info(file_id); + if (info->download_priority_ != 0) { + info->download_priority_ = 0; + if (info->download_callback_) { + info->download_callback_->on_download_error(file_id, status.clone()); + info->download_callback_.reset(); + } + } + if (info->upload_priority_ != 0) { + info->upload_priority_ = 0; + if (info->upload_callback_) { + info->upload_callback_->on_upload_error(file_id, status.clone()); + info->upload_callback_.reset(); + } + } + } +} + +std::pair<FileManager::Query, bool> FileManager::finish_query(QueryId query_id) { + SCOPE_EXIT { + queries_container_.erase(query_id); + }; + auto query = queries_container_.get(query_id); + CHECK(query != nullptr); + + auto res = *query; + auto node = get_file_node(res.file_id_); + if (!node) { + return std::make_pair(res, false); + } + bool was_active = false; + if (node->generate_id_ == query_id) { + node->generate_id_ = 0; + node->generate_was_update_ = false; + node->set_generate_priority(0, 0); + was_active = true; + } + if (node->download_id_ == query_id) { + node->download_id_ = 0; + node->is_download_started_ = false; + node->set_download_priority(0); + was_active = true; + } + if (node->upload_id_ == query_id) { + node->upload_id_ = 0; + node->set_upload_priority(0); + was_active = true; + } + return std::make_pair(res, was_active); +} + +FullRemoteFileLocation *FileManager::get_remote(int32 key) { + if (key == 0) { + return nullptr; + } + return &remote_location_info_.get(key).remote_; +} + +void FileManager::hangup() { + file_db_.reset(); + file_generate_manager_.reset(); + file_load_manager_.reset(); + stop(); +} + +void FileManager::tear_down() { + parent_.reset(); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileManager.h b/libs/tdlib/td/td/telegram/files/FileManager.h new file mode 100644 index 0000000000..10f961c213 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileManager.h @@ -0,0 +1,475 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/td_api.h" +#include "td/telegram/telegram_api.h" + +#include "td/telegram/DialogId.h" +#include "td/telegram/files/FileDb.h" +#include "td/telegram/files/FileGenerateManager.h" +#include "td/telegram/files/FileId.h" +#include "td/telegram/files/FileLoadManager.h" +#include "td/telegram/files/FileLocation.h" +#include "td/telegram/files/FileStats.h" + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/Container.h" +#include "td/utils/Enumerator.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <map> +#include <memory> +#include <set> +#include <utility> + +namespace td { + +enum class FileLocationSource : int8 { None, FromUser, FromDb, FromServer }; + +class FileNode { + public: + FileNode(LocalFileLocation local, RemoteFileLocation remote, unique_ptr<FullGenerateFileLocation> generate, + int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id, + FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority) + : local_(std::move(local)) + , remote_(std::move(remote)) + , generate_(std::move(generate)) + , size_(size) + , expected_size_(expected_size) + , remote_name_(std::move(remote_name)) + , url_(std::move(url)) + , owner_dialog_id_(owner_dialog_id) + , encryption_key_(std::move(key)) + , main_file_id_(main_file_id) + , main_file_id_priority_(main_file_id_priority) { + } + void set_local_location(const LocalFileLocation &local, int64 ready_size); + void set_remote_location(const RemoteFileLocation &remote, FileLocationSource source, int64 ready_size); + void set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate); + void set_size(int64 size); + void set_expected_size(int64 expected_size); + void set_remote_name(string remote_name); + void set_url(string url); + void set_owner_dialog_id(DialogId owner_id); + void set_encryption_key(FileEncryptionKey key); + + void set_download_priority(int8 priority); + void set_upload_priority(int8 priority); + void set_generate_priority(int8 download_priority, int8 upload_priority); + + void on_changed(); + void on_info_changed(); + void on_pmc_changed(); + + bool need_info_flush() const; + bool need_pmc_flush() const; + + void on_pmc_flushed(); + void on_info_flushed(); + + string suggested_name() const; + + private: + friend class FileView; + friend class FileManager; + + LocalFileLocation local_; + FileLoadManager::QueryId upload_id_ = 0; + int64 local_ready_size_ = 0; + + RemoteFileLocation remote_; + FileLoadManager::QueryId download_id_ = 0; + int64 remote_ready_size_ = 0; + + unique_ptr<FullGenerateFileLocation> generate_; + FileLoadManager::QueryId generate_id_ = 0; + + int64 size_ = 0; + int64 expected_size_ = 0; + string remote_name_; + string url_; + DialogId owner_dialog_id_; + FileEncryptionKey encryption_key_; + FileDbId pmc_id_ = 0; + std::vector<FileId> file_ids_; + + FileId main_file_id_; + + FileId upload_pause_; + int8 upload_priority_ = 0; + int8 download_priority_ = 0; + int8 generate_priority_ = 0; + + int8 generate_download_priority_ = 0; + int8 generate_upload_priority_ = 0; + + int8 main_file_id_priority_ = 0; + + FileLocationSource remote_source_ = FileLocationSource::FromUser; + + bool get_by_hash_ = false; + bool can_search_locally_{true}; + + bool is_download_started_ = false; + bool generate_was_update_ = false; + + bool need_load_from_pmc_ = false; + + bool pmc_changed_flag_{false}; + bool info_changed_flag_{false}; +}; + +class FileManager; + +class FileNodePtr { + public: + FileNodePtr() = default; + FileNodePtr(FileId file_id, FileManager *file_manager) : file_id_(file_id), file_manager_(file_manager) { + } + + FileNode *operator->() const; + FileNode &operator*() const; + FileNode *get() const; + FullRemoteFileLocation *get_remote() const; + explicit operator bool() const; + + private: + FileId file_id_; + FileManager *file_manager_ = nullptr; + FileNode *get_unsafe() const; +}; + +class ConstFileNodePtr { + public: + ConstFileNodePtr() = default; + ConstFileNodePtr(FileNodePtr file_node_ptr) : file_node_ptr_(file_node_ptr) { + } + + const FileNode *operator->() const { + return file_node_ptr_.operator->(); + } + const FileNode &operator*() const { + return file_node_ptr_.operator*(); + } + + explicit operator bool() const { + return bool(file_node_ptr_); + } + const FullRemoteFileLocation *get_remote() const { + return file_node_ptr_.get_remote(); + } + + private: + FileNodePtr file_node_ptr_; +}; + +class FileView { + public: + FileView() = default; + explicit FileView(ConstFileNodePtr node); + + bool empty() const; + + bool has_local_location() const; + const FullLocalFileLocation &local_location() const; + bool has_remote_location() const; + const FullRemoteFileLocation &remote_location() const; + bool has_generate_location() const; + const FullGenerateFileLocation &generate_location() const; + + bool has_url() const; + const string &url() const; + + const string &remote_name() const; + + string suggested_name() const; + + DialogId owner_dialog_id() const; + + bool get_by_hash() const; + + FileId file_id() const { + return node_->main_file_id_; + } + + int64 size() const; + int64 expected_size() const; + bool is_downloading() const; + int64 local_size() const; + int64 local_total_size() const; + bool is_uploading() const; + int64 remote_size() const; + string path() const; + + bool can_download_from_server() const; + bool can_generate() const; + bool can_delete() const; + + FileType get_type() const { + if (has_local_location()) { + return local_location().file_type_; + } + if (has_remote_location()) { + return remote_location().file_type_; + } + if (has_generate_location()) { + return generate_location().file_type_; + } + return FileType::Temp; + } + bool is_encrypted() const { + return get_type() == FileType::Encrypted; + } + const FileEncryptionKey &encryption_key() const { + return node_->encryption_key_; + } + + private: + ConstFileNodePtr node_{}; +}; + +class Td; +class FileManager : public FileLoadManager::Callback { + public: + class DownloadCallback { + public: + DownloadCallback() = default; + DownloadCallback(const DownloadCallback &) = delete; + DownloadCallback &operator=(const DownloadCallback &) = delete; + virtual ~DownloadCallback() = default; + virtual void on_progress(FileId file_id) { + } + + virtual void on_download_ok(FileId file_id) = 0; + virtual void on_download_error(FileId file_id, Status error) = 0; + }; + + class UploadCallback { + public: + UploadCallback() = default; + UploadCallback(const UploadCallback &) = delete; + UploadCallback &operator=(const UploadCallback &) = delete; + virtual ~UploadCallback() = default; + + virtual void on_progress(FileId file_id) { + } + + // After on_upload_ok all uploads of this file will be paused till merge, delete_partial_remote_location or + // explicit upload request with the same file_id. + // Also upload may be resumed after some other merges. + virtual void on_upload_ok(FileId file_id, tl_object_ptr<telegram_api::InputFile> input_file) = 0; + virtual void on_upload_encrypted_ok(FileId file_id, tl_object_ptr<telegram_api::InputEncryptedFile> input_file) = 0; + virtual void on_upload_error(FileId file_id, Status error) = 0; + }; + + class Context { + public: + virtual void on_new_file(int64 size) = 0; + virtual void on_file_updated(FileId size) = 0; + virtual ActorShared<> create_reference() = 0; + Context() = default; + Context(const Context &) = delete; + Context &operator=(const Context &) = delete; + virtual ~Context() = default; + }; + + explicit FileManager(std::unique_ptr<Context> context); + FileManager(const FileManager &other) = delete; + FileManager &operator=(const FileManager &other) = delete; + FileManager(FileManager &&other) = delete; + FileManager &operator=(FileManager &&other) = delete; + ~FileManager() override; + + void init_actor(); + + FileId dup_file_id(FileId file_id); + + void on_file_unlink(const FullLocalFileLocation &location); + + FileId register_empty(FileType type); + Result<FileId> register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, + bool get_by_hash = false, bool force = false) TD_WARN_UNUSED_RESULT; + FileId register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source, + DialogId owner_dialog_id, int64 size, int64 expected_size, string name) TD_WARN_UNUSED_RESULT; + Result<FileId> register_generate(FileType file_type, FileLocationSource file_location_source, string original_path, + string conversion, DialogId owner_dialog_id, + int64 expected_size) TD_WARN_UNUSED_RESULT; + Result<FileId> register_file(FileData data, FileLocationSource file_location_source, const char *source, bool force); + + Result<FileId> merge(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT; + + bool set_encryption_key(FileId file_id, FileEncryptionKey key); + bool set_content(FileId file_id, BufferSlice bytes); + + void download(FileId file_id, std::shared_ptr<DownloadCallback> callback, int32 new_priority); + void upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority, uint64 upload_order); + void resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback, + int32 new_priority, uint64 upload_order); + bool delete_partial_remote_location(FileId file_id); + void get_content(FileId file_id, Promise<BufferSlice> promise); + + void delete_file(FileId file_id, Promise<Unit> promise, const char *source); + + void external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, Promise<> promise); + void external_file_generate_finish(int64 id, Status status, Promise<> promise); + + static constexpr char PERSISTENT_ID_VERSION = 2; + Result<string> to_persistent_id(FileId file_id) TD_WARN_UNUSED_RESULT; + Result<FileId> from_persistent_id(CSlice persistent_id, FileType file_type) TD_WARN_UNUSED_RESULT; + FileView get_file_view(FileId file_id) const; + FileView get_sync_file_view(FileId file_id); + tl_object_ptr<td_api::file> get_file_object(FileId file_id, bool with_main_file_id = true); + + Result<FileId> get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> &thumb_input_file, + DialogId owner_dialog_id, bool is_encrypted) TD_WARN_UNUSED_RESULT; + Result<FileId> get_input_file_id(FileType type, const tl_object_ptr<td_api::InputFile> &file, + DialogId owner_dialog_id, bool allow_zero, bool is_encrypted, + bool get_by_hash = false) TD_WARN_UNUSED_RESULT; + + vector<tl_object_ptr<telegram_api::InputDocument>> get_input_documents(const vector<FileId> &file_ids); + + template <class T> + void store_file(FileId file_id, T &storer, int32 ttl = 5) const; + + template <class T> + FileId parse_file(T &parser); + + private: + Result<FileId> check_input_file_id(FileType type, Result<FileId> result, bool is_encrypted, + bool allow_zero) TD_WARN_UNUSED_RESULT; + + FileId register_url(string url, FileType file_type, FileLocationSource file_location_source, + DialogId owner_dialog_id); + + static constexpr int8 FROM_BYTES_PRIORITY = 10; + using FileNodeId = int32; + using QueryId = FileLoadManager::QueryId; + class Query { + public: + FileId file_id_; + enum Type { UploadByHash, Upload, Download, SetContent, Generate } type_; + }; + struct FileIdInfo { + FileNodeId node_id_{0}; + bool send_updates_flag_{false}; + bool pin_flag_{false}; + + int8 download_priority_{0}; + int8 upload_priority_{0}; + + uint64 upload_order_; + + std::shared_ptr<DownloadCallback> download_callback_; + std::shared_ptr<UploadCallback> upload_callback_; + }; + + ActorShared<> parent_; + std::unique_ptr<Context> context_; + std::shared_ptr<FileDbInterface> file_db_; + + FileIdInfo *get_file_id_info(FileId file_id); + + struct RemoteInfo { + // mutible is set to to enable changing access hash + mutable FullRemoteFileLocation remote_; + FileId file_id_; + bool operator==(const RemoteInfo &other) const { + return this->remote_ == other.remote_; + } + bool operator<(const RemoteInfo &other) const { + return this->remote_ < other.remote_; + } + }; + Enumerator<RemoteInfo> remote_location_info_; + + std::map<FullLocalFileLocation, FileId> local_location_to_file_id_; + std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_; + std::map<FileDbId, int32> pmc_id_to_file_node_id_; + + vector<FileIdInfo> file_id_info_; + vector<int32> empty_file_ids_; + vector<std::unique_ptr<FileNode>> file_nodes_; + ActorOwn<FileLoadManager> file_load_manager_; + ActorOwn<FileGenerateManager> file_generate_manager_; + + Container<Query> queries_container_; + + std::set<std::string> bad_paths_; + + FileId next_file_id(); + FileNodeId next_file_node_id(); + int32 next_pmc_file_id(); + FileId create_file_id(int32 file_node_id, FileNode *file_node); + void try_forget_file_id(FileId file_id); + + void load_from_pmc(FileId file_id, FullLocalFileLocation full_local); + void load_from_pmc(FileId file_id, const FullRemoteFileLocation &full_remote); + void load_from_pmc(FileId file_id, const FullGenerateFileLocation &full_generate); + template <class LocationT> + void load_from_pmc_impl(FileId file_id, const LocationT &location); + void load_from_pmc_result(FileId file_id, Result<FileData> &&result); + FileId register_pmc_file_data(FileData &&data); + + Status check_local_location(FileNodePtr node); + Status check_local_location(FullLocalFileLocation &location, int64 &size); + void try_flush_node(FileNodePtr node, bool new_remote = false, bool new_local = false, bool new_generate = false, + FileDbId other_pmc_id = Auto()); + void try_flush_node_info(FileNodePtr node); + void clear_from_pmc(FileNodePtr node); + void flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate); + void load_from_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate); + + string get_persistent_id(const FullRemoteFileLocation &location); + + string fix_file_extension(Slice file_name, Slice file_type, Slice file_extension); + string get_file_name(FileType file_type, Slice path); + + ConstFileNodePtr get_file_node(FileId file_id) const { + return ConstFileNodePtr{FileNodePtr{file_id, const_cast<FileManager *>(this)}}; + } + FileNodePtr get_file_node(FileId file_id) { + return FileNodePtr{file_id, this}; + } + FileNode *get_file_node_raw(FileId file_id, FileNodeId *file_node_id = nullptr); + + FileNodePtr get_sync_file_node(FileId file_id); + + // void release_file_node(FileNodeId id); + void cancel_download(FileNodePtr node); + void cancel_upload(FileNodePtr node); + void cancel_generate(FileNodePtr node); + void run_upload(FileNodePtr node, std::vector<int> bad_parts); + void run_download(FileNodePtr node); + void run_generate(FileNodePtr node); + + void on_start_download(QueryId query_id) override; + void on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size) override; + void on_partial_upload(QueryId query_id, const PartialRemoteFileLocation &partial_remote, int64 ready_size) override; + void on_download_ok(QueryId query_id, const FullLocalFileLocation &local, int64 size) override; + void on_upload_ok(QueryId query_id, FileType file_type, const PartialRemoteFileLocation &partial_remote, + int64 size) override; + void on_upload_full_ok(QueryId query_id, const FullRemoteFileLocation &remote) override; + void on_error(QueryId query_id, Status status) override; + + void on_error_impl(FileNodePtr node, Query::Type type, bool was_active, Status status); + + void on_partial_generate(QueryId, const PartialLocalFileLocation &partial_local, int32 expected_size); + void on_generate_ok(QueryId, const FullLocalFileLocation &local); + + std::pair<Query, bool> finish_query(QueryId query_id); + + FullRemoteFileLocation *get_remote(int32 key); + + void hangup() override; + void tear_down() override; + + friend class FileNodePtr; +}; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileManager.hpp b/libs/tdlib/td/td/telegram/files/FileManager.hpp new file mode 100644 index 0000000000..28f481308e --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileManager.hpp @@ -0,0 +1,222 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/DialogId.h" +#include "td/telegram/files/FileManager.h" +#include "td/telegram/Version.h" + +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/tl_helpers.h" + +namespace td { + +enum class FileStoreType : int32 { Empty, Url, Generate, Local, Remote }; + +template <class StorerT> +void FileManager::store_file(FileId file_id, StorerT &storer, int32 ttl) const { + auto file_store_type = FileStoreType::Empty; + auto file_view = get_file_view(file_id); + if (file_view.empty() || ttl <= 0) { + } else if (file_view.has_remote_location()) { + file_store_type = FileStoreType::Remote; + } else if (file_view.has_local_location()) { + file_store_type = FileStoreType::Local; + } else if (file_view.has_url()) { + file_store_type = FileStoreType::Url; + } else if (file_view.has_generate_location()) { + file_store_type = FileStoreType::Generate; + } + + store(file_store_type, storer); + + bool has_encryption_key = false; + bool has_expected_size = + file_store_type == FileStoreType::Remote && file_view.size() == 0 && file_view.expected_size() != 0; + if (file_store_type != FileStoreType::Empty) { + has_encryption_key = !file_view.empty() && file_view.is_encrypted(); + BEGIN_STORE_FLAGS(); + STORE_FLAG(has_encryption_key); + STORE_FLAG(has_expected_size); + END_STORE_FLAGS(); + } + + switch (file_store_type) { + case FileStoreType::Empty: + break; + case FileStoreType::Url: + store(file_view.get_type(), storer); + store(file_view.url(), storer); + store(file_view.owner_dialog_id(), storer); + break; + case FileStoreType::Remote: { + store(file_view.remote_location(), storer); + if (has_expected_size) { + store(narrow_cast<int32>(file_view.expected_size()), storer); + } else { + store(narrow_cast<int32>(file_view.size()), storer); + } + store(file_view.remote_name(), storer); + store(file_view.owner_dialog_id(), storer); + break; + } + case FileStoreType::Local: { + store(file_view.local_location(), storer); + store(narrow_cast<int32>(file_view.size()), storer); + store(static_cast<int32>(file_view.get_by_hash()), storer); + store(file_view.owner_dialog_id(), storer); + break; + } + case FileStoreType::Generate: { + auto generate_location = file_view.generate_location(); + FileId from_file_id; + bool have_file_id = false; + if (generate_location.conversion_ == "#_file_id#") { + break; + } else if (begins_with(generate_location.conversion_, "#file_id#")) { + // It is not the best possible way to serialize file_id + from_file_id = + FileId(to_integer<int32>(Slice(generate_location.conversion_).remove_prefix(Slice("#file_id#").size())), 0); + generate_location.conversion_ = "#_file_id#"; + have_file_id = true; + } + store(generate_location, storer); + store(static_cast<int32>(0), storer); // expected_size + store(static_cast<int32>(0), storer); + store(file_view.owner_dialog_id(), storer); + + if (have_file_id) { + store_file(from_file_id, storer, ttl - 1); + } + break; + } + } + if (has_encryption_key) { + store(file_view.encryption_key(), storer); + } +} + +template <class ParserT> +FileId FileManager::parse_file(ParserT &parser) { + if (parser.version() < static_cast<int32>(Version::StoreFileId)) { + return FileId(); + } + + FileStoreType file_store_type; + parse(file_store_type, parser); + + bool has_encryption_key = false; + bool has_expected_size = false; + if (file_store_type != FileStoreType::Empty) { + if (parser.version() >= static_cast<int32>(Version::StoreFileEncryptionKey)) { + BEGIN_PARSE_FLAGS(); + PARSE_FLAG(has_encryption_key); + PARSE_FLAG(has_expected_size); + END_PARSE_FLAGS(); + } + } + + auto file_id = [&] { + switch (file_store_type) { + case FileStoreType::Empty: + return FileId(); + case FileStoreType::Remote: { + FullRemoteFileLocation full_remote_location; + parse(full_remote_location, parser); + int32 size = 0; + int32 expected_size = 0; + if (has_expected_size) { + parse(expected_size, parser); + } else { + parse(size, parser); + } + string name; + parse(name, parser); + DialogId owner_dialog_id; + if (parser.version() >= static_cast<int32>(Version::StoreFileOwnerId)) { + parse(owner_dialog_id, parser); + } + return register_remote(full_remote_location, FileLocationSource::FromDb, owner_dialog_id, size, expected_size, + name); + } + case FileStoreType::Local: { + FullLocalFileLocation full_local_location; + parse(full_local_location, parser); + int32 size; + parse(size, parser); + int32 get_by_hash; + parse(get_by_hash, parser); + DialogId owner_dialog_id; + if (parser.version() >= static_cast<int32>(Version::StoreFileOwnerId)) { + parse(owner_dialog_id, parser); + } + auto r_file_id = register_local(full_local_location, owner_dialog_id, size, get_by_hash != 0); + if (r_file_id.is_ok()) { + return r_file_id.move_as_ok(); + } + LOG(ERROR) << "Can't resend local file " << full_local_location.path_; + return register_empty(full_local_location.file_type_); + } + case FileStoreType::Generate: { + FullGenerateFileLocation full_generated_location; + parse(full_generated_location, parser); + int32 expected_size; + parse(expected_size, parser); // expected_size + int32 zero; + parse(zero, parser); + DialogId owner_dialog_id; + if (parser.version() >= static_cast<int32>(Version::StoreFileOwnerId)) { + parse(owner_dialog_id, parser); + } + if (begins_with(full_generated_location.conversion_, "#file_id#")) { + LOG(ERROR) << "Can't resend message with '#file_id#...' location"; + return register_empty(full_generated_location.file_type_); + } + if (full_generated_location.conversion_ == "#_file_id#") { + auto file_id = parse_file(parser); + if (file_id.empty()) { + return register_empty(full_generated_location.file_type_); + } + auto download_file_id = dup_file_id(file_id); + full_generated_location.conversion_ = PSTRING() << "#file_id#" << download_file_id.get(); + } + + auto r_file_id = register_generate(full_generated_location.file_type_, FileLocationSource::FromDb, + full_generated_location.original_path_, full_generated_location.conversion_, + owner_dialog_id, expected_size); + if (r_file_id.is_ok()) { + return r_file_id.move_as_ok(); + } + return register_empty(full_generated_location.file_type_); + } + case FileStoreType::Url: { + FileType type; + string url; + parse(type, parser); + parse(url, parser); + DialogId owner_dialog_id; + if (parser.version() >= static_cast<int32>(Version::StoreFileOwnerId)) { + parse(owner_dialog_id, parser); + } + return register_url(url, type, FileLocationSource::FromDb, owner_dialog_id); + } + } + return FileId(); + }(); + + if (has_encryption_key) { + FileEncryptionKey encryption_key; + parse(encryption_key, parser); + set_encryption_key(file_id, std::move(encryption_key)); + } + + return file_id; +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileStats.cpp b/libs/tdlib/td/td/telegram/files/FileStats.cpp new file mode 100644 index 0000000000..8c93f01ce3 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileStats.cpp @@ -0,0 +1,221 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileStats.h" + +#include "td/telegram/td_api.h" + +#include "td/telegram/files/FileLoaderUtils.h" +#include "td/telegram/Global.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" + +#include <algorithm> +#include <unordered_set> +#include <utility> + +namespace td { + +tl_object_ptr<td_api::storageStatisticsFast> FileStatsFast::as_td_api() const { + return make_tl_object<td_api::storageStatisticsFast>(size, count, db_size); +} + +void FileStats::add(StatByType &by_type, FileType file_type, int64 size) { + auto pos = static_cast<size_t>(file_type); + CHECK(pos < stat_by_type.size()); + by_type[pos].size += size; + by_type[pos].cnt++; +} + +void FileStats::add(FullFileInfo &&info) { + if (split_by_owner_dialog_id) { + add(stat_by_owner_dialog_id[info.owner_dialog_id], info.file_type, info.size); + } else { + add(stat_by_type, info.file_type, info.size); + } + if (need_all_files) { + all_files.emplace_back(std::move(info)); + } +} + +FileTypeStat get_nontemp_stat(const FileStats::StatByType &by_type) { + FileTypeStat stat; + for (size_t i = 0; i < file_type_size; i++) { + if (FileType(i) != FileType::Temp) { + stat.size += by_type[i].size; + stat.cnt += by_type[i].cnt; + } + } + return stat; +} +FileTypeStat FileStats::get_total_nontemp_stat() const { + if (!split_by_owner_dialog_id) { + return get_nontemp_stat(stat_by_type); + } + FileTypeStat stat; + for (auto &dialog : stat_by_owner_dialog_id) { + auto tmp = get_nontemp_stat(dialog.second); + stat.size += tmp.size; + stat.cnt += tmp.cnt; + } + return stat; +} +void FileStats::apply_dialog_limit(int32 limit) { + if (limit == -1) { + return; + } + if (!split_by_owner_dialog_id) { + return; + } + + std::vector<std::pair<int64, DialogId>> dialogs; + for (auto &dialog : stat_by_owner_dialog_id) { + if (!dialog.first.is_valid()) { + continue; + } + int64 size = 0; + for (auto &it : dialog.second) { + size += it.size; + } + dialogs.push_back(std::make_pair(size, dialog.first)); + } + size_t prefix = dialogs.size(); + if (prefix > static_cast<size_t>(limit)) { + prefix = static_cast<size_t>(limit); + } + std::partial_sort(dialogs.begin(), dialogs.begin() + prefix, dialogs.end(), + [](const auto &x, const auto &y) { return x.first > y.first; }); + dialogs.resize(prefix); + + std::unordered_set<DialogId, DialogIdHash> all_dialogs; + + for (auto &dialog : dialogs) { + all_dialogs.insert(dialog.second); + } + + StatByType other_stats; + bool other_flag = false; + for (auto it = stat_by_owner_dialog_id.begin(); it != stat_by_owner_dialog_id.end();) { + if (all_dialogs.count(it->first)) { + it++; + } else { + for (size_t i = 0; i < file_type_size; i++) { + other_stats[i].size += it->second[i].size; + other_stats[i].cnt += it->second[i].cnt; + } + other_flag = true; + it = stat_by_owner_dialog_id.erase(it); + } + } + + if (other_flag) { + DialogId other_dialog_id; + stat_by_owner_dialog_id[other_dialog_id] = other_stats; + } +} + +tl_object_ptr<td_api::storageStatisticsByChat> as_td_api(DialogId dialog_id, + const FileStats::StatByType &stat_by_type) { + auto stats = make_tl_object<td_api::storageStatisticsByChat>(dialog_id.get(), 0, 0, Auto()); + for (size_t i = 0; i < file_type_size; i++) { + if (stat_by_type[i].size == 0) { + continue; + } + stats->size_ += stat_by_type[i].size; + stats->count_ += stat_by_type[i].cnt; + stats->by_file_type_.push_back(make_tl_object<td_api::storageStatisticsByFileType>( + as_td_api(FileType(i)), stat_by_type[i].size, stat_by_type[i].cnt)); + } + return stats; +} + +tl_object_ptr<td_api::storageStatistics> FileStats::as_td_api() const { + auto stats = make_tl_object<td_api::storageStatistics>(0, 0, Auto()); + if (!split_by_owner_dialog_id) { + stats->by_chat_.reserve(1); + stats->by_chat_.push_back(::td::as_td_api(DialogId(), stat_by_type)); + } else { + stats->by_chat_.reserve(stat_by_owner_dialog_id.size()); + for (auto &by_dialog : stat_by_owner_dialog_id) { + stats->by_chat_.push_back(::td::as_td_api(by_dialog.first, by_dialog.second)); + } + std::sort(stats->by_chat_.begin(), stats->by_chat_.end(), [](const auto &x, const auto &y) { + if (x->chat_id_ == 0 || y->chat_id_ == 0) { + return (x->chat_id_ == 0) < (y->chat_id_ == 0); + } + return x->size_ > y->size_; + }); + } + for (const auto &by_dialog : stats->by_chat_) { + stats->size_ += by_dialog->size_; + stats->count_ += by_dialog->count_; + } + return stats; +} + +std::vector<DialogId> FileStats::get_dialog_ids() const { + std::vector<DialogId> res; + if (!split_by_owner_dialog_id) { + return res; + } + res.reserve(stat_by_owner_dialog_id.size()); + for (auto &by_dialog : stat_by_owner_dialog_id) { + if (by_dialog.first.is_valid()) { + res.push_back(by_dialog.first); + } + } + return res; +} + +StringBuilder &operator<<(StringBuilder &sb, const FileTypeStat &stat) { + return sb << tag("size", format::as_size(stat.size)) << tag("count", stat.cnt); +} + +StringBuilder &operator<<(StringBuilder &sb, const FileStats &file_stats) { + if (!file_stats.split_by_owner_dialog_id) { + FileTypeStat total_stat; + for (auto &type_stat : file_stats.stat_by_type) { + total_stat.size += type_stat.size; + total_stat.cnt += type_stat.cnt; + } + + sb << "[FileStat " << tag("total", total_stat); + for (int i = 0; i < file_type_size; i++) { + sb << tag(Slice(file_type_name[i]), file_stats.stat_by_type[i]); + } + sb << "]"; + } else { + { + FileTypeStat total_stat; + for (auto &by_type : file_stats.stat_by_owner_dialog_id) { + for (auto &type_stat : by_type.second) { + total_stat.size += type_stat.size; + total_stat.cnt += type_stat.cnt; + } + } + sb << "[FileStat " << tag("total", total_stat); + } + for (auto &by_type : file_stats.stat_by_owner_dialog_id) { + FileTypeStat dialog_stat; + for (auto &type_stat : by_type.second) { + dialog_stat.size += type_stat.size; + dialog_stat.cnt += type_stat.cnt; + } + + sb << "[FileStat " << tag("owner_dialog_id", by_type.first) << tag("total", dialog_stat); + for (int i = 0; i < file_type_size; i++) { + sb << tag(Slice(file_type_name[i]), by_type.second[i]); + } + sb << "]"; + } + sb << "]"; + } + + return sb; +} +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileStats.h b/libs/tdlib/td/td/telegram/files/FileStats.h new file mode 100644 index 0000000000..5e02fd249e --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileStats.h @@ -0,0 +1,88 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/DialogId.h" +#include "td/telegram/files/FileLocation.h" + +#include "td/utils/common.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/tl_helpers.h" + +#include <array> +#include <unordered_map> + +namespace td { +namespace td_api { +class storageStatistics; +class storageStatisticsFast; +} // namespace td_api +} // namespace td + +namespace td { + +struct FileTypeStat { + int64 size{0}; + int32 cnt{0}; +}; + +template <class T> +void store(const FileTypeStat &stat, T &storer) { + using ::td::store; + store(stat.size, storer); + store(stat.cnt, storer); +} +template <class T> +void parse(FileTypeStat &stat, T &parser) { + using ::td::parse; + parse(stat.size, parser); + parse(stat.cnt, parser); +} + +struct FullFileInfo { + FileType file_type; + string path; + DialogId owner_dialog_id; + int64 size; + uint64 atime_nsec; + uint64 mtime_nsec; +}; + +struct FileStatsFast { + int64 size{0}; + int32 count{0}; + int64 db_size{0}; + FileStatsFast(int64 size, int32 count, int64 db_size) : size(size), count(count), db_size(db_size) { + } + tl_object_ptr<td_api::storageStatisticsFast> as_td_api() const; +}; + +struct FileStats { + bool need_all_files{false}; + bool split_by_owner_dialog_id{false}; + + using StatByType = std::array<FileTypeStat, file_type_size>; + + StatByType stat_by_type; + std::unordered_map<DialogId, StatByType, DialogIdHash> stat_by_owner_dialog_id; + + std::vector<FullFileInfo> all_files; + + void add(FullFileInfo &&info); + void apply_dialog_limit(int32 limit); + + tl_object_ptr<td_api::storageStatistics> as_td_api() const; + std::vector<DialogId> get_dialog_ids() const; + FileTypeStat get_total_nontemp_stat() const; + + private: + void add(StatByType &by_type, FileType file_type, int64 size); +}; + +StringBuilder &operator<<(StringBuilder &sb, const FileStats &file_stats); + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileStatsWorker.cpp b/libs/tdlib/td/td/telegram/files/FileStatsWorker.cpp new file mode 100644 index 0000000000..999402ae2b --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileStatsWorker.cpp @@ -0,0 +1,184 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileStatsWorker.h" + +#include "td/telegram/DialogId.h" +#include "td/telegram/files/FileDb.h" +#include "td/telegram/files/FileLoaderUtils.h" +#include "td/telegram/Global.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/PathView.h" +#include "td/utils/port/path.h" +#include "td/utils/port/Stat.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/tl_helpers.h" + +#include <functional> +#include <unordered_map> + +namespace td { +namespace { +// Performance ideas: +// - use slice instead of string +// - use arena memory allocator +// - store FileType or dir, no both +// - store dir relative to G()->files_dir() + +struct DbFileInfo { + FileType file_type; + string path; + DialogId owner_dialog_id; + int64 size; +}; + +// long and blocking +template <class CallbackT> +Status scan_db(CallbackT &&callback) { + G()->td_db()->get_file_db_shared()->pmc().get_by_range("file0", "file:", [&](Slice key, Slice value) { + // skip reference to other data + if (value.substr(0, 2) == "@@") { + return; + } + FileData data; + auto status = unserialize(data, value); + if (status.is_error()) { + LOG(ERROR) << "Invalid FileData in the database " << tag("value", format::escaped(value)); + return; + } + DbFileInfo info; + if (data.local_.type() == LocalFileLocation::Type::Full) { + info.file_type = data.local_.full().file_type_; + info.path = data.local_.full().path_; + } else if (data.local_.type() == LocalFileLocation::Type::Partial) { + info.file_type = data.local_.partial().file_type_; + info.path = data.local_.partial().path_; + } else { + return; + } + PathView path_view(info.path); + if (path_view.is_relative()) { + info.path = get_files_base_dir(info.file_type) + info.path; + } + // LOG(INFO) << "Found file in the database: " << data << " " << info.path; + info.owner_dialog_id = data.owner_dialog_id_; + info.size = data.size_; + if (info.size == 0 && data.local_.type() == LocalFileLocation::Type::Full) { + LOG(ERROR) << "Unknown size in the database"; + return; + } + callback(info); + }); + return Status::OK(); +} + +struct FsFileInfo { + FileType file_type; + string path; + int64 size; + uint64 atime_nsec; + uint64 mtime_nsec; +}; + +// long and blocking +template <class CallbackT> +Status scan_fs(CallbackT &&callback) { + for (int i = 0; i < file_type_size; i++) { + auto file_type = static_cast<FileType>(i); + auto files_dir = get_files_dir(file_type); + td::walk_path(files_dir, [&](CSlice path, bool is_dir) { + if (is_dir) { + // TODO: skip subdirs + return; + } + auto r_stat = stat(path); + if (r_stat.is_error()) { + LOG(WARNING) << "Stat in files gc failed: " << r_stat.error(); + return; + } + auto stat = r_stat.move_as_ok(); + FsFileInfo info; + info.path = path.str(); + info.size = stat.size_; + info.file_type = file_type; + info.atime_nsec = stat.atime_nsec_; + info.mtime_nsec = stat.mtime_nsec_; + callback(info); + }); + } + return Status::OK(); +} +} // namespace + +void FileStatsWorker::get_stats(bool need_all_files, bool split_by_owner_dialog_id, Promise<FileStats> promise) { + if (!G()->parameters().use_chat_info_db) { + split_by_owner_dialog_id = false; + } + if (!split_by_owner_dialog_id) { + FileStats file_stats; + file_stats.need_all_files = need_all_files; + auto start = Time::now(); + scan_fs([&](FsFileInfo &fs_info) { + FullFileInfo info; + info.file_type = fs_info.file_type; + info.path = std::move(fs_info.path); + info.size = fs_info.size; + info.atime_nsec = fs_info.atime_nsec; + info.mtime_nsec = fs_info.mtime_nsec; + file_stats.add(std::move(info)); + }); + auto passed = Time::now() - start; + LOG_IF(INFO, passed > 0.5) << "Get file stats took: " << format::as_time(passed); + promise.set_value(std::move(file_stats)); + } else { + auto start = Time::now(); + + std::vector<FullFileInfo> full_infos; + scan_fs([&](FsFileInfo &fs_info) { + FullFileInfo info; + info.file_type = fs_info.file_type; + info.path = std::move(fs_info.path); + info.size = fs_info.size; + info.atime_nsec = fs_info.atime_nsec; + info.mtime_nsec = fs_info.mtime_nsec; + + // LOG(INFO) << "Found file of size " << info.size << " at " << info.path; + + full_infos.push_back(std::move(info)); + }); + + std::unordered_map<size_t, size_t> hash_to_pos; + size_t pos = 0; + for (auto &full_info : full_infos) { + hash_to_pos[std::hash<std::string>()(full_info.path)] = pos; + pos++; + } + scan_db([&](DbFileInfo &db_info) { + auto it = hash_to_pos.find(std::hash<std::string>()(db_info.path)); + if (it == hash_to_pos.end()) { + return; + } + // LOG(INFO) << "Match! " << db_info.path << " from " << db_info.owner_dialog_id; + full_infos[it->second].owner_dialog_id = db_info.owner_dialog_id; + }); + + FileStats file_stats; + file_stats.need_all_files = need_all_files; + file_stats.split_by_owner_dialog_id = split_by_owner_dialog_id; + for (auto &full_info : full_infos) { + file_stats.add(std::move(full_info)); + } + auto passed = Time::now() - start; + LOG_IF(INFO, passed > 0.5) << "Get file stats took: " << format::as_time(passed); + promise.set_value(std::move(file_stats)); + } +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileStatsWorker.h b/libs/tdlib/td/td/telegram/files/FileStatsWorker.h new file mode 100644 index 0000000000..f70fcac1e0 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileStatsWorker.h @@ -0,0 +1,26 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileStats.h" + +namespace td { + +class FileStatsWorker : public Actor { + public: + explicit FileStatsWorker(ActorShared<> parent) : parent_(std::move(parent)) { + } + void get_stats(bool need_all_files, bool split_by_owner_dialog_id, Promise<FileStats> promise); + + private: + ActorShared<> parent_; +}; + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileUploader.cpp b/libs/tdlib/td/td/telegram/files/FileUploader.cpp new file mode 100644 index 0000000000..635e5c8933 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileUploader.cpp @@ -0,0 +1,292 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/FileUploader.h" + +#include "td/telegram/telegram_api.h" + +#include "td/telegram/Global.h" +#include "td/telegram/net/NetQueryDispatcher.h" + +#include "td/utils/buffer.h" +#include "td/utils/crypto.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Random.h" +#include "td/utils/ScopeGuard.h" + +namespace td { +FileUploader::FileUploader(const LocalFileLocation &local, const RemoteFileLocation &remote, int64 expected_size, + const FileEncryptionKey &encryption_key, std::vector<int> bad_parts, + std::unique_ptr<Callback> callback) + : local_(local) + , remote_(remote) + , expected_size_(expected_size) + , encryption_key_(encryption_key) + , bad_parts_(std::move(bad_parts)) + , callback_(std::move(callback)) { + if (!encryption_key_.empty()) { + iv_ = encryption_key_.mutable_iv(); + generate_iv_ = encryption_key_.iv_slice().str(); + } +} + +Result<FileLoader::FileInfo> FileUploader::init() { + if (remote_.type() == RemoteFileLocation::Type::Full) { + return Status::Error("File is already uploaded"); + } + + TRY_RESULT(prefix_info, on_update_local_location(local_)); + (void)prefix_info; + + int offset = 0; + int part_size = 0; + if (remote_.type() == RemoteFileLocation::Type::Partial) { + const auto &partial = remote_.partial(); + file_id_ = partial.file_id_; + part_size = partial.part_size_; + big_flag_ = partial.is_big_ != 0; + offset = partial.ready_part_count_; + } else { + file_id_ = Random::secure_int64(); + big_flag_ = expected_size_ > 10 * (1 << 20); + } + + std::vector<bool> ok(offset, true); + for (auto bad_id : bad_parts_) { + if (bad_id >= 0 && bad_id < offset) { + ok[bad_id] = false; + } + } + std::vector<int> parts; + for (int i = 0; i < offset; i++) { + if (ok[i]) { + parts.push_back(i); + } + } + if (!ok.empty() && !ok[0]) { + parts.clear(); + } + FileInfo res; + res.size = local_size_; + res.is_size_final = local_is_ready_; + res.part_size = part_size; + res.ready_parts = std::move(parts); + return res; +} + +Result<FileLoader::PrefixInfo> FileUploader::on_update_local_location(const LocalFileLocation &location) { + SCOPE_EXIT { + try_release_fd(); + }; + string path; + int64 local_size = 0; + bool local_is_ready{false}; + FileType file_type{FileType::Temp}; + if (location.type() == LocalFileLocation::Type::Empty) { + path = ""; + local_size = 0; + local_is_ready = false; + file_type = FileType::Temp; + } else if (location.type() == LocalFileLocation::Type::Partial) { + path = location.partial().path_; + local_size = static_cast<int64>(location.partial().part_size_) * location.partial().ready_part_count_; + local_is_ready = false; + file_type = location.partial().file_type_; + } else { + path = location.full().path_; + local_is_ready = true; + file_type = location.full().file_type_; + } + + if (!path.empty() && path != fd_path_) { + auto res_fd = FileFd::open(path, FileFd::Read); + + // Race: partial location could be already deleted. Just ignore such locations + if (res_fd.is_error()) { + if (location.type() == LocalFileLocation::Type::Partial) { + PrefixInfo info; + info.size = local_size_; + info.is_ready = local_is_ready_; + return info; + } + return res_fd.move_as_error(); + } + + fd_.close(); + fd_ = res_fd.move_as_ok(); + fd_path_ = path; + } + + if (local_is_ready) { + CHECK(!fd_.empty()); + local_size = fd_.get_size(); + if (local_size == 0) { + return Status::Error("Can't upload empty file"); + } + } else if (!fd_.empty()) { + auto real_local_size = fd_.get_size(); + if (real_local_size < local_size) { + LOG(ERROR) << tag("real_local_size", real_local_size) << " < " << tag("local_size", local_size); + PrefixInfo info; + info.size = local_size_; + info.is_ready = local_is_ready_; + return info; + } + } + + local_size_ = local_size; + if (expected_size_ < local_size_) { + expected_size_ = local_size_; + } + local_is_ready_ = local_is_ready; + file_type_ = file_type; + + PrefixInfo info; + info.size = local_size_; + info.is_ready = local_is_ready_; + return info; +} + +Status FileUploader::on_ok(int64 size) { + fd_.close(); + return Status::OK(); +} +void FileUploader::on_error(Status status) { + fd_.close(); + callback_->on_error(std::move(status)); +} + +Status FileUploader::generate_iv_map() { + LOG(INFO) << "generate iv_map " << generate_offset_ << " " << local_size_; + auto part_size = get_part_size(); + auto encryption_key = FileEncryptionKey(encryption_key_.key_slice(), generate_iv_); + BufferSlice bytes(part_size); + if (iv_map_.empty()) { + iv_map_.push_back(encryption_key.mutable_iv()); + } + CHECK(!fd_.empty()); + for (; generate_offset_ + static_cast<int64>(part_size) < local_size_; + generate_offset_ += static_cast<int64>(part_size)) { + TRY_RESULT(read_size, fd_.pread(bytes.as_slice(), generate_offset_)); + if (read_size != part_size) { + return Status::Error("Failed to read file part (for iv_map)"); + } + aes_ige_encrypt(encryption_key.key(), &encryption_key.mutable_iv(), bytes.as_slice(), bytes.as_slice()); + iv_map_.push_back(encryption_key.mutable_iv()); + } + generate_iv_ = encryption_key.iv_slice().str(); + return Status::OK(); +} + +Status FileUploader::before_start_parts() { + auto status = acquire_fd(); + if (status.is_error() && !local_is_ready_) { + return Status::Error(1, "Can't open temporary file"); + } + return status; +} +void FileUploader::after_start_parts() { + try_release_fd(); +} + +Result<std::pair<NetQueryPtr, bool>> FileUploader::start_part(Part part, int32 part_count) { + auto padded_size = part.size; + if (!encryption_key_.empty()) { + padded_size = (padded_size + 15) & ~15; + } + BufferSlice bytes(padded_size); + TRY_RESULT(size, fd_.pread(bytes.as_slice().truncate(part.size), part.offset)); + if (!encryption_key_.empty()) { + Random::secure_bytes(bytes.as_slice().substr(part.size)); + if (next_offset_ == part.offset) { + aes_ige_encrypt(encryption_key_.key(), &iv_, bytes.as_slice(), bytes.as_slice()); + next_offset_ += static_cast<int64>(bytes.size()); + } else { + if (part.id >= static_cast<int32>(iv_map_.size())) { + TRY_STATUS(generate_iv_map()); + } + CHECK(part.id < static_cast<int32>(iv_map_.size()) && part.id >= 0); + auto iv = iv_map_[part.id]; + aes_ige_encrypt(encryption_key_.key(), &iv, bytes.as_slice(), bytes.as_slice()); + } + } + + if (size != part.size) { + LOG(ERROR) << "Need to read " << part.size << " bytes, but read " << size << " bytes instead"; + return Status::Error("Failed to read file part"); + } + + NetQueryPtr net_query; + if (big_flag_) { + auto query = + telegram_api::upload_saveBigFilePart(file_id_, part.id, local_is_ready_ ? part_count : -1, std::move(bytes)); + net_query = G()->net_query_creator().create(create_storer(query), DcId::main(), NetQuery::Type::Upload, + NetQuery::AuthFlag::On, NetQuery::GzipFlag::Off); + } else { + auto query = telegram_api::upload_saveFilePart(file_id_, part.id, std::move(bytes)); + net_query = G()->net_query_creator().create(create_storer(query), DcId::main(), NetQuery::Type::Upload, + NetQuery::AuthFlag::On, NetQuery::GzipFlag::Off); + } + net_query->file_type_ = narrow_cast<int32>(file_type_); + return std::make_pair(std::move(net_query), false); +} + +Result<size_t> FileUploader::process_part(Part part, NetQueryPtr net_query) { + if (net_query->is_error()) { + return std::move(net_query->error()); + } + Result<bool> result; + if (big_flag_) { + result = fetch_result<telegram_api::upload_saveBigFilePart>(net_query->ok()); + } else { + result = fetch_result<telegram_api::upload_saveFilePart>(net_query->ok()); + } + if (result.is_error()) { + return result.move_as_error(); + } + if (!result.ok()) { + // TODO: it is possible + return Status::Error(500, "Internal Server Error"); + } + return part.size; +} + +void FileUploader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, bool is_ready, + int64 ready_size) { + callback_->on_partial_upload(PartialRemoteFileLocation{file_id_, part_count, part_size, ready_part_count, big_flag_}, + ready_size); + if (is_ready) { + callback_->on_ok(file_type_, + PartialRemoteFileLocation{file_id_, part_count, part_size, ready_part_count, big_flag_}, + local_size_); + } +} +FileLoader::Callback *FileUploader::get_callback() { + return static_cast<FileLoader::Callback *>(callback_.get()); +} + +void FileUploader::keep_fd_flag(bool keep_fd) { + keep_fd_ = keep_fd; + try_release_fd(); +} + +void FileUploader::try_release_fd() { + if (!keep_fd_ && !fd_.empty()) { + fd_.close(); + } +} + +Status FileUploader::acquire_fd() { + if (fd_.empty()) { + TRY_RESULT(fd, FileFd::open(fd_path_, FileFd::Read)); + fd_ = std::move(fd); + } + return Status::OK(); +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/FileUploader.h b/libs/tdlib/td/td/telegram/files/FileUploader.h new file mode 100644 index 0000000000..7f2ccfe8e8 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/FileUploader.h @@ -0,0 +1,76 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileLoader.h" +#include "td/telegram/files/FileLocation.h" + +#include "td/utils/port/FileFd.h" +#include "td/utils/Status.h" + +#include <utility> + +namespace td { +class FileUploader : public FileLoader { + public: + class Callback : public FileLoader::Callback { + public: + virtual void on_partial_upload(const PartialRemoteFileLocation &partial_remote, int64 ready_size) = 0; + virtual void on_ok(FileType file_type, const PartialRemoteFileLocation &partial_remote, int64 size) = 0; + virtual void on_error(Status status) = 0; + }; + + FileUploader(const LocalFileLocation &local, const RemoteFileLocation &remote, int64 expected_size, + const FileEncryptionKey &encryption_key, std::vector<int> bad_parts, std::unique_ptr<Callback> callback); + + // Should just implement all parent pure virtual methods. + // Must not call any of them... + private: + ResourceState resource_state_; + LocalFileLocation local_; + RemoteFileLocation remote_; + int64 expected_size_; + FileEncryptionKey encryption_key_; + std::vector<int> bad_parts_; + std::unique_ptr<Callback> callback_; + int64 local_size_ = 0; + bool local_is_ready_ = false; + FileType file_type_ = FileType::Temp; + + std::vector<UInt256> iv_map_; + UInt256 iv_; + string generate_iv_; + int64 generate_offset_ = 0; + int64 next_offset_ = 0; + + FileFd fd_; + string fd_path_; + int64 file_id_; + bool big_flag_; + + Result<FileInfo> init() override TD_WARN_UNUSED_RESULT; + Status on_ok(int64 size) override TD_WARN_UNUSED_RESULT; + void on_error(Status status) override; + Status before_start_parts() override; + void after_start_parts() override; + Result<std::pair<NetQueryPtr, bool>> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT; + Result<size_t> process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT; + void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, bool is_ready, int64 ready_size) override; + FileLoader::Callback *get_callback() override; + Result<PrefixInfo> on_update_local_location(const LocalFileLocation &location) override TD_WARN_UNUSED_RESULT; + + Status generate_iv_map(); + + bool keep_fd_ = false; + void keep_fd_flag(bool keep_fd) override; + void try_release_fd(); + Status acquire_fd() TD_WARN_UNUSED_RESULT; +}; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/PartsManager.cpp b/libs/tdlib/td/td/telegram/files/PartsManager.cpp new file mode 100644 index 0000000000..75debb7af1 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/PartsManager.cpp @@ -0,0 +1,333 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/PartsManager.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" + +#include <limits> +#include <numeric> + +namespace td { +/*** PartsManager ***/ + +namespace { +int64 calc_parts_count(int64 size, int64 part_size) { + CHECK(part_size != 0); + return (size + part_size - 1) / part_size; +} +} // namespace + +Status PartsManager::init_known_prefix(int64 known_prefix, size_t part_size, const std::vector<int> &ready_parts) { + known_prefix_flag_ = true; + known_prefix_size_ = known_prefix; + return init_no_size(part_size, ready_parts); +} + +Status PartsManager::init_no_size(size_t part_size, const std::vector<int> &ready_parts) { + unknown_size_flag_ = true; + size_ = 0; + min_size_ = 0; + max_size_ = std::numeric_limits<int64>::max(); + + if (part_size != 0) { + part_size_ = part_size; + } else { + part_size_ = 32 * (1 << 10); + while (use_part_count_limit_ && calc_parts_count(expected_size_, part_size_) > MAX_PART_COUNT) { + part_size_ *= 2; + CHECK(part_size_ <= MAX_PART_SIZE); + } + // just in case if expected_size_ is wrong + if (part_size_ < MAX_PART_SIZE) { + part_size_ *= 2; + } + } + part_count_ = 0; + if (known_prefix_flag_) { + part_count_ = static_cast<int>(known_prefix_size_ / part_size_); + } + part_count_ = max(part_count_, std::accumulate(ready_parts.begin(), ready_parts.end(), 0, + [](auto a, auto b) { return max(a, b + 1); })); + + init_common(ready_parts); + return Status::OK(); +} + +Status PartsManager::init(int64 size, int64 expected_size, bool is_size_final, size_t part_size, + const std::vector<int> &ready_parts, bool use_part_count_limit) { + CHECK(expected_size >= size); + use_part_count_limit_ = use_part_count_limit; + expected_size_ = expected_size; + if (expected_size_ > MAX_FILE_SIZE) { + return Status::Error("Too big file"); + } + if (!is_size_final) { + return init_known_prefix(size, part_size, ready_parts); + } + if (size == 0) { + return init_no_size(part_size, ready_parts); + } + CHECK(size > 0) << tag("size", size); + unknown_size_flag_ = false; + size_ = size; + + if (part_size != 0) { + part_size_ = part_size; + if (use_part_count_limit_ && calc_parts_count(expected_size_, part_size_) > MAX_PART_COUNT) { + return Status::Error("FILE_UPLOAD_RESTART"); + } + } else { + // TODO choose part_size_ depending on size + part_size_ = 64 * (1 << 10); + while (use_part_count_limit && calc_parts_count(expected_size_, part_size_) > MAX_PART_COUNT) { + part_size_ *= 2; + CHECK(part_size_ <= MAX_PART_SIZE); + } + } + CHECK(1 <= size_) << tag("size_", size_); + CHECK(!use_part_count_limit || calc_parts_count(expected_size_, part_size_) <= MAX_PART_COUNT) + << tag("size_", size_) << tag("expected_size", size_) << tag("is_size_final", is_size_final) + << tag("part_size_", part_size_) << tag("ready_parts", ready_parts.size()); + part_count_ = static_cast<int>(calc_parts_count(size_, part_size_)); + + init_common(ready_parts); + return Status::OK(); +} + +bool PartsManager::unchecked_ready() { + VLOG(files) << "Check readiness. Ready size is " << ready_size_ << ", total size is " << size_ + << ", unknown_size_flag = " << unknown_size_flag_ << ", need_check = " << need_check_ + << ", checked_prefix_size = " << checked_prefix_size_; + return !unknown_size_flag_ && ready_size_ == size_; +} +bool PartsManager::ready() { + return unchecked_ready() && (!need_check_ || checked_prefix_size_ == size_); +} + +Status PartsManager::finish() { + if (!ready()) { + return Status::Error("File transferring not finished"); + } + return Status::OK(); +} + +void PartsManager::update_first_empty_part() { + while (first_empty_part_ < part_count_ && part_status_[first_empty_part_] != PartStatus::Empty) { + first_empty_part_++; + } +} + +void PartsManager::update_first_not_ready_part() { + while (first_not_ready_part_ < part_count_ && part_status_[first_not_ready_part_] == PartStatus::Ready) { + first_not_ready_part_++; + } +} + +int32 PartsManager::get_ready_prefix_count() { + update_first_not_ready_part(); + auto res = first_not_ready_part_; + if (need_check_) { + auto checked_parts = narrow_cast<int32>(checked_prefix_size_ / part_size_); + if (checked_parts < res) { + res = checked_parts; + } + } + return res; +} + +Result<Part> PartsManager::start_part() { + update_first_empty_part(); + if (first_empty_part_ == part_count_) { + if (unknown_size_flag_) { + if (known_prefix_flag_ == false) { + part_count_++; + if (part_count_ > MAX_PART_COUNT) { + return Status::Error("Too big file with unknown size"); + } + part_status_.push_back(PartStatus::Empty); + } else { + return Status::Error(1, "Wait for prefix to be known"); + } + } else { + return get_empty_part(); + } + } + CHECK(part_status_[first_empty_part_] == PartStatus::Empty); + int id = first_empty_part_; + on_part_start(id); + return get_part(id); +} + +Status PartsManager::set_known_prefix(size_t size, bool is_ready) { + CHECK(known_prefix_flag_); + CHECK(size >= static_cast<size_t>(known_prefix_size_)); + known_prefix_size_ = narrow_cast<int64>(size); + expected_size_ = max(known_prefix_size_, expected_size_); + + CHECK(static_cast<size_t>(part_count_) == part_status_.size()); + if (is_ready) { + part_count_ = static_cast<int>(calc_parts_count(size, part_size_)); + + size_ = narrow_cast<int64>(size); + unknown_size_flag_ = false; + } else { + part_count_ = static_cast<int>(size / part_size_); + } + CHECK(static_cast<size_t>(part_count_) >= part_status_.size()) + << size << " " << is_ready << " " << part_count_ << " " << part_size_ << " " << part_status_.size(); + part_status_.resize(part_count_); + if (use_part_count_limit_ && calc_parts_count(expected_size_, part_size_) > MAX_PART_COUNT) { + return Status::Error("FILE_UPLOAD_RESTART"); + } + return Status::OK(); +} + +Status PartsManager::on_part_ok(int32 id, size_t part_size, size_t actual_size) { + CHECK(part_status_[id] == PartStatus::Pending); + pending_count_--; + + part_status_[id] = PartStatus::Ready; + ready_size_ += narrow_cast<int64>(actual_size); + + VLOG(files) << "Transferred part " << id << " of size " << part_size << ", total ready size = " << ready_size_; + + int64 offset = narrow_cast<int64>(part_size_) * id; + int64 end_offset = offset + narrow_cast<int64>(actual_size); + if (unknown_size_flag_) { + CHECK(part_size == part_size_); + if (actual_size < part_size_) { + max_size_ = min(max_size_, end_offset); + } + if (actual_size) { + min_size_ = max(min_size_, end_offset); + } + if (min_size_ > max_size_) { + auto status = Status::Error(PSLICE() << "Failed to transfer file: " << tag("min_size", min_size_) + << tag("max_size", max_size_)); + LOG(ERROR) << status; + return status; + } else if (min_size_ == max_size_) { + unknown_size_flag_ = false; + size_ = min_size_; + } + } else { + if ((actual_size < part_size && offset < size_) || (offset >= size_ && actual_size > 0)) { + auto status = Status::Error(PSLICE() << "Failed to transfer file: " << tag("size", size_) << tag("offset", offset) + << tag("transferred size", actual_size) << tag("part size", part_size)); + LOG(ERROR) << status; + return status; + } + } + return Status::OK(); +} + +void PartsManager::on_part_failed(int32 id) { + CHECK(part_status_[id] == PartStatus::Pending); + pending_count_--; + part_status_[id] = PartStatus::Empty; + if (id < first_empty_part_) { + first_empty_part_ = id; + } +} + +int64 PartsManager::get_size() const { + CHECK(!unknown_size_flag_); + return size_; +} +int64 PartsManager::get_size_or_zero() const { + return size_; +} + +int64 PartsManager::get_ready_size() const { + return ready_size_; +} + +int64 PartsManager::get_expected_size() const { + if (unknown_size_flag_) { + return max(static_cast<int64>(512 * (1 << 10)), get_ready_size() * 2); + } + return get_size(); +} + +size_t PartsManager::get_part_size() const { + return part_size_; +} + +int32 PartsManager::get_part_count() const { + return part_count_; +} + +void PartsManager::init_common(const std::vector<int> &ready_parts) { + ready_size_ = 0; + pending_count_ = 0; + first_empty_part_ = 0; + first_not_ready_part_ = 0; + part_status_ = vector<PartStatus>(part_count_); + + for (auto i : ready_parts) { + CHECK(0 <= i && i < part_count_) << tag("i", i) << tag("part_count", part_count_); + part_status_[i] = PartStatus::Ready; + auto part = get_part(i); + ready_size_ += narrow_cast<int64>(part.size); + } + + checked_prefix_size_ = get_ready_prefix_count() * narrow_cast<int64>(part_size_); +} + +void PartsManager::set_need_check() { + need_check_ = true; +} + +void PartsManager::set_checked_prefix_size(int64 size) { + checked_prefix_size_ = size; +} + +int64 PartsManager::get_checked_prefix_size() const { + return checked_prefix_size_; +} +int64 PartsManager::get_unchecked_ready_prefix_size() { + update_first_not_ready_part(); + auto count = first_not_ready_part_; + if (count == 0) { + return 0; + } + auto part = get_part(count - 1); + int64 res = part.offset; + if (!unknown_size_flag_) { + res += narrow_cast<int64>(part.size); + res = min(res, get_size()); + } + return res; +} + +Part PartsManager::get_part(int id) { + int64 offset = narrow_cast<int64>(part_size_) * id; + int64 size = narrow_cast<int64>(part_size_); + if (!unknown_size_flag_) { + auto total_size = get_size(); + if (total_size < offset) { + size = 0; + } else { + size = min(size, total_size - offset); + } + } + return Part{id, offset, static_cast<size_t>(size)}; +} + +Part PartsManager::get_empty_part() { + return Part{-1, 0, 0}; +} + +void PartsManager::on_part_start(int32 id) { + CHECK(part_status_[id] == PartStatus::Empty); + part_status_[id] = PartStatus::Pending; + pending_count_++; +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/PartsManager.h b/libs/tdlib/td/td/telegram/files/PartsManager.h new file mode 100644 index 0000000000..26c31d5174 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/PartsManager.h @@ -0,0 +1,87 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Status.h" + +namespace td { + +/*** PartsManager***/ +struct Part { + int id; + int64 offset; + size_t size; +}; + +class PartsManager { + public: + Status init(int64 size, int64 expected_size, bool is_size_final, size_t part_size, + const std::vector<int> &ready_parts, bool use_part_count_limit) TD_WARN_UNUSED_RESULT; + bool ready(); + bool unchecked_ready(); + Status finish() TD_WARN_UNUSED_RESULT; + + // returns empty part if nothing to return + Result<Part> start_part() TD_WARN_UNUSED_RESULT; + Status on_part_ok(int32 id, size_t part_size, size_t actual_size) TD_WARN_UNUSED_RESULT; + void on_part_failed(int32 id); + Status set_known_prefix(size_t size, bool is_ready); + void set_need_check(); + void set_checked_prefix_size(int64 size); + + int64 get_checked_prefix_size() const; + int64 get_unchecked_ready_prefix_size(); + int64 get_size() const; + int64 get_size_or_zero() const; + int64 get_expected_size() const; + int64 get_ready_size() const; + size_t get_part_size() const; + int32 get_part_count() const; + int32 get_ready_prefix_count(); + + private: + static constexpr int MAX_PART_COUNT = 3000; + static constexpr int MAX_PART_SIZE = 512 * (1 << 10); + static constexpr int64 MAX_FILE_SIZE = MAX_PART_SIZE * MAX_PART_COUNT; + + enum class PartStatus { Empty, Pending, Ready }; + + bool need_check_{false}; + int64 checked_prefix_size_{0}; + + bool known_prefix_flag_{false}; + int64 known_prefix_size_; + + int64 size_; + int64 expected_size_; + int64 min_size_; + int64 max_size_; + bool unknown_size_flag_; + int64 ready_size_; + + size_t part_size_; + int part_count_; + int pending_count_; + int first_empty_part_; + int first_not_ready_part_; + vector<PartStatus> part_status_; + bool use_part_count_limit_; + + void init_common(const vector<int> &ready_parts); + Status init_known_prefix(int64 known_prefix, size_t part_size, + const std::vector<int> &ready_parts) TD_WARN_UNUSED_RESULT; + Status init_no_size(size_t part_size, const std::vector<int> &ready_parts) TD_WARN_UNUSED_RESULT; + + Part get_part(int id); + Part get_empty_part(); + void on_part_start(int32 id); + void update_first_empty_part(); + void update_first_not_ready_part(); +}; + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/ResourceManager.cpp b/libs/tdlib/td/td/telegram/files/ResourceManager.cpp new file mode 100644 index 0000000000..00677ddd76 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/ResourceManager.cpp @@ -0,0 +1,179 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/files/ResourceManager.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/ScopeGuard.h" + +#include <algorithm> + +namespace td { + +void ResourceManager::register_worker(ActorShared<FileLoaderActor> callback, int8 priority) { + auto node_id = nodes_container_.create(); + auto *node_ptr = nodes_container_.get(node_id); + *node_ptr = std::make_unique<Node>(); + auto *node = (*node_ptr).get(); + CHECK(node); + node->node_id = node_id; + node->callback_ = std::move(callback); + + add_node(node_id, priority); + send_closure(node->callback_, &FileLoaderActor::set_resource_manager, actor_shared(this, node_id)); +} + +void ResourceManager::update_priority(int8 priority) { + if (stop_flag_) { + return; + } + auto node_id = get_link_token(); + if (!remove_node(node_id)) { + return; + } + add_node(node_id, priority); +} + +void ResourceManager::update_resources(const ResourceState &resource_state) { + if (stop_flag_) { + return; + } + auto node_id = get_link_token(); + auto node_ptr = nodes_container_.get(node_id); + if (node_ptr == nullptr) { + return; + } + auto node = (*node_ptr).get(); + CHECK(node); + VLOG(files) << "before total: " << resource_state_; + VLOG(files) << "before " << tag("node_id", node_id) << ": " << node->resource_state_; + resource_state_ -= node->resource_state_; + node->resource_state_.update_master(resource_state); + resource_state_ += node->resource_state_; + VLOG(files) << "after total: " << resource_state_; + VLOG(files) << "after " << tag("node_id", node_id) << ": " << node->resource_state_; + + if (mode_ == Mode::Greedy) { + add_to_heap(node); + } + loop(); +} + +void ResourceManager::hangup_shared() { + auto node_id = get_link_token(); + auto node_ptr = nodes_container_.get(node_id); + if (node_ptr == nullptr) { + return; + } + auto node = (*node_ptr).get(); + CHECK(node); + if (node->in_heap()) { + by_estimated_extra_.erase(node->as_heap_node()); + } + resource_state_ -= node->resource_state_; + remove_node(node_id); + nodes_container_.erase(node_id); + loop(); +} + +void ResourceManager::add_to_heap(Node *node) { + auto *heap_node = node->as_heap_node(); + auto key = node->resource_state_.estimated_extra(); + if (heap_node->in_heap()) { + if (key != 0) { + by_estimated_extra_.fix(key, heap_node); + } else { + by_estimated_extra_.erase(heap_node); + } + } else { + if (key != 0) { + by_estimated_extra_.insert(key, heap_node); + } + } +} + +bool ResourceManager::satisfy_node(NodeId file_node_id) { + auto file_node_ptr = nodes_container_.get(file_node_id); + CHECK(file_node_ptr); + auto file_node = (*file_node_ptr).get(); + CHECK(file_node); + auto part_size = narrow_cast<int64>(file_node->resource_state_.unit_size()); + auto need = file_node->resource_state_.estimated_extra(); + VLOG(files) << tag("need", need) << tag("part_size", part_size); + need = (need + part_size - 1) / part_size * part_size; + VLOG(files) << tag("need", need); + if (need == 0) { + return true; + } + auto give = resource_state_.unused(); + give = min(need, give); + give -= give % part_size; + VLOG(files) << tag("give", give); + if (give == 0) { + return false; + } + resource_state_.start_use(give); + file_node->resource_state_.update_limit(give); + send_closure(file_node->callback_, &FileLoaderActor::update_resources, file_node->resource_state_); + return true; +} + +void ResourceManager::loop() { + if (stop_flag_) { + if (nodes_container_.empty()) { + stop(); + } + return; + } + auto active_limit = resource_state_.active_limit(); + resource_state_.update_limit(2 * 1024 * (1 << 10) - active_limit); + LOG(INFO) << tag("unused", resource_state_.unused()); + + if (mode_ == Mode::Greedy) { + std::vector<Node *> to_add; + while (!by_estimated_extra_.empty()) { + auto *node = Node::from_heap_node(by_estimated_extra_.pop()); + SCOPE_EXIT { + to_add.push_back(node); + }; + if (!satisfy_node(node->node_id)) { + break; + } + } + for (auto *node : to_add) { + add_to_heap(node); + } + } else if (mode_ == Mode::Baseline) { + // plain + for (auto &it : to_xload_) { + auto file_node_id = it.second; + if (!satisfy_node(file_node_id)) { + break; + } + } + } +} +void ResourceManager::add_node(NodeId node_id, int8 priority) { + if (priority >= 0) { + auto it = std::find_if(to_xload_.begin(), to_xload_.end(), [&](auto &x) { return x.first <= priority; }); + to_xload_.insert(it, std::make_pair(priority, node_id)); + } else { + auto it = std::find_if(to_xload_.begin(), to_xload_.end(), [&](auto &x) { return x.first < -priority; }); + to_xload_.insert(it, std::make_pair(-priority, node_id)); + } +} +bool ResourceManager::remove_node(NodeId node_id) { + auto it = std::find_if(to_xload_.begin(), to_xload_.end(), [&](auto &x) { return x.second == node_id; }); + if (it != to_xload_.end()) { + to_xload_.erase(it); + return true; + } + return false; +} + +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/ResourceManager.h b/libs/tdlib/td/td/telegram/files/ResourceManager.h new file mode 100644 index 0000000000..48a32f18f6 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/ResourceManager.h @@ -0,0 +1,66 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileLoaderActor.h" +#include "td/telegram/files/ResourceState.h" + +#include "td/utils/Container.h" +#include "td/utils/Heap.h" + +#include <utility> + +namespace td { +class ResourceManager : public Actor { + public: + enum class Mode { Baseline, Greedy }; + explicit ResourceManager(Mode mode) : mode_(mode) { + } + // use through ActorShared + void update_priority(int8 priority); + void update_resources(const ResourceState &resource_state); + + void register_worker(ActorShared<FileLoaderActor> callback, int8 priority); + + private: + Mode mode_; + using NodeId = uint64; + struct Node : public HeapNode { + NodeId node_id; + + ResourceState resource_state_; + ActorShared<FileLoaderActor> callback_; + + HeapNode *as_heap_node() { + return static_cast<HeapNode *>(this); + } + static Node *from_heap_node(HeapNode *heap_node) { + return static_cast<Node *>(heap_node); + } + }; + + Container<std::unique_ptr<Node>> nodes_container_; + vector<std::pair<int8, NodeId>> to_xload_; + KHeap<int64> by_estimated_extra_; + ResourceState resource_state_; + + ActorShared<> parent_; + bool stop_flag_ = false; + + void hangup_shared() override; + + void loop() override; + + void add_to_heap(Node *node); + bool satisfy_node(NodeId file_node_id); + void add_node(NodeId node_id, int8 priority); + bool remove_node(NodeId node_id); +}; +} // namespace td diff --git a/libs/tdlib/td/td/telegram/files/ResourceState.h b/libs/tdlib/td/td/telegram/files/ResourceState.h new file mode 100644 index 0000000000..e6a251ce95 --- /dev/null +++ b/libs/tdlib/td/td/telegram/files/ResourceState.h @@ -0,0 +1,104 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/StringBuilder.h" + +namespace td { + +class ResourceState { + public: + void start_use(int64 x) { + using_ += x; + CHECK(used_ + using_ <= limit_); + } + + void stop_use(int64 x) { + CHECK(x <= using_); + using_ -= x; + used_ += x; + } + + void update_limit(int64 extra) { + limit_ += extra; + } + + bool update_estimated_limit(int64 extra) { + auto new_estimated_limit = used_ + extra; + if (new_estimated_limit == estimated_limit_) { + return false; + } + estimated_limit_ = new_estimated_limit; + return true; + } + + void set_unit_size(size_t new_unit_size) { + unit_size_ = new_unit_size; + } + + int64 active_limit() const { + return limit_ - used_; + } + + int64 get_using() const { + return using_; + } + + int64 unused() const { + return limit_ - using_ - used_; + } + + int64 estimated_extra() const { + auto new_unused = max(limit_, estimated_limit_) - using_ - used_; + new_unused = static_cast<int64>((new_unused + unit_size() - 1) / unit_size() * unit_size()); + return new_unused + using_ + used_ - limit_; + } + + size_t unit_size() const { + return unit_size_; + } + + ResourceState &operator+=(const ResourceState &other) { + using_ += other.active_limit(); + used_ += other.used_; + return *this; + } + + ResourceState &operator-=(const ResourceState &other) { + using_ -= other.active_limit(); + used_ -= other.used_; + return *this; + } + + void update_master(const ResourceState &other) { + estimated_limit_ = other.estimated_limit_; + used_ = other.used_; + using_ = other.using_; + unit_size_ = other.unit_size_; + } + + void update_slave(const ResourceState &other) { + limit_ = other.limit_; + } + + friend StringBuilder &operator<<(StringBuilder &sb, const ResourceState &state) { + return sb << tag("estimated_limit", state.estimated_limit_) << tag("used", state.used_) + << tag("using", state.using_) << tag("limit", state.limit_); + } + + private: + int64 estimated_limit_ = 0; // me + int64 limit_ = 0; // master + int64 used_ = 0; // me + int64 using_ = 0; // me + size_t unit_size_ = 1; // me +}; + +} // namespace td |