diff options
Diffstat (limited to 'protocols/Telegram/tdlib/td/td/telegram/files/FileManager.cpp')
| -rw-r--r-- | protocols/Telegram/tdlib/td/td/telegram/files/FileManager.cpp | 1927 |
1 files changed, 1386 insertions, 541 deletions
diff --git a/protocols/Telegram/tdlib/td/td/telegram/files/FileManager.cpp b/protocols/Telegram/tdlib/td/td/telegram/files/FileManager.cpp index 31220c6127..a53aacb8b2 100644 --- a/protocols/Telegram/tdlib/td/td/telegram/files/FileManager.cpp +++ b/protocols/Telegram/tdlib/td/td/telegram/files/FileManager.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2023 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -72,24 +72,42 @@ StringBuilder &operator<<(StringBuilder &string_builder, FileLocationSource sour } } -StringBuilder &operator<<(StringBuilder &string_builder, FileManager::Query::Type type) { +StringBuilder &operator<<(StringBuilder &string_builder, const NewRemoteFileLocation &location) { + if (location.is_full_alive) { + string_builder << "alive "; + } + if (location.full) { + string_builder << location.full.value(); + } else { + string_builder << "[no location]"; + } + return string_builder << " from " << location.full_source; +} + +StringBuilder &operator<<(StringBuilder &string_builder, FileManager::DownloadQuery::Type type) { switch (type) { - case FileManager::Query::Type::UploadByHash: - return string_builder << "UploadByHash"; - case FileManager::Query::Type::UploadWaitFileReference: - return string_builder << "UploadWaitFileReference"; - case FileManager::Query::Type::Upload: - return string_builder << "Upload"; - case FileManager::Query::Type::DownloadWaitFileReference: + case FileManager::DownloadQuery::Type::DownloadWaitFileReference: return string_builder << "DownloadWaitFileReference"; - case FileManager::Query::Type::DownloadReloadDialog: + case FileManager::DownloadQuery::Type::DownloadReloadDialog: return string_builder << "DownloadReloadDialog"; - case FileManager::Query::Type::Download: + case FileManager::DownloadQuery::Type::Download: return string_builder << "Download"; - case FileManager::Query::Type::SetContent: + case FileManager::DownloadQuery::Type::SetContent: return string_builder << "SetContent"; - case FileManager::Query::Type::Generate: - return string_builder << "Generate"; + default: + UNREACHABLE(); + return string_builder << "Unknown"; + } +} + +StringBuilder &operator<<(StringBuilder &string_builder, FileManager::UploadQuery::Type type) { + switch (type) { + case FileManager::UploadQuery::Type::UploadByHash: + return string_builder << "UploadByHash"; + case FileManager::UploadQuery::Type::UploadWaitFileReference: + return string_builder << "UploadWaitFileReference"; + case FileManager::UploadQuery::Type::Upload: + return string_builder << "Upload"; default: UNREACHABLE(); return string_builder << "Unknown"; @@ -120,6 +138,382 @@ RemoteFileLocation NewRemoteFileLocation::partial_or_empty() const { return {}; } +class FileManager::FileInfoLocal final : public FileManager::FileInfo { + FullLocalFileLocation location_; + int64 size_ = 0; + unique_ptr<PartialRemoteFileLocation> partial_remote_location_; + const FileIdInfo *remote_file_info_ = nullptr; + + public: + FileInfoLocal(FullLocalFileLocation location, int64 size) : location_(std::move(location)), size_(size) { + } + + FileInfoType get_file_info_type() const final { + return FileInfoType::Local; + } + + FileType get_file_type() const final { + return location_.file_type_; + } + + int64 get_local_size() const final { + return size_; + } + + int64 get_remote_size() const final { + if (remote_file_info_ != nullptr) { + if (remote_file_info_->file_info_ != nullptr) { + return remote_file_info_->file_info_->get_remote_size(); + } + return 0; + } + if (partial_remote_location_ != nullptr) { + return partial_remote_location_->ready_size_; + } + return 0; + } + + int64 get_size() const final { + return size_; + } + + int64 get_expected_size(bool) const final { + return size_; + } + + const FullLocalFileLocation *get_local_location() const final { + return &location_; + } + + const FullGenerateFileLocation *get_generate_location() const final { + return nullptr; + } + + const FullRemoteFileLocation *get_remote_location() const final { + if (remote_file_info_ != nullptr && remote_file_info_->file_info_ != nullptr) { + return remote_file_info_->file_info_->get_remote_location(); + } + return nullptr; + } + + const string *get_url() const final { + return nullptr; + } + + string get_path() const final { + return location_.path_; + } + + string get_suggested_path() const final { + return location_.path_; + } + + string get_remote_name() const final { + return string(); + } + + string get_persistent_file_id() const final { + return string(); + } + + string get_unique_file_id() const final { + return string(); + } + + bool can_be_deleted() const final { + return begins_with(location_.path_, get_files_dir(get_file_type())); + } + + unique_ptr<FileInfo> clone() const final { + auto result = make_unique<FileInfoLocal>(location_, size_); + result->remote_file_info_ = remote_file_info_; + return result; + } +}; + +class FileManager::FileInfoGenerate final : public FileManager::FileInfo { + FullGenerateFileLocation location_; + int64 expected_size_ = 0; + string url_; + unique_ptr<PartialLocalFileLocation> partial_local_location_; + unique_ptr<PartialRemoteFileLocation> partial_remote_location_; + const FileIdInfo *local_file_info_ = nullptr; + + public: + FileInfoGenerate(FullGenerateFileLocation location, int64 expected_size, string url) + : location_(std::move(location)), expected_size_(expected_size), url_(std::move(url)) { + } + + FileInfoType get_file_info_type() const final { + return FileInfoType::Generate; + } + + FileType get_file_type() const final { + return location_.file_type_; + } + + int64 get_local_size() const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_local_size(); + } + return 0; + } + if (partial_local_location_ != nullptr) { + return partial_local_location_->ready_size_; + } + return 0; + } + + int64 get_remote_size() const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_remote_size(); + } + return 0; + } + if (partial_remote_location_ != nullptr) { + return partial_remote_location_->ready_size_; + } + return 0; + } + + int64 get_size() const final { + if (local_file_info_ == nullptr) { + return 0; + } + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_size(); + } + return 0; + } + + int64 get_expected_size(bool may_guess) const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_size(); + } + return 0; + } + int64 current_size = 0; + if (partial_local_location_ != nullptr) { + current_size = partial_local_location_->ready_size_; + } + if (expected_size_ != 0) { + return max(current_size, expected_size_); + } + return may_guess ? current_size * 3 : current_size; + } + + const FullLocalFileLocation *get_local_location() const final { + if (local_file_info_ != nullptr && local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_local_location(); + } + return nullptr; + } + + const FullGenerateFileLocation *get_generate_location() const final { + return &location_; + } + + const FullRemoteFileLocation *get_remote_location() const final { + if (local_file_info_ != nullptr && local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_remote_location(); + } + return nullptr; + } + + const string *get_url() const final { + return url_.empty() ? nullptr : &url_; + } + + string get_path() const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_path(); + } + return string(); + } + if (partial_local_location_ != nullptr) { + return partial_local_location_->path_; + } + return string(); + } + + string get_suggested_path() const final { + if (!url_.empty()) { + return get_url_file_name(url_); + } + return location_.original_path_; + } + + string get_remote_name() const final { + return string(); + } + + string get_persistent_file_id() const final { + if (!url_.empty()) { + return url_; + } + if (FileManager::is_remotely_generated_file(location_.conversion_)) { + return FileNode::get_persistent_id(location_); + } + return string(); + } + + string get_unique_file_id() const final { + if (FileManager::is_remotely_generated_file(location_.conversion_)) { + return FileNode::get_unique_id(location_); + } + return string(); + } + + bool can_be_deleted() const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->can_be_deleted(); + } + return false; + } + return partial_local_location_ != nullptr; + } + + unique_ptr<FileInfo> clone() const final { + return td::make_unique<FileInfoGenerate>(location_, expected_size_, url_); + } +}; + +class FileManager::FileInfoRemote final : public FileManager::FileInfo { + FullRemoteFileLocation location_; + int64 size_ = 0; + int64 expected_size_ = 0; + string remote_name_; + string url_; + unique_ptr<PartialLocalFileLocation> partial_local_location_; + const FileIdInfo *local_file_info_ = nullptr; + + public: + FileInfoRemote(FullRemoteFileLocation location, int64 size, int64 expected_size, string remote_name, string url) + : location_(std::move(location)) + , size_(size) + , expected_size_(expected_size) + , remote_name_(std::move(remote_name)) + , url_(std::move(url)) { + } + + FileInfoType get_file_info_type() const final { + return FileInfoType::Remote; + } + + FileType get_file_type() const final { + return location_.file_type_; + } + + int64 get_local_size() const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_local_size(); + } + return 0; + } + if (partial_local_location_ != nullptr) { + return partial_local_location_->ready_size_; + } + return 0; + } + + int64 get_remote_size() const final { + return size_; + } + + int64 get_size() const final { + return size_; + } + + int64 get_expected_size(bool) const final { + if (size_ != 0) { + return size_; + } + if (partial_local_location_ != nullptr) { + return max(partial_local_location_->ready_size_, expected_size_); + } + return expected_size_; + } + + const FullLocalFileLocation *get_local_location() const final { + if (local_file_info_ != nullptr && local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_local_location(); + } + return nullptr; + } + + const FullGenerateFileLocation *get_generate_location() const final { + return nullptr; + } + + const FullRemoteFileLocation *get_remote_location() const final { + return &location_; + } + + const string *get_url() const final { + return url_.empty() ? nullptr : &url_; + } + + string get_path() const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->get_path(); + } + return string(); + } + if (partial_local_location_ != nullptr) { + return partial_local_location_->path_; + } + return string(); + } + + string get_suggested_path() const final { + if (!remote_name_.empty()) { + return remote_name_; + } + if (!url_.empty()) { + return get_url_file_name(url_); + } + return string(); + } + + string get_remote_name() const final { + return remote_name_; + } + + string get_persistent_file_id() const final { + return FileNode::get_persistent_id(location_); + } + + string get_unique_file_id() const final { + if (location_.is_web()) { + return string(); + } + return FileNode::get_unique_id(location_); + } + + bool can_be_deleted() const final { + if (local_file_info_ != nullptr) { + if (local_file_info_->file_info_ != nullptr) { + return local_file_info_->file_info_->can_be_deleted(); + } + return false; + } + return partial_local_location_ != nullptr; + } + + unique_ptr<FileInfo> clone() const final { + auto result = td::make_unique<FileInfoRemote>(location_, size_, expected_size_, remote_name_, url_); + result->local_file_info_ = local_file_info_; + return result; + } +}; + FileNode *FileNodePtr::operator->() const { return get(); } @@ -170,9 +564,10 @@ void FileNode::init_ready_size() { if (local_.type() != LocalFileLocation::Type::Partial) { return; } - auto bitmask = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_); - local_ready_prefix_size_ = bitmask.get_ready_prefix_size(0, local_.partial().part_size_, size_); - local_ready_size_ = bitmask.get_total_size(local_.partial().part_size_, size_); + auto &partial = local_.partial(); + auto bitmask = Bitmask(Bitmask::Decode{}, partial.ready_bitmask_); + local_ready_prefix_size_ = bitmask.get_ready_prefix_size(0, partial.part_size_, size_); + partial.ready_size_ = bitmask.get_total_size(partial.part_size_, size_); } void FileNode::set_download_offset(int64 download_offset) { @@ -233,17 +628,10 @@ void FileNode::set_ignore_download_limit(bool ignore_download_limit) { } void FileNode::drop_local_location() { - set_local_location(LocalFileLocation(), 0, -1, -1); + set_local_location(LocalFileLocation(), -1, -1); } -void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset, - int64 ready_prefix_size) { - if (local_ready_size_ != ready_size) { - VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size from " << local_ready_size_ - << " to " << ready_size; - local_ready_size_ = ready_size; - on_info_changed(); - } +void FileNode::set_local_location(const LocalFileLocation &local, int64 prefix_offset, int64 ready_prefix_size) { if (local_ != local) { VLOG(update_file) << "File " << main_file_id_ << " has changed local location"; local_ = local; @@ -280,7 +668,7 @@ void FileNode::set_new_remote_location(NewRemoteFileLocation new_remote) { } if (new_remote.partial) { - set_partial_remote_location(*new_remote.partial, new_remote.ready_size); + set_partial_remote_location(*new_remote.partial); } else { delete_partial_remote_location(); } @@ -291,25 +679,13 @@ void FileNode::delete_partial_remote_location() { remote_.partial.reset(); on_changed(); } - if (remote_.ready_size != 0) { - VLOG(update_file) << "File " << main_file_id_ << " has changed remote ready size from " << remote_.ready_size - << " to " << 0; - remote_.ready_size = 0; - on_info_changed(); - } } -void FileNode::set_partial_remote_location(PartialRemoteFileLocation remote, int64 ready_size) { +void FileNode::set_partial_remote_location(PartialRemoteFileLocation remote) { if (remote_.is_full_alive) { VLOG(update_file) << "File " << main_file_id_ << " remote is still alive, so there is NO reason to update partial"; return; } - if (remote_.ready_size != ready_size) { - VLOG(update_file) << "File " << main_file_id_ << " has changed remote ready size from " << remote_.ready_size - << " to " << ready_size; - remote_.ready_size = ready_size; - on_info_changed(); - } if (remote_.partial && *remote_.partial == remote) { VLOG(update_file) << "Partial location of " << main_file_id_ << " is NOT changed"; return; @@ -515,16 +891,18 @@ string FileNode::suggested_path() const { } /*** FileView ***/ -bool FileView::has_local_location() const { +bool FileView::has_full_local_location() const { return node_->local_.type() == LocalFileLocation::Type::Full; } -const FullLocalFileLocation &FileView::local_location() const { - CHECK(has_local_location()); - return node_->local_.full(); +const FullLocalFileLocation *FileView::get_full_local_location() const { + if (!has_full_local_location()) { + return nullptr; + } + return &node_->local_.full(); } -bool FileView::has_remote_location() const { +bool FileView::has_full_remote_location() const { return static_cast<bool>(node_->remote_.full); } @@ -533,49 +911,54 @@ bool FileView::has_alive_remote_location() const { } bool FileView::has_active_upload_remote_location() const { - if (!has_remote_location()) { + const auto *main_remote_location = get_main_remote_location(); + if (main_remote_location == nullptr) { return false; } if (!has_alive_remote_location()) { return false; } - if (main_remote_location().is_encrypted_any()) { + if (main_remote_location->is_encrypted_any()) { return true; } - return main_remote_location().has_file_reference(); + return main_remote_location->has_file_reference(); } bool FileView::has_active_download_remote_location() const { - if (!has_remote_location()) { + const auto *full_remote_location = get_full_remote_location(); + if (full_remote_location == nullptr) { return false; } - if (remote_location().is_encrypted_any()) { + if (full_remote_location->is_encrypted_any()) { return true; } - return remote_location().has_file_reference(); + return full_remote_location->has_file_reference(); } -const FullRemoteFileLocation &FileView::remote_location() const { - CHECK(has_remote_location()); - auto *remote = node_.get_remote(); - if (remote) { - return *remote; +const FullRemoteFileLocation *FileView::get_full_remote_location() const { + const auto *remote = node_.get_remote(); + if (remote != nullptr) { + return remote; + } + if (!has_full_remote_location()) { + return nullptr; } - return node_->remote_.full.value(); + return &node_->remote_.full.value(); } -const FullRemoteFileLocation &FileView::main_remote_location() const { - CHECK(has_remote_location()); - return node_->remote_.full.value(); +const FullRemoteFileLocation *FileView::get_main_remote_location() const { + if (!has_full_remote_location()) { + return nullptr; + } + return &node_->remote_.full.value(); } bool FileView::has_generate_location() const { return node_->generate_ != nullptr; } -const FullGenerateFileLocation &FileView::generate_location() const { - CHECK(has_generate_location()); - return *node_->generate_; +const FullGenerateFileLocation *FileView::get_generate_location() const { + return node_->generate_.get(); } int64 FileView::size() const { @@ -594,124 +977,149 @@ int64 FileView::get_allocated_local_size() const { return r_stat.ok().real_size_; } -int64 FileView::expected_size(bool may_guess) const { - if (node_->size_ != 0) { - return node_->size_; +int64 FileNode::expected_size(bool may_guess) const { + if (size_ != 0) { + return size_; } int64 current_size = local_total_size(); // TODO: this is not the best approximation - if (node_->expected_size_ != 0) { - return max(current_size, node_->expected_size_); + if (expected_size_ != 0) { + return max(current_size, expected_size_); } - if (may_guess && node_->local_.type() == LocalFileLocation::Type::Partial) { + if (may_guess && local_.type() == LocalFileLocation::Type::Partial) { current_size *= 3; } return current_size; } +int64 FileView::expected_size(bool may_guess) const { + return node_->expected_size(may_guess); +} + +bool FileNode::is_downloading() const { + return download_priority_ != 0 || generate_download_priority_ != 0; +} + bool FileView::is_downloading() const { - return node_->download_priority_ != 0 || node_->generate_download_priority_ != 0; + return node_->is_downloading(); } int64 FileView::download_offset() const { return node_->download_offset_; } -int64 FileView::downloaded_prefix(int64 offset) const { - switch (node_->local_.type()) { +int64 FileNode::downloaded_prefix(int64 offset) const { + switch (local_.type()) { case LocalFileLocation::Type::Empty: return 0; case LocalFileLocation::Type::Full: - if (offset < node_->size_) { - return node_->size_ - offset; + if (offset < size_) { + return size_ - offset; } return 0; case LocalFileLocation::Type::Partial: - if (is_encrypted_secure()) { + if (get_type() == FileType::SecureEncrypted) { // File is not decrypted and verified yet return 0; } - return Bitmask(Bitmask::Decode{}, node_->local_.partial().ready_bitmask_) - .get_ready_prefix_size(offset, node_->local_.partial().part_size_, node_->size_); + return Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_) + .get_ready_prefix_size(offset, local_.partial().part_size_, size_); default: UNREACHABLE(); return 0; } } -int64 FileView::local_prefix_size() const { - switch (node_->local_.type()) { +int64 FileView::downloaded_prefix(int64 offset) const { + return node_->downloaded_prefix(offset); +} + +int64 FileNode::local_prefix_size() const { + switch (local_.type()) { case LocalFileLocation::Type::Full: - return node_->download_offset_ <= node_->size_ ? node_->size_ - node_->download_offset_ : 0; + return download_offset_ <= size_ ? size_ - download_offset_ : 0; case LocalFileLocation::Type::Partial: { - if (is_encrypted_secure()) { + if (get_type() == FileType::SecureEncrypted) { // File is not decrypted and verified yet return 0; } - return node_->local_ready_prefix_size_; + return local_ready_prefix_size_; } default: return 0; } } -int64 FileView::local_total_size() const { - switch (node_->local_.type()) { + +int64 FileView::local_prefix_size() const { + return node_->local_prefix_size(); +} + +int64 FileNode::local_total_size() const { + switch (local_.type()) { case LocalFileLocation::Type::Empty: return 0; case LocalFileLocation::Type::Full: - return node_->size_; + return size_; case LocalFileLocation::Type::Partial: - VLOG(update_file) << "Have local_ready_prefix_size = " << node_->local_ready_prefix_size_ - << " and local_ready_size = " << node_->local_ready_size_; - return max(node_->local_ready_prefix_size_, node_->local_ready_size_); + return local_.partial().ready_size_; default: UNREACHABLE(); return 0; } } +int64 FileView::local_total_size() const { + return node_->local_total_size(); +} + +bool FileNode::is_uploading() const { + return upload_priority_ != 0 || generate_upload_priority_ != 0 || upload_pause_.is_valid(); +} + bool FileView::is_uploading() const { - return node_->upload_priority_ != 0 || node_->generate_upload_priority_ != 0 || node_->upload_pause_.is_valid(); + return node_->is_uploading(); } -int64 FileView::remote_size() const { - if (node_->remote_.is_full_alive) { - return node_->size_; +int64 FileNode::remote_size() const { + if (remote_.is_full_alive) { + return size_; } - if (node_->remote_.partial) { - auto part_size = static_cast<int64>(node_->remote_.partial->part_size_); - auto ready_part_count = node_->remote_.partial->ready_part_count_; - auto remote_ready_size = node_->remote_.ready_size; - VLOG(update_file) << "Have part_size = " << part_size << ", remote_ready_part_count = " << ready_part_count - << ", remote_ready_size = " << remote_ready_size << ", size = " << size(); - auto res = max(part_size * ready_part_count, remote_ready_size); - if (size() != 0 && size() < res) { - res = size(); - } - return res; + if (remote_.partial) { + return remote_.partial->ready_size_; } - return node_->remote_.ready_size; //??? + return 0; } -string FileView::path() const { - switch (node_->local_.type()) { +int64 FileView::remote_size() const { + return node_->remote_size(); +} + +string FileNode::path() const { + switch (local_.type()) { case LocalFileLocation::Type::Full: - return node_->local_.full().path_; + return local_.full().path_; case LocalFileLocation::Type::Partial: - return node_->local_.partial().path_; + return local_.partial().path_; default: return string(); } } +string FileView::path() const { + return node_->path(); +} + bool FileView::has_url() const { - return !node_->url_.empty(); + return get_url() != nullptr; } -const string &FileView::url() const { - return node_->url_; +const string *FileView::get_url() const { + if (node_->url_.empty()) { + return nullptr; + } + return &node_->url_; } -const string &FileView::remote_name() const { +string FileView::remote_name() const { return node_->remote_name_; } @@ -735,19 +1143,20 @@ bool FileView::empty() const { } bool FileView::can_download_from_server() const { - if (!has_remote_location()) { + const auto *full_remote_location = get_full_remote_location(); + if (full_remote_location == nullptr) { return false; } - if (remote_location().file_type_ == FileType::Encrypted && encryption_key().empty()) { + if (full_remote_location->file_type_ == FileType::Encrypted && encryption_key().empty()) { return false; } - if (remote_location().is_web()) { + if (full_remote_location->is_web()) { return true; } - if (remote_location().get_dc_id().is_empty()) { + if (full_remote_location->get_dc_id().is_empty()) { return false; } - if (!remote_location().is_encrypted_any() && !remote_location().has_file_reference() && + if (!full_remote_location->is_encrypted_any() && !full_remote_location->has_file_reference() && ((node_->download_id_ == 0 && node_->download_was_update_file_reference_) || !node_->remote_.is_full_alive)) { return false; } @@ -758,60 +1167,60 @@ bool FileView::can_generate() const { return has_generate_location(); } -bool FileView::can_delete() const { - if (has_local_location()) { - return begins_with(local_location().path_, get_files_dir(get_type())); +bool FileNode::can_delete() const { + if (local_.type() == LocalFileLocation::Type::Full) { + return begins_with(local_.full().path_, get_files_dir(get_type())); } - return node_->local_.type() == LocalFileLocation::Type::Partial; + return local_.type() == LocalFileLocation::Type::Partial; } -string FileView::get_unique_id(const FullGenerateFileLocation &location) { +bool FileView::can_delete() const { + return node_->can_delete(); +} + +string FileNode::get_unique_id(const FullGenerateFileLocation &location) { return base64url_encode(zero_encode('\xff' + serialize(location))); } -string FileView::get_unique_id(const FullRemoteFileLocation &location) { +string FileNode::get_unique_id(const FullRemoteFileLocation &location) { return base64url_encode(zero_encode(serialize(location.as_unique()))); } -string FileView::get_persistent_id(const FullGenerateFileLocation &location) { +string FileNode::get_persistent_id(const FullGenerateFileLocation &location) { auto binary = serialize(location); binary = zero_encode(binary); - binary.push_back(FileNode::PERSISTENT_ID_VERSION_GENERATED); + binary.push_back(PERSISTENT_ID_VERSION_GENERATED); return base64url_encode(binary); } -string FileView::get_persistent_id(const FullRemoteFileLocation &location) { +string FileNode::get_persistent_id(const FullRemoteFileLocation &location) { auto binary = serialize(location); binary = zero_encode(binary); binary.push_back(static_cast<char>(narrow_cast<uint8>(Version::Next) - 1)); - binary.push_back(FileNode::PERSISTENT_ID_VERSION); + binary.push_back(PERSISTENT_ID_VERSION); return base64url_encode(binary); } -string FileView::get_persistent_file_id() const { - if (!empty()) { - if (has_alive_remote_location()) { - return get_persistent_id(remote_location()); - } else if (has_url()) { - return url(); - } else if (has_generate_location() && FileManager::is_remotely_generated_file(generate_location().conversion_)) { - return get_persistent_id(generate_location()); - } +string FileNode::get_persistent_file_id() const { + if (remote_.is_full_alive) { + return get_persistent_id(remote_.full.value()); + } else if (!url_.empty()) { + return url_; + } else if (generate_ != nullptr && FileManager::is_remotely_generated_file(generate_->conversion_)) { + return get_persistent_id(*generate_); } return string(); } -string FileView::get_unique_file_id() const { - if (!empty()) { - if (has_alive_remote_location()) { - if (!remote_location().is_web()) { - return get_unique_id(remote_location()); - } - } else if (has_generate_location() && FileManager::is_remotely_generated_file(generate_location().conversion_)) { - return get_unique_id(generate_location()); +string FileNode::get_unique_file_id() const { + if (remote_.is_full_alive) { + if (!remote_.full.value().is_web()) { + return get_unique_id(remote_.full.value()); } + } else if (generate_ != nullptr && FileManager::is_remotely_generated_file(generate_->conversion_)) { + return get_unique_id(*generate_); } return string(); } @@ -839,8 +1248,13 @@ FileManager::FileManager(unique_ptr<Context> context) : context_(std::move(conte } void FileManager::init_actor() { - file_load_manager_ = create_actor_on_scheduler<FileLoadManager>("FileLoadManager", G()->get_slow_net_scheduler_id(), - actor_shared(this), context_->create_reference()); + file_download_manager_ = create_actor_on_scheduler<FileDownloadManager>( + "FileDownloadManager", G()->get_slow_net_scheduler_id(), make_unique<FileDownloadManagerCallback>(actor_id(this)), + context_->create_reference()); + file_load_manager_ = create_actor_on_scheduler<FileLoadManager>("FileLoadManager", G()->get_slow_net_scheduler_id()); + file_upload_manager_ = create_actor_on_scheduler<FileUploadManager>( + "FileUploadManager", G()->get_slow_net_scheduler_id(), make_unique<FileUploadManagerCallback>(actor_id(this)), + context_->create_reference()); file_generate_manager_ = create_actor_on_scheduler<FileGenerateManager>( "FileGenerateManager", G()->get_slow_net_scheduler_id(), context_->create_reference()); } @@ -867,6 +1281,7 @@ string FileManager::get_file_name(FileType file_type, Slice path) { break; case FileType::ProfilePhoto: case FileType::Photo: + case FileType::PhotoStory: if (extension != "jpg" && extension != "jpeg" && extension != "gif" && extension != "png" && extension != "tif" && extension != "bmp") { return fix_file_extension(file_name, "photo", "jpg"); @@ -885,6 +1300,11 @@ string FileManager::get_file_name(FileType file_type, Slice path) { return fix_file_extension(file_name, "video", "mp4"); } break; + case FileType::VideoStory: + if (extension != "mp4") { + return fix_file_extension(file_name, "video", "mp4"); + } + break; case FileType::Audio: if (extension != "ogg" && extension != "oga" && extension != "mp3" && extension != "mpeg3" && extension != "m4a") { @@ -927,6 +1347,23 @@ bool FileManager::is_remotely_generated_file(Slice conversion) { return begins_with(conversion, "#map#") || begins_with(conversion, "#audio_t#"); } +vector<int> FileManager::get_missing_file_parts(const Status &error) { + vector<int> result; + auto error_message = error.message(); + if (begins_with(error_message, "FILE_PART_") && ends_with(error_message, "_MISSING")) { + auto r_file_part = to_integer_safe<int>(error_message.substr(10, error_message.size() - 18)); + if (r_file_part.is_error()) { + LOG(ERROR) << "Receive error " << error; + } else { + result.push_back(r_file_part.ok()); + } + } + if (error_message == "FILE_PART_INVALID") { + result.push_back(0); + } + return result; +} + void FileManager::check_local_location(FileId file_id, bool skip_file_size_checks) { auto node = get_sync_file_node(file_id); if (node) { @@ -1090,16 +1527,15 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) { partial.part_size_ = new_part_size; partial.ready_bitmask_ = new_mask.encode(); - - auto ready_size = new_mask.get_total_size(partial.part_size_, node->size_); - node->set_local_location(LocalFileLocation(std::move(partial)), ready_size, -1, -1); + partial.ready_size_ = new_mask.get_total_size(partial.part_size_, node->size_); + node->set_local_location(LocalFileLocation(std::move(partial)), -1, -1); LOG(INFO) << " ok: increase part_size " << old_part_size << "->" << new_part_size; return true; } FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) { CHECK(static_cast<size_t>(file_id.get()) < file_id_info_.size()); - return &file_id_info_[file_id.get()]; + return file_id_info_[file_id.get()].get(); } FileId FileManager::dup_file_id(FileId file_id, const char *source) { @@ -1108,7 +1544,15 @@ FileId FileManager::dup_file_id(FileId file_id, const char *source) { if (!file_node) { return FileId(); } - auto result_file_id = FileId(create_file_id(file_node_id, file_node).get(), file_id.get_remote()); + auto new_file_id = next_file_id(); + auto file_id_info = get_file_id_info(new_file_id); + file_id_info->node_id_ = file_node_id; + auto old_file_info = get_file_id_info(file_id)->file_info_.get(); + if (old_file_info != nullptr) { + file_id_info->file_info_ = old_file_info->clone(); + } + file_node->file_ids_.push_back(new_file_id); + auto result_file_id = FileId(new_file_id.get(), file_id.get_remote()); LOG(INFO) << "Dup file " << file_id << " to " << result_file_id << " from " << source; return result_file_id; } @@ -1117,32 +1561,24 @@ FileId FileManager::copy_file_id(FileId file_id, FileType file_type, DialogId ow auto file_view = get_file_view(file_id); auto download_file_id = dup_file_id(file_id, source); auto result_file_id = - register_generate(file_type, FileLocationSource::FromServer, file_view.suggested_path(), - PSTRING() << "#file_id#" << download_file_id.get(), owner_dialog_id, file_view.size()) - .ok(); + register_generate(file_type, file_view.suggested_path(), PSTRING() << "#file_id#" << download_file_id.get(), + owner_dialog_id, file_view.size()); LOG(INFO) << "Copy file " << file_id << " to " << result_file_id << " from " << source; return result_file_id; } -FileId FileManager::create_file_id(int32 file_node_id, FileNode *file_node) { - auto file_id = next_file_id(); - get_file_id_info(file_id)->node_id_ = file_node_id; - file_node->file_ids_.push_back(file_id); - return file_id; -} - -void FileManager::try_forget_file_id(FileId file_id) { +bool FileManager::try_forget_file_id(FileId file_id) { auto *info = get_file_id_info(file_id); if (info->send_updates_flag_ || info->pin_flag_ || info->sent_file_id_flag_) { LOG(DEBUG) << "Can't forget file " << file_id << ", because of" << (info->send_updates_flag_ ? " (sent updates)" : "") << (info->pin_flag_ ? " (pin)" : "") << (info->sent_file_id_flag_ ? " (sent file identifier)" : ""); - return; + return false; } auto file_node = get_file_node(file_id); if (file_node->main_file_id_ == file_id) { LOG(DEBUG) << "Can't forget main file " << file_id; - return; + return false; } LOG(DEBUG) << "Forget file " << file_id; @@ -1150,10 +1586,30 @@ void FileManager::try_forget_file_id(FileId file_id) { CHECK(is_removed); *info = FileIdInfo(); empty_file_ids_.push_back(file_id.get()); + return true; } FileId FileManager::register_empty(FileType type) { - return register_local(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok(); + auto location = FullLocalFileLocation(type, "", 0); + auto &file_id = local_location_to_file_id_[location]; + if (file_id.is_valid()) { + return file_id; + } + file_id = next_file_id(); + + LOG(INFO) << "Register empty file as " << file_id; + auto file_info = STORE_FILE_INFO ? td::make_unique<FileInfoLocal>(location, 0) : nullptr; + auto file_node_id = next_file_node_id(); + file_nodes_[file_node_id] = + td::make_unique<FileNode>(LocalFileLocation(std::move(location)), NewRemoteFileLocation(), nullptr, 0, 0, + string(), string(), DialogId(), FileEncryptionKey(), file_id, static_cast<int8>(0)); + + auto file_id_info = get_file_id_info(file_id); + file_id_info->node_id_ = file_node_id; + file_id_info->file_info_ = std::move(file_info); + file_id_info->pin_flag_ = true; + + return file_id; } void FileManager::on_file_unlink(const FullLocalFileLocation &location) { @@ -1171,48 +1627,142 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) { } Result<FileId> FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, - bool get_by_hash, bool force, bool skip_file_size_checks, - FileId merge_file_id) { - // TODO: use get_by_hash - FileData data; - data.local_ = LocalFileLocation(std::move(location)); - data.owner_dialog_id_ = owner_dialog_id; - data.size_ = size; - return register_file(std::move(data), FileLocationSource::None /*won't be used*/, merge_file_id, "register_local", - force, skip_file_size_checks); + bool get_by_hash, bool skip_file_size_checks, FileId merge_file_id) { + TRY_RESULT(info, check_full_local_location({std::move(location), size}, skip_file_size_checks)); + location = std::move(info.location_); + size = info.size_; + + if (bad_paths_.count(location.path_) != 0) { + return Status::Error(400, "Sending of internal database files is forbidden"); + } + + auto &file_id = local_location_to_file_id_[location]; + bool is_new = false; + if (!file_id.is_valid()) { + file_id = next_file_id(); + LOG(INFO) << "Register " << location << " as " << file_id; + + auto file_info = STORE_FILE_INFO ? td::make_unique<FileInfoLocal>(location, size) : nullptr; + auto file_node_id = next_file_node_id(); + auto &node = file_nodes_[file_node_id]; + node = td::make_unique<FileNode>(LocalFileLocation(std::move(location)), NewRemoteFileLocation(), nullptr, size, 0, + string(), string(), owner_dialog_id, FileEncryptionKey(), file_id, + static_cast<int8>(0)); + node->need_load_from_pmc_ = true; + auto file_id_info = get_file_id_info(file_id); + file_id_info->node_id_ = file_node_id; + file_id_info->file_info_ = std::move(file_info); + is_new = true; + } + + if (merge_file_id.is_valid()) { + auto status = merge(file_id, merge_file_id); + if (status.is_ok()) { + auto node = get_file_node(file_id); + auto main_file_id = node->main_file_id_; + if (main_file_id != file_id) { + if (is_new) { + bool is_removed = try_forget_file_id(file_id); + CHECK(is_removed); + node = get_file_node(main_file_id); + } + file_id = main_file_id; + } + try_flush_node(node, "register_local"); + } + if (is_new) { + get_file_id_info(file_id)->pin_flag_ = true; + } + if (status.is_error()) { + return std::move(status); + } + } else if (is_new) { + get_file_id_info(file_id)->pin_flag_ = true; + } + return file_id; } FileId FileManager::register_remote(FullRemoteFileLocation location, FileLocationSource file_location_source, DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) { - FileData data; + if (size < 0) { + LOG(ERROR) << "Receive file " << location << " of size " << size; + size = 0; + } + if (expected_size < 0) { + LOG(ERROR) << "Receive file " << location << " of expected size " << expected_size; + expected_size = 0; + } auto url = location.get_url(); - data.remote_ = RemoteFileLocation(std::move(location)); - data.owner_dialog_id_ = owner_dialog_id; - data.size_ = size; - data.expected_size_ = expected_size; - data.remote_name_ = std::move(remote_name); - - auto file_id = register_file(std::move(data), file_location_source, FileId(), "register_remote", false).move_as_ok(); - if (!url.empty()) { - auto file_node = get_file_node(file_id); - CHECK(file_node); - file_node->set_url(url); + + FileId file_id; + FileId merge_file_id; + int32 remote_key = 0; + if (context_->keep_exact_remote_location()) { + file_id = next_file_id(); + RemoteInfo info{location, file_location_source, file_id}; + remote_key = remote_location_info_.add(info); + auto &stored_info = remote_location_info_.get(remote_key); + if (stored_info.file_id_ != file_id) { + merge_file_id = stored_info.file_id_; + if (merge_choose_remote_location(location, file_location_source, stored_info.remote_, + stored_info.file_location_source_) == 0) { + stored_info.remote_ = location; + stored_info.file_location_source_ = file_location_source; + } + } + } else { + auto &other_id = remote_location_to_file_id_[location]; + if (other_id.is_valid() && get_file_node(other_id)->remote_.full_source == FileLocationSource::FromServer) { + // if the file has already been received from the server, then we don't need merge or create new file + // skip merging of dc_id, file_reference, and access_hash + return other_id; + } + + file_id = next_file_id(); + if (other_id.empty()) { + other_id = file_id; + } else { + merge_file_id = other_id; + } } - return file_id; + + LOG(INFO) << "Register " << location << " as " << file_id; + auto file_info = + STORE_FILE_INFO ? td::make_unique<FileInfoRemote>(location, size, expected_size, remote_name, url) : nullptr; + auto file_node_id = next_file_node_id(); + auto &node = file_nodes_[file_node_id]; + node = td::make_unique<FileNode>(LocalFileLocation(), + NewRemoteFileLocation(RemoteFileLocation(std::move(location)), file_location_source), + nullptr, size, expected_size, std::move(remote_name), std::move(url), + owner_dialog_id, FileEncryptionKey(), file_id, static_cast<int8>(1)); + auto file_id_info = get_file_id_info(file_id); + file_id_info->node_id_ = file_node_id; + file_id_info->file_info_ = std::move(file_info); + + auto main_file_id = file_id; + if (!merge_file_id.is_valid()) { + node->need_load_from_pmc_ = true; + get_file_id_info(main_file_id)->pin_flag_ = true; + } else { + // may invalidate node + merge(file_id, merge_file_id, true).ignore(); + try_flush_node(get_file_node(file_id), "register_remote"); + + main_file_id = get_file_node(file_id)->main_file_id_; + if (main_file_id != file_id) { + try_forget_file_id(file_id); + } + } + return FileId(main_file_id.get(), remote_key); } -FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source, - DialogId owner_dialog_id) { - auto file_id = register_generate(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok(); - auto file_node = get_file_node(file_id); - CHECK(file_node); - file_node->set_url(url); - return file_id; +FileId FileManager::register_url(string url, FileType file_type, DialogId owner_dialog_id) { + return do_register_generate(td::make_unique<FullGenerateFileLocation>(file_type, url, "#url#"), owner_dialog_id, 0, + url); } -Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSource file_location_source, - string original_path, string conversion, DialogId owner_dialog_id, - int64 expected_size) { +FileId FileManager::register_generate(FileType file_type, string original_path, string conversion, + DialogId owner_dialog_id, int64 expected_size) { // add #mtime# into conversion if (!original_path.empty() && conversion[0] != '#' && PathView(original_path).is_absolute()) { auto file_paths = log_interface->get_file_paths(); @@ -1222,21 +1772,39 @@ Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSo conversion = PSTRING() << "#mtime#" << lpad0(to_string(mtime), 20) << '#' << conversion; } } + return do_register_generate( + td::make_unique<FullGenerateFileLocation>(file_type, std::move(original_path), std::move(conversion)), + owner_dialog_id, max(expected_size, static_cast<int64>(0)), string()); +} - FileData data; - data.generate_ = - td::make_unique<FullGenerateFileLocation>(file_type, std::move(original_path), std::move(conversion)); - data.owner_dialog_id_ = owner_dialog_id; - data.expected_size_ = expected_size; - return register_file(std::move(data), file_location_source, FileId(), "register_generate", false); +FileId FileManager::do_register_generate(unique_ptr<FullGenerateFileLocation> generate, DialogId owner_dialog_id, + int64 expected_size, string url) { + auto &file_id = generate_location_to_file_id_[*generate]; + if (!file_id.is_valid()) { + file_id = next_file_id(); + LOG(INFO) << "Register " << *generate << " as " << file_id; + + auto file_node_id = next_file_node_id(); + auto &node = file_nodes_[file_node_id]; + auto file_info = STORE_FILE_INFO ? td::make_unique<FileInfoGenerate>(*generate, expected_size, url) : nullptr; + node = td::make_unique<FileNode>(LocalFileLocation(), NewRemoteFileLocation(), std::move(generate), 0, + expected_size, string(), std::move(url), owner_dialog_id, FileEncryptionKey(), + file_id, static_cast<int8>(0)); + node->need_load_from_pmc_ = true; + + auto file_id_info = get_file_id_info(file_id); + file_id_info->node_id_ = file_node_id; + file_id_info->file_info_ = std::move(file_info); + file_id_info->pin_flag_ = true; + } + return file_id; } Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource file_location_source, - FileId merge_file_id, const char *source, bool force, - bool skip_file_size_checks) { + const char *source) { bool has_remote = data.remote_.type() == RemoteFileLocation::Type::Full; bool has_generate = data.generate_ != nullptr; - if (data.local_.type() == LocalFileLocation::Type::Full && !force) { + if (data.local_.type() == LocalFileLocation::Type::Full) { bool is_from_database = file_location_source == FileLocationSource::FromBinlog || file_location_source == FileLocationSource::FromDatabase; if (is_from_database) { @@ -1249,7 +1817,7 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi if (file_location_source != FileLocationSource::FromDatabase) { Status status; - auto r_info = check_full_local_location({data.local_.full(), data.size_}, skip_file_size_checks); + auto r_info = check_full_local_location({data.local_.full(), data.size_}, false); if (r_info.is_error()) { status = r_info.move_as_error(); } else if (bad_paths_.count(r_info.ok().location_.path_) != 0) { @@ -1271,7 +1839,7 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi } } else { // the location has been checked previously, but recheck it just in case - recheck_full_local_location({data.local_.full(), data.size_}, skip_file_size_checks); + recheck_full_local_location({data.local_.full(), data.size_}, false); } } bool has_local = data.local_.type() == LocalFileLocation::Type::Full; @@ -1281,14 +1849,25 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi } if (data.size_ < 0) { - LOG(ERROR) << "Receive file of size " << data.size_; + LOG(ERROR) << "Receive file of size " << data.size_ << " from " << source; data.size_ = 0; } if (data.expected_size_ < 0) { - LOG(ERROR) << "Receive file of expected size " << data.expected_size_; + LOG(ERROR) << "Receive file of expected size " << data.expected_size_ << " from " << source; data.expected_size_ = 0; } + if (data.remote_.type() == RemoteFileLocation::Type::Partial) { + auto &partial = data.remote_.partial(); + auto part_size = static_cast<int64>(partial.part_size_); + auto ready_part_count = partial.ready_part_count_; + auto remote_ready_size = partial.ready_size_; + partial.ready_size_ = max(part_size * ready_part_count, remote_ready_size); + if (data.size_ != 0 && data.size_ < partial.ready_size_) { + partial.ready_size_ = data.size_; + } + } + FileId file_id = next_file_id(); LOG(INFO) << "Register file data " << data << " as " << file_id << " from " << source; @@ -1300,8 +1879,8 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_, std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote)); node->pmc_id_ = FileDbId(data.pmc_id_); - get_file_id_info(file_id)->node_id_ = file_node_id; - node->file_ids_.push_back(file_id); + auto file_id_info = get_file_id_info(file_id); + file_id_info->node_id_ = file_node_id; FileView file_view(get_file_node(file_id)); @@ -1319,9 +1898,10 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi bool new_remote = false; FileId *new_remote_file_id = nullptr; int32 remote_key = 0; - if (file_view.has_remote_location()) { + const auto *full_remote_location = file_view.get_full_remote_location(); + if (full_remote_location != nullptr) { if (context_->keep_exact_remote_location()) { - RemoteInfo info{file_view.remote_location(), file_location_source, file_id}; + RemoteInfo info{*full_remote_location, file_location_source, file_id}; remote_key = remote_location_info_.add(info); auto &stored_info = remote_location_info_.get(remote_key); if (stored_info.file_id_ == file_id) { @@ -1329,29 +1909,31 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi new_remote = true; } else { to_merge.push_back(stored_info.file_id_); - if (merge_choose_remote_location(file_view.remote_location(), file_location_source, stored_info.remote_, + if (merge_choose_remote_location(*full_remote_location, file_location_source, stored_info.remote_, stored_info.file_location_source_) == 0) { - stored_info.remote_ = file_view.remote_location(); + stored_info.remote_ = *full_remote_location; stored_info.file_location_source_ = file_location_source; } } } else { - new_remote_file_id = register_location(file_view.remote_location(), remote_location_to_file_id_); + new_remote_file_id = register_location(*full_remote_location, remote_location_to_file_id_); new_remote = new_remote_file_id != nullptr; } } FileId *new_local_file_id = nullptr; - if (file_view.has_local_location()) { - new_local_file_id = register_location(file_view.local_location(), local_location_to_file_id_); + const auto *full_local_location = file_view.get_full_local_location(); + if (full_local_location != nullptr) { + new_local_file_id = register_location(*full_local_location, local_location_to_file_id_); } FileId *new_generate_file_id = nullptr; - if (file_view.has_generate_location()) { - new_generate_file_id = register_location(file_view.generate_location(), generate_location_to_file_id_); + const auto *generate_location = file_view.get_generate_location(); + if (generate_location != nullptr) { + new_generate_file_id = register_location(*generate_location, generate_location_to_file_id_); } td::unique(to_merge); int new_cnt = new_remote + (new_local_file_id != nullptr) + (new_generate_file_id != nullptr); - if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) { + if (data.pmc_id_ == 0 && new_cnt > 0) { node->need_load_from_pmc_ = true; } bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0; @@ -1359,10 +1941,6 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi // may invalidate node merge(file_id, id, no_sync_merge).ignore(); } - Status status; - if (merge_file_id.is_valid()) { - status = merge(file_id, merge_file_id); - } try_flush_node(get_file_node(file_id), "register_file"); auto main_file_id = get_file_node(file_id)->main_file_id_; @@ -1389,9 +1967,6 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi context_->add_file_source(main_file_id, file_source_id); } } - if (status.is_error()) { - return std::move(status); - } return FileId(main_file_id.get(), remote_key); } @@ -1526,7 +2101,7 @@ void FileManager::do_cancel_download(FileNodePtr node) { if (node->download_id_ == 0) { return; } - send_closure(file_load_manager_, &FileLoadManager::cancel, node->download_id_); + send_closure(file_download_manager_, &FileDownloadManager::cancel, node->download_id_); node->download_id_ = 0; node->is_download_started_ = false; node->download_was_update_file_reference_ = false; @@ -1537,7 +2112,7 @@ void FileManager::do_cancel_upload(FileNodePtr node) { if (node->upload_id_ == 0) { return; } - send_closure(file_load_manager_, &FileLoadManager::cancel, node->upload_id_); + send_closure(file_upload_manager_, &FileUploadManager::cancel, node->upload_id_); node->upload_id_ = 0; node->upload_was_update_file_reference_ = false; node->set_upload_priority(0); @@ -1634,8 +2209,9 @@ Status FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { if (size_i == -1) { try_flush_node_info(x_node, "merge 2"); try_flush_node_info(y_node, "merge 3"); - return Status::Error( - 400, PSLICE() << "Can't merge files. Different size: " << x_node->size_ << " and " << y_node->size_); + return Status::Error(400, PSLICE() << "Can't merge files " << x_node->local_ << '/' << x_node->remote_ << " and " + << y_node->local_ << '/' << y_node->remote_ + << ". Different size: " << x_node->size_ << " and " << y_node->size_); } if (encryption_key_i == -1) { if (nodes[remote_i]->remote_.full && nodes[local_i]->local_.type() != LocalFileLocation::Type::Partial) { @@ -1678,8 +2254,7 @@ Status FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { if (local_i == other_node_i) { do_cancel_download(node); node->set_download_offset(other_node->download_offset_); - node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_, - other_node->local_ready_prefix_size_); + node->set_local_location(other_node->local_, other_node->download_offset_, other_node->local_ready_prefix_size_); node->download_id_ = other_node->download_id_; node->download_was_update_file_reference_ = other_node->download_was_update_file_reference_; node->is_download_started_ |= other_node->is_download_started_; @@ -1777,7 +2352,7 @@ Status FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { bool send_updates_flag = false; auto other_pmc_id = other_node->pmc_id_; - node->file_ids_.insert(node->file_ids_.end(), other_node->file_ids_.begin(), other_node->file_ids_.end()); + append(node->file_ids_, other_node->file_ids_); for (auto file_id : other_node->file_ids_) { auto file_id_info = get_file_id_info(file_id); @@ -1801,7 +2376,7 @@ Status FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { // Check if some download/upload queries are ready for (auto file_id : vector<FileId>(node->file_ids_)) { auto *info = get_file_id_info(file_id); - if (info->download_priority_ != 0 && file_view.has_local_location()) { + if (info->download_priority_ != 0 && file_view.has_full_local_location()) { info->download_priority_ = 0; if (info->download_callback_) { info->download_callback_->on_download_ok(file_id); @@ -1832,6 +2407,31 @@ Status FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { return Status::OK(); } +void FileManager::try_merge_documents(FileId old_file_id, FileId new_file_id) { + if (!old_file_id.is_valid() || !new_file_id.is_valid()) { + return; + } + FileView old_file_view = get_file_view(old_file_id); + FileView new_file_view = get_file_view(new_file_id); + // if file type has changed, but file size remains the same, we are trying to update local location of the new + // file with the old local location + if (old_file_view.has_full_local_location() && !new_file_view.has_full_local_location() && + old_file_view.size() != 0 && old_file_view.size() == new_file_view.size()) { + auto old_file_type = old_file_view.get_type(); + auto new_file_type = new_file_view.get_type(); + + if (is_document_file_type(old_file_type) && is_document_file_type(new_file_type)) { + const auto *old_location = old_file_view.get_full_local_location(); + auto r_file_id = + register_local(FullLocalFileLocation(new_file_type, old_location->path_, old_location->mtime_nsec_), + DialogId(), old_file_view.size()); + if (r_file_id.is_ok()) { + LOG_STATUS(merge(new_file_id, r_file_id.ok())); + } + } + } +} + void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) { auto node = get_sync_file_node(file_id); // synchronously load the file to preload known file sources if (!node) { @@ -1967,11 +2567,11 @@ void FileManager::clear_from_pmc(FileNodePtr node) { LOG(INFO) << "Delete files " << format::as_array(node->file_ids_) << " from pmc"; FileData data; auto file_view = FileView(node); - if (file_view.has_local_location()) { + if (file_view.has_full_local_location()) { data.local_ = node->local_; prepare_path_for_pmc(data.local_.full().file_type_, data.local_.full().path_); } - if (file_view.has_remote_location()) { + if (file_view.has_full_remote_location()) { data.remote_ = RemoteFileLocation(*node->remote_.full); } if (file_view.has_generate_location()) { @@ -1986,7 +2586,7 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local if (!file_db_) { return; } - FileView view(node); + FileView file_view(node); bool create_flag = false; if (node->pmc_id_.empty()) { create_flag = true; @@ -2023,8 +2623,8 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local data.encryption_key_ = node->encryption_key_; data.url_ = node->url_; data.owner_dialog_id_ = node->owner_dialog_id_; - data.file_source_ids_ = context_->get_some_file_sources(view.get_main_file_id()); - VLOG(file_references) << "Save file " << view.get_main_file_id() << " to database with " << data.file_source_ids_ + data.file_source_ids_ = context_->get_some_file_sources(file_view.get_main_file_id()); + VLOG(file_references) << "Save file " << file_view.get_main_file_id() << " to database with " << data.file_source_ids_ << " from " << source; file_db_->set_file_data(node->pmc_id_, data, (create_flag || new_remote), (create_flag || new_local), @@ -2035,7 +2635,7 @@ FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_i if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) { return nullptr; } - FileNodeId node_id = file_id_info_[file_id.get()].node_id_; + FileNodeId node_id = file_id_info_[file_id.get()]->node_id_; if (node_id == 0) { return nullptr; } @@ -2058,37 +2658,48 @@ void FileManager::load_from_pmc(FileNodePtr node, bool new_remote, bool new_loca if (!node->need_load_from_pmc_) { return; } - auto file_id = node->main_file_id_; node->need_load_from_pmc_ = false; if (!file_db_) { return; } + auto file_id = node->main_file_id_; auto file_view = get_file_view(file_id); CHECK(!file_view.empty()); FullRemoteFileLocation remote; FullLocalFileLocation local; FullGenerateFileLocation generate; - new_remote &= file_view.has_remote_location(); if (new_remote) { - remote = file_view.remote_location(); + const auto *full_remote_location = file_view.get_full_remote_location(); + if (full_remote_location != nullptr) { + remote = *full_remote_location; + } else { + new_remote = false; + } } - new_local &= file_view.has_local_location(); if (new_local) { - local = file_view.local_location(); - prepare_path_for_pmc(local.file_type_, local.path_); + const auto *full_local_location = file_view.get_full_local_location(); + if (full_local_location != nullptr) { + local = *full_local_location; + prepare_path_for_pmc(local.file_type_, local.path_); + } else { + new_local = false; + } } - new_generate &= file_view.has_generate_location(); if (new_generate) { - generate = file_view.generate_location(); + const auto *generate_location = file_view.get_generate_location(); + if (generate_location != nullptr) { + generate = *generate_location; + } else { + new_generate = false; + } } LOG(DEBUG) << "Load from pmc file " << file_id << '/' << file_view.get_main_file_id() << ", new_remote = " << new_remote << ", new_local = " << new_local << ", new_generate = " << new_generate; auto load = [&](auto location, const char *source) { TRY_RESULT(file_data, file_db_->get_file_data_sync(location)); - TRY_RESULT(new_file_id, - register_file(std::move(file_data), FileLocationSource::FromDatabase, FileId(), source, false)); + TRY_RESULT(new_file_id, register_file(std::move(file_data), FileLocationSource::FromDatabase, source)); TRY_STATUS(merge(file_id, new_file_id)); // merge manually to keep merge parameters order return Status::OK(); }; @@ -2108,8 +2719,8 @@ bool FileManager::set_encryption_key(FileId file_id, FileEncryptionKey key) { if (!node) { return false; } - auto view = FileView(node); - if (view.has_local_location() && view.has_remote_location()) { + auto file_view = FileView(node); + if (file_view.has_full_local_location() && file_view.has_full_remote_location()) { return false; } if (!node->encryption_key_.empty()) { @@ -2146,11 +2757,12 @@ bool FileManager::set_content(FileId file_id, BufferSlice bytes) { node->set_download_priority(FROM_BYTES_PRIORITY); - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::SetContent}); + FileDownloadManager::QueryId query_id = + download_queries_.create(DownloadQuery{file_id, DownloadQuery::Type::SetContent}); node->download_id_ = query_id; node->is_download_started_ = true; - send_closure(file_load_manager_, &FileLoadManager::from_bytes, query_id, node->remote_.full.value().file_type_, - std::move(bytes), node->suggested_path()); + send_closure(file_download_manager_, &FileDownloadManager::from_bytes, query_id, + node->remote_.full.value().file_type_, std::move(bytes), node->suggested_path()); return true; } @@ -2162,11 +2774,12 @@ void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) { check_local_location(node, true).ignore(); auto file_view = FileView(node); - if (!file_view.has_local_location()) { + const auto *full_local_location = file_view.get_full_local_location(); + if (full_local_location == nullptr) { return promise.set_error(Status::Error("No local location")); } - send_closure(file_load_manager_, &FileLoadManager::get_content, node->local_.full().path_, std::move(promise)); + send_closure(file_load_manager_, &FileLoadManager::get_content, full_local_location->path_, std::move(promise)); } void FileManager::read_file_part(FileId file_id, int64 offset, int64 count, int left_tries, @@ -2204,8 +2817,9 @@ void FileManager::read_file_part(FileId file_id, int64 offset, int64 count, int const string *path = nullptr; bool is_partial = false; - if (file_view.has_local_location()) { - path = &file_view.local_location().path_; + const auto *full_local_location = file_view.get_full_local_location(); + if (full_local_location != nullptr) { + path = &full_local_location->path_; if (!begins_with(*path, get_files_dir(file_view.get_type()))) { return promise.set_error(Status::Error(400, "File is not inside the cache")); } @@ -2254,8 +2868,8 @@ void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char send_closure(G()->download_manager(), &DownloadManager::remove_file_if_finished, file_view.get_main_file_id()); string path; - if (file_view.has_local_location()) { - if (begins_with(file_view.local_location().path_, get_files_dir(file_view.get_type()))) { + if (file_view.has_full_local_location()) { + if (begins_with(file_view.get_full_local_location()->path_, get_files_dir(file_view.get_type()))) { clear_from_pmc(node); if (context_->need_notify_on_new_files()) { context_->on_new_file(-file_view.size(), -file_view.get_allocated_local_size(), -1); @@ -2432,7 +3046,7 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { LOG(INFO) << "Update download offset and limits of file " << node->main_file_id_; CHECK(node->download_id_ != 0); if (force_update_priority || priority != old_priority) { - send_closure(file_load_manager_, &FileLoadManager::update_priority, node->download_id_, priority); + send_closure(file_download_manager_, &FileDownloadManager::update_priority, node->download_id_, priority); } if (need_update_limit || need_update_offset) { auto download_offset = node->download_offset_; @@ -2443,8 +3057,8 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { download_limit += download_offset; download_offset = 0; } - send_closure(file_load_manager_, &FileLoadManager::update_downloaded_part, node->download_id_, download_offset, - download_limit); + send_closure(file_download_manager_, &FileDownloadManager::update_downloaded_part, node->download_id_, + download_offset, download_limit); } return; } @@ -2455,10 +3069,11 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { if (node->need_reload_photo_ && file_view.may_reload_photo()) { LOG(INFO) << "Reload photo from file " << node->main_file_id_; - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::DownloadReloadDialog}); + FileDownloadManager::QueryId query_id = + download_queries_.create(DownloadQuery{file_id, DownloadQuery::Type::DownloadReloadDialog}); node->download_id_ = query_id; - context_->reload_photo(file_view.remote_location().get_source(), - PromiseCreator::lambda([query_id, actor_id = actor_id(this), file_id](Result<Unit> res) { + context_->reload_photo(file_view.get_full_remote_location()->get_source(), + PromiseCreator::lambda([actor_id = actor_id(this), query_id, file_id](Result<Unit> res) { Status error; if (res.is_ok()) { error = Status::Error("FILE_DOWNLOAD_ID_INVALID"); @@ -2467,7 +3082,7 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { } VLOG(file_references) << "Receive result from reload photo for file " << file_id << ": " << error; - send_closure(actor_id, &FileManager::on_error, query_id, std::move(error)); + send_closure(actor_id, &FileManager::on_download_error, query_id, std::move(error)); })); node->need_reload_photo_ = false; return; @@ -2476,15 +3091,16 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { // If file reference is needed if (!file_view.has_active_download_remote_location()) { VLOG(file_references) << "Do not have valid file_reference for file " << file_id; - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::DownloadWaitFileReference}); + FileDownloadManager::QueryId query_id = + download_queries_.create(DownloadQuery{file_id, DownloadQuery::Type::DownloadWaitFileReference}); node->download_id_ = query_id; if (node->download_was_update_file_reference_) { - return on_error(query_id, Status::Error("Can't download file: have no valid file reference")); + return on_download_error(query_id, Status::Error("Can't download file: have no valid file reference")); } node->download_was_update_file_reference_ = true; context_->repair_file_reference( - file_id, PromiseCreator::lambda([query_id, actor_id = actor_id(this), file_id](Result<Unit> res) { + file_id, PromiseCreator::lambda([actor_id = actor_id(this), query_id, file_id](Result<Unit> res) { Status error; if (res.is_ok()) { error = Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE"); @@ -2492,12 +3108,13 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { error = res.move_as_error(); } VLOG(file_references) << "Receive result from FileSourceManager for file " << file_id << ": " << error; - send_closure(actor_id, &FileManager::on_error, query_id, std::move(error)); + send_closure(actor_id, &FileManager::on_download_error, query_id, std::move(error)); })); return; } - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::Download}); + FileDownloadManager::QueryId query_id = + download_queries_.create(DownloadQuery{file_id, DownloadQuery::Type::Download}); node->download_id_ = query_id; node->is_download_started_ = false; LOG(INFO) << "Run download of file " << file_id << " of size " << node->size_ << " from " @@ -2511,9 +3128,9 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { download_limit += download_offset; download_offset = 0; } - send_closure(file_load_manager_, &FileLoadManager::download, query_id, node->remote_.full.value(), node->local_, - node->size_, node->suggested_path(), node->encryption_key_, node->can_search_locally_, download_offset, - download_limit, priority); + send_closure(file_download_manager_, &FileDownloadManager::download, query_id, node->remote_.full.value(), + node->local_, node->size_, node->suggested_path(), node->encryption_key_, node->can_search_locally_, + download_offset, download_limit, priority); } class FileManager::ForceUploadActor final : public Actor { @@ -2679,7 +3296,11 @@ void FileManager::resume_upload(FileId file_id, vector<int> bad_parts, std::shar .release(); return; } - LOG(INFO) << "Resume upload of file " << file_id << " with priority " << new_priority << " and force = " << force; + if (new_priority == 0) { + LOG(INFO) << "Cancel upload of file " << file_id; + } else { + LOG(INFO) << "Resume upload of file " << file_id << " with priority " << new_priority << " and force = " << force; + } if (force) { node->remote_.is_full_alive = false; @@ -2695,21 +3316,22 @@ void FileManager::resume_upload(FileId file_id, vector<int> bad_parts, std::shar }; FileView file_view(node); if (file_view.has_active_upload_remote_location() && can_reuse_remote_file(file_view.get_type())) { - LOG(INFO) << "File " << file_id << " is already uploaded"; + LOG(INFO) << "Upload of file " << file_id << " has already been completed"; if (callback) { callback->on_upload_ok(file_id, nullptr); } return; } - if (file_view.has_local_location() && new_priority != 0) { + if (file_view.has_full_local_location() && new_priority != 0) { auto status = check_local_location(node, false); if (status.is_error()) { LOG(INFO) << "Full local location of file " << file_id << " for upload is invalid: " << status; } } - if (!file_view.has_local_location() && !file_view.has_generate_location() && !file_view.has_alive_remote_location()) { + if (!file_view.has_full_local_location() && !file_view.has_generate_location() && + !file_view.has_alive_remote_location()) { LOG(INFO) << "File " << file_id << " can't be uploaded"; if (callback) { callback->on_upload_error( @@ -2718,7 +3340,7 @@ void FileManager::resume_upload(FileId file_id, vector<int> bad_parts, std::shar return; } if (file_view.get_type() == FileType::Thumbnail && - (!file_view.has_local_location() && file_view.can_download_from_server())) { + (!file_view.has_full_local_location() && file_view.can_download_from_server())) { // TODO if (callback) { callback->on_upload_error(file_id, Status::Error(400, "Failed to upload thumbnail without local location")); @@ -2786,6 +3408,14 @@ bool FileManager::delete_partial_remote_location(FileId file_id) { return true; } +void FileManager::delete_partial_remote_location_if_needed(FileId file_id, const Status &error) { + if (error.code() != 429 && error.code() < 500 && !G()->close_flag()) { + delete_partial_remote_location(file_id); + } else { + cancel_upload(file_id); + } +} + void FileManager::delete_file_reference(FileId file_id, Slice file_reference) { VLOG(file_references) << "Delete file reference of file " << file_id << " " << tag("reference_base64", base64_encode(file_reference)); @@ -2810,18 +3440,19 @@ void FileManager::delete_file_reference(FileId file_id, Slice file_reference) { void FileManager::external_file_generate_write_part(int64 generation_id, int64 offset, string data, Promise<> promise) { send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, - static_cast<uint64>(generation_id), offset, std::move(data), std::move(promise)); + static_cast<FileGenerateManager::QueryId>(generation_id), offset, std::move(data), std::move(promise)); } void FileManager::external_file_generate_progress(int64 generation_id, int64 expected_size, int64 local_prefix_size, Promise<> promise) { send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, - static_cast<uint64>(generation_id), expected_size, local_prefix_size, std::move(promise)); + static_cast<FileGenerateManager::QueryId>(generation_id), expected_size, local_prefix_size, + std::move(promise)); } void FileManager::external_file_generate_finish(int64 generation_id, Status status, Promise<> promise) { send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, - static_cast<uint64>(generation_id), std::move(status), std::move(promise)); + static_cast<FileGenerateManager::QueryId>(generation_id), std::move(status), std::move(promise)); } void FileManager::run_generate(FileNodePtr node) { @@ -2834,7 +3465,7 @@ void FileManager::run_generate(FileNodePtr node) { // LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " can't be generated"; return; } - if (file_view.has_local_location()) { + if (file_view.has_full_local_location()) { LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " has local location"; return; } @@ -2878,30 +3509,31 @@ void FileManager::run_generate(FileNodePtr node) { return; } - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::Generate}); + FileGenerateManager::QueryId query_id = generate_queries_.create(GenerateQuery{file_id}); node->generate_id_ = query_id; - send_closure( - file_generate_manager_, &FileGenerateManager::generate_file, query_id, *node->generate_, node->local_, - node->suggested_path(), [file_manager = this, query_id] { - class Callback final : public FileGenerateCallback { - ActorId<FileManager> actor_; - uint64 query_id_; - - public: - Callback(ActorId<FileManager> actor, QueryId query_id) : actor_(std::move(actor)), query_id_(query_id) { - } - void on_partial_generate(PartialLocalFileLocation partial_local, int64 expected_size) final { - send_closure(actor_, &FileManager::on_partial_generate, query_id_, std::move(partial_local), expected_size); - } - void on_ok(FullLocalFileLocation local) final { - send_closure(actor_, &FileManager::on_generate_ok, query_id_, std::move(local)); - } - void on_error(Status error) final { - send_closure(actor_, &FileManager::on_error, query_id_, std::move(error)); - } - }; - return make_unique<Callback>(file_manager->actor_id(file_manager), query_id); - }()); + send_closure(file_generate_manager_, &FileGenerateManager::generate_file, query_id, *node->generate_, node->local_, + node->suggested_path(), [file_manager = this, query_id] { + class Callback final : public FileGenerateCallback { + ActorId<FileManager> actor_; + uint64 query_id_; + + public: + Callback(ActorId<FileManager> actor, FileGenerateManager::QueryId query_id) + : actor_(std::move(actor)), query_id_(query_id) { + } + void on_partial_generate(PartialLocalFileLocation partial_local, int64 expected_size) final { + send_closure(actor_, &FileManager::on_partial_generate, query_id_, std::move(partial_local), + expected_size); + } + void on_ok(FullLocalFileLocation local) final { + send_closure(actor_, &FileManager::on_generate_ok, query_id_, std::move(local)); + } + void on_error(Status error) final { + send_closure(actor_, &FileManager::on_generate_error, query_id_, std::move(error)); + } + }; + return make_unique<Callback>(file_manager->actor_id(file_manager), query_id); + }()); LOG(INFO) << "File " << file_id << " generate request has sent to FileGenerateManager"; } @@ -2938,13 +3570,14 @@ void FileManager::run_upload(FileNodePtr node, vector<int> bad_parts) { } FileView file_view(node); - if (!file_view.has_local_location() && !file_view.has_remote_location()) { + if (!file_view.has_full_local_location() && !file_view.has_full_remote_location()) { if (node->get_by_hash_ || node->generate_id_ == 0 || !node->generate_was_update_) { LOG(INFO) << "Have no local location for file: get_by_hash = " << node->get_by_hash_ << ", generate_id = " << node->generate_id_ << ", generate_was_update = " << node->generate_was_update_; return; } - if (file_view.has_generate_location() && file_view.generate_location().file_type_ == FileType::SecureEncrypted) { + auto generate_location = file_view.get_generate_location(); + if (generate_location != nullptr && generate_location->file_type_ == FileType::SecureEncrypted) { // Can't upload secure file before its size is known LOG(INFO) << "Can't upload secure file " << node->main_file_id_ << " before it's size is known"; return; @@ -2953,9 +3586,12 @@ void FileManager::run_upload(FileNodePtr node, vector<int> bad_parts) { node->set_upload_priority(priority); + auto generate_location = file_view.get_generate_location(); + auto full_local_location = file_view.get_full_local_location(); + // create encryption key if necessary - if (((file_view.has_generate_location() && file_view.generate_location().file_type_ == FileType::Encrypted) || - (file_view.has_local_location() && file_view.local_location().file_type_ == FileType::Encrypted)) && + if (((generate_location != nullptr && generate_location->file_type_ == FileType::Encrypted) || + (full_local_location != nullptr && full_local_location->file_type_ == FileType::Encrypted)) && file_view.encryption_key().empty()) { CHECK(!node->file_ids_.empty()); bool success = set_encryption_key(node->file_ids_[0], FileEncryptionKey::create()); @@ -2963,7 +3599,7 @@ void FileManager::run_upload(FileNodePtr node, vector<int> bad_parts) { } // create encryption key if necessary - if (file_view.has_local_location() && file_view.local_location().file_type_ == FileType::SecureEncrypted && + if (full_local_location != nullptr && full_local_location->file_type_ == FileType::SecureEncrypted && file_view.encryption_key().empty()) { CHECK(!node->file_ids_.empty()); bool success = set_encryption_key(node->file_ids_[0], FileEncryptionKey::create_secure_key()); @@ -2973,23 +3609,25 @@ void FileManager::run_upload(FileNodePtr node, vector<int> bad_parts) { if (old_priority != 0) { LOG(INFO) << "File " << file_id << " is already uploading"; CHECK(node->upload_id_ != 0); - send_closure(file_load_manager_, &FileLoadManager::update_priority, node->upload_id_, narrow_cast<int8>(-priority)); + send_closure(file_upload_manager_, &FileUploadManager::update_priority, node->upload_id_, + narrow_cast<int8>(-priority)); return; } CHECK(node->upload_id_ == 0); if (file_view.has_alive_remote_location() && !file_view.has_active_upload_remote_location() && can_reuse_remote_file(file_view.get_type())) { - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::UploadWaitFileReference}); + FileUploadManager::QueryId query_id = + upload_queries_.create(UploadQuery{file_id, UploadQuery::Type::UploadWaitFileReference}); node->upload_id_ = query_id; if (node->upload_was_update_file_reference_) { - return on_error(query_id, Status::Error("Can't upload file: have no valid file reference")); + return on_upload_error(query_id, Status::Error("Can't upload file: have no valid file reference")); } node->upload_was_update_file_reference_ = true; context_->repair_file_reference(node->main_file_id_, - PromiseCreator::lambda([query_id, actor_id = actor_id(this)](Result<Unit> res) { - send_closure(actor_id, &FileManager::on_error, query_id, + PromiseCreator::lambda([actor_id = actor_id(this), query_id](Result<Unit> res) { + send_closure(actor_id, &FileManager::on_upload_error, query_id, Status::Error("FILE_UPLOAD_RESTART_WITH_FILE_REFERENCE")); })); return; @@ -2997,10 +3635,10 @@ void FileManager::run_upload(FileNodePtr node, vector<int> bad_parts) { if (!node->remote_.partial && node->get_by_hash_) { LOG(INFO) << "Get file " << node->main_file_id_ << " by hash"; - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::UploadByHash}); + FileUploadManager::QueryId query_id = upload_queries_.create(UploadQuery{file_id, UploadQuery::Type::UploadByHash}); node->upload_id_ = query_id; - send_closure(file_load_manager_, &FileLoadManager::upload_by_hash, query_id, node->local_.full(), node->size_, + send_closure(file_upload_manager_, &FileUploadManager::upload_by_hash, query_id, node->local_.full(), node->size_, narrow_cast<int8>(-priority)); return; } @@ -3013,12 +3651,13 @@ void FileManager::run_upload(FileNodePtr node, vector<int> bad_parts) { expected_size = 10 << 20; } - QueryId query_id = queries_container_.create(Query{file_id, Query::Type::Upload}); + FileUploadManager::QueryId query_id = upload_queries_.create(UploadQuery{file_id, UploadQuery::Type::Upload}); node->upload_id_ = query_id; - send_closure(file_load_manager_, &FileLoadManager::upload, query_id, node->local_, node->remote_.partial_or_empty(), - expected_size, node->encryption_key_, new_priority, std::move(bad_parts)); + send_closure(file_upload_manager_, &FileUploadManager::upload, query_id, node->local_, + node->remote_.partial_or_empty(), expected_size, node->encryption_key_, new_priority, + std::move(bad_parts)); - LOG(INFO) << "File " << file_id << " upload request has sent to FileLoadManager"; + LOG(INFO) << "File " << file_id << " upload request has sent to FileUploadManager"; } void FileManager::upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority, @@ -3044,7 +3683,7 @@ Result<FileId> FileManager::from_persistent_id(CSlice persistent_id, FileType fi if (!clean_input_string(url)) { return Status::Error(400, "URL must be in UTF-8"); } - return register_url(std::move(url), file_type, FileLocationSource::FromUser, DialogId()); + return register_url(std::move(url), file_type, DialogId()); } auto r_binary = base64url_decode(persistent_id); @@ -3083,10 +3722,8 @@ Result<FileId> FileManager::from_persistent_id_generated(Slice binary, FileType if (!is_remotely_generated_file(generate_location.conversion_)) { return Status::Error(400, "Unexpected conversion type"); } - FileData data; - data.generate_ = make_unique<FullGenerateFileLocation>(std::move(generate_location)); - return register_file(std::move(data), FileLocationSource::FromUser, FileId(), "from_persistent_id_generated", false) - .move_as_ok(); + return do_register_generate(make_unique<FullGenerateFileLocation>(std::move(generate_location)), DialogId(), 0, + string()); } Result<FileId> FileManager::from_persistent_id_v23(Slice binary, FileType file_type, int32 version) { @@ -3111,11 +3748,7 @@ Result<FileId> FileManager::from_persistent_id_v23(Slice binary, FileType file_t } else if (real_file_type != file_type && file_type != FileType::Temp) { return Status::Error(400, PSLICE() << "Can't use file of type " << real_file_type << " as " << file_type); } - FileData data; - data.remote_ = RemoteFileLocation(std::move(remote_location)); - auto file_id = register_file(std::move(data), FileLocationSource::FromUser, FileId(), "from_persistent_id_v23", false) - .move_as_ok(); - return file_id; + return register_remote(std::move(remote_location), FileLocationSource::FromUser, DialogId(), 0, 0, string()); } Result<FileId> FileManager::from_persistent_id_v2(Slice binary, FileType file_type) { @@ -3150,45 +3783,50 @@ FileView FileManager::get_sync_file_view(FileId file_id) { } td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool with_main_file_id) { - auto file_view = get_sync_file_view(file_id); - - if (file_view.empty()) { + auto file_node_ptr = get_sync_file_node(file_id); + if (!file_node_ptr) { return td_api::make_object<td_api::file>(0, 0, 0, td_api::make_object<td_api::localFile>(), td_api::make_object<td_api::remoteFile>()); } - string persistent_file_id = file_view.get_persistent_file_id(); - string unique_file_id = file_view.get_unique_file_id(); + const FileNode *file_node = file_node_ptr.get(); + string persistent_file_id = file_node->get_persistent_file_id(); + string unique_file_id = file_node->get_unique_file_id(); + bool is_downloading_completed = file_node->local_.type() == LocalFileLocation::Type::Full; bool is_uploading_completed = !persistent_file_id.empty(); - auto size = file_view.size(); - auto expected_size = file_view.expected_size(); - auto download_offset = file_view.download_offset(); - auto local_prefix_size = file_view.local_prefix_size(); - auto local_total_size = file_view.local_total_size(); - auto remote_size = file_view.remote_size(); - string path = file_view.path(); + auto size = file_node->size_; + auto download_offset = file_node->download_offset_; + auto expected_size = file_node->expected_size(); + auto local_prefix_size = file_node->local_prefix_size(); + auto local_total_size = file_node->local_total_size(); + auto remote_size = file_node->remote_size(); + string path = file_node->path(); + bool can_be_deleted = file_node->can_delete(); + + auto file_view = FileView(file_node_ptr); bool can_be_downloaded = file_view.can_download_from_server() || file_view.can_generate(); - bool can_be_deleted = file_view.can_delete(); auto result_file_id = file_id; auto *file_info = get_file_id_info(result_file_id); + FileId main_file_id; if (with_main_file_id) { + main_file_id = file_node->main_file_id_; if (!file_info->send_updates_flag_) { - result_file_id = file_view.get_main_file_id(); + result_file_id = main_file_id; } - file_info = get_file_id_info(file_view.get_main_file_id()); + file_info = get_file_id_info(main_file_id); } file_info->send_updates_flag_ = true; VLOG(update_file) << "Send file " << file_id << " as " << result_file_id << " and update send_updates_flag_ for file " - << (with_main_file_id ? file_view.get_main_file_id() : result_file_id); + << (with_main_file_id ? main_file_id : result_file_id); return td_api::make_object<td_api::file>( result_file_id.get(), size, expected_size, td_api::make_object<td_api::localFile>(std::move(path), can_be_downloaded, can_be_deleted, - file_view.is_downloading(), file_view.has_local_location(), - download_offset, local_prefix_size, local_total_size), + file_node->is_downloading(), is_downloading_completed, download_offset, + local_prefix_size, local_total_size), td_api::make_object<td_api::remoteFile>(std::move(persistent_file_id), std::move(unique_file_id), - file_view.is_uploading(), is_uploading_completed, remote_size)); + file_node->is_uploading(), is_uploading_completed, remote_size)); } vector<int32> FileManager::get_file_ids_object(const vector<FileId> &file_ids, bool with_main_file_id) { @@ -3197,10 +3835,11 @@ vector<int32> FileManager::get_file_ids_object(const vector<FileId> &file_ids, b auto result_file_id = file_id; auto *file_info = get_file_id_info(result_file_id); if (with_main_file_id) { + auto main_file_id = file_view.get_main_file_id(); if (!file_info->sent_file_id_flag_ && !file_info->send_updates_flag_) { - result_file_id = file_view.get_main_file_id(); + result_file_id = main_file_id; } - file_info = get_file_id_info(file_view.get_main_file_id()); + file_info = get_file_id_info(main_file_id); } file_info->sent_file_id_flag_ = true; @@ -3226,13 +3865,16 @@ Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> re if (real_type != type && !(real_type == FileType::Temp && file_view.has_url()) && !(is_document_file_type(real_type) && is_document_file_type(type)) && !(is_background_type(real_type) && is_background_type(type)) && - !(file_view.is_encrypted() && type == FileType::Ringtone)) { + !(file_view.is_encrypted() && type == FileType::Ringtone) && + !(real_type == FileType::PhotoStory && type == FileType::Photo) && + !(real_type == FileType::Photo && type == FileType::PhotoStory)) { // TODO: send encrypted file to unencrypted chat return Status::Error(400, PSLICE() << "Can't use file of type " << real_type << " as " << type); } } - if (!file_view.has_remote_location()) { + const auto *full_remote_location = file_view.get_full_remote_location(); + if (full_remote_location == nullptr) { // There are no reasons to dup file_id, because it will be duped anyway before upload/reupload // It will not be duped in dup_message_content only if has_input_media(), // but currently in this case the file never needs to be reuploaded @@ -3248,7 +3890,7 @@ Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> re int32 remote_id = file_id.get_remote(); if (remote_id == 0 && context_->keep_exact_remote_location()) { - RemoteInfo info{file_view.remote_location(), FileLocationSource::FromUser, file_id}; + RemoteInfo info{*full_remote_location, FileLocationSource::FromUser, file_id}; remote_id = remote_location_info_.add(info); if (remote_location_info_.get(remote_id).file_id_ == file_id) { get_file_id_info(file_id)->pin_flag_ = true; @@ -3277,8 +3919,8 @@ Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_a case td_api::inputFileGenerated::ID: { auto *generated_thumbnail = static_cast<const td_api::inputFileGenerated *>(thumbnail_input_file.get()); return register_generate(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, - FileLocationSource::FromUser, generated_thumbnail->original_path_, - generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_); + generated_thumbnail->original_path_, generated_thumbnail->conversion_, owner_dialog_id, + generated_thumbnail->expected_size_); } default: UNREACHABLE(); @@ -3320,17 +3962,24 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr auto file_id = file_hash_to_file_id_.get(hash); LOG(INFO) << "Found file " << file_id << " by hash " << hex_encode(hash); if (file_id.is_valid()) { - auto file_view = get_file_view(file_id); + auto file_node = get_file_node(file_id); + auto file_view = FileView(file_node); if (!file_view.empty()) { if (force_reuse) { return file_id; } - if (file_view.has_remote_location() && !file_view.remote_location().is_web()) { + const auto *full_remote_location = file_view.get_full_remote_location(); + if (full_remote_location != nullptr && !full_remote_location->is_web()) { return file_id; } if (file_view.is_uploading()) { + CHECK(file_node); + LOG(DEBUG) << "File " << file_id << " is still uploading: " << file_node->upload_priority_ << ' ' + << file_node->generate_upload_priority_ << ' ' << file_node->upload_pause_; hash.clear(); } + } else { + LOG(DEBUG) << "File " << file_id << " isn't found"; } } } @@ -3358,8 +4007,8 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr } case td_api::inputFileGenerated::ID: { auto *generated_file = static_cast<const td_api::inputFileGenerated *>(file.get()); - return register_generate(new_type, FileLocationSource::FromUser, generated_file->original_path_, - generated_file->conversion_, owner_dialog_id, generated_file->expected_size_); + return register_generate(new_type, generated_file->original_path_, generated_file->conversion_, owner_dialog_id, + generated_file->expected_size_); } default: UNREACHABLE(); @@ -3400,7 +4049,7 @@ Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 z << scale << '#'; return register_generate( owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail, - FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0); + string(), std::move(conversion), owner_dialog_id, 0); } Result<FileId> FileManager::get_audio_thumbnail_file_id(string title, string performer, bool is_small, @@ -3430,7 +4079,7 @@ Result<FileId> FileManager::get_audio_thumbnail_file_id(string title, string per string conversion = PSTRING() << "#audio_t#" << title << '#' << performer << '#' << (is_small ? '1' : '0') << '#'; return register_generate( owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail, - FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0); + string(), std::move(conversion), owner_dialog_id, 0); } FileType FileManager::guess_file_type(const tl_object_ptr<td_api::InputFile> &file) { @@ -3438,34 +4087,6 @@ FileType FileManager::guess_file_type(const tl_object_ptr<td_api::InputFile> &fi return FileType::Temp; } - auto guess_file_type_by_path = [](const string &file_path) { - PathView path_view(file_path); - auto file_name = path_view.file_name(); - auto extension = path_view.extension(); - if (extension == "jpg" || extension == "jpeg") { - return FileType::Photo; - } - if (extension == "ogg" || extension == "oga" || extension == "opus") { - return FileType::VoiceNote; - } - if (extension == "3gp" || extension == "mov") { - return FileType::Video; - } - if (extension == "mp3" || extension == "mpeg3" || extension == "m4a") { - return FileType::Audio; - } - if (extension == "webp" || extension == "tgs" || extension == "webm") { - return FileType::Sticker; - } - if (extension == "gif") { - return FileType::Animation; - } - if (extension == "mp4" || extension == "mpeg4") { - return to_lower(file_name).find("-gif-") != string::npos ? FileType::Animation : FileType::Video; - } - return FileType::Document; - }; - switch (file->get_id()) { case td_api::inputFileLocal::ID: return guess_file_type_by_path(static_cast<const td_api::inputFileLocal *>(file.get())->path_); @@ -3503,32 +4124,60 @@ vector<tl_object_ptr<telegram_api::InputDocument>> FileManager::get_input_docume for (auto file_id : file_ids) { auto file_view = get_file_view(file_id); CHECK(!file_view.empty()); - CHECK(file_view.has_remote_location()); - CHECK(!file_view.remote_location().is_web()); - result.push_back(file_view.remote_location().as_input_document()); + const auto *full_remote_location = file_view.get_full_remote_location(); + CHECK(full_remote_location != nullptr); + CHECK(!full_remote_location->is_web()); + result.push_back(full_remote_location->as_input_document()); } return result; } -bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputMedia> &input_media) { +bool FileManager::extract_was_uploaded(const telegram_api::object_ptr<telegram_api::InputMedia> &input_media) { if (input_media == nullptr) { return false; } auto input_media_id = input_media->get_id(); + if (input_media_id == telegram_api::inputMediaPaidMedia::ID) { + auto &extended_media = static_cast<const telegram_api::inputMediaPaidMedia *>(input_media.get())->extended_media_; + if (extended_media.size() > 1u) { + for (auto &media : extended_media) { + CHECK(!extract_was_uploaded(media)); + } + return false; + } + CHECK(extended_media.size() == 1u); + return extract_was_uploaded(extended_media[0]); + } return input_media_id == telegram_api::inputMediaUploadedPhoto::ID || input_media_id == telegram_api::inputMediaUploadedDocument::ID; } -bool FileManager::extract_was_thumbnail_uploaded(const tl_object_ptr<telegram_api::InputMedia> &input_media) { - if (input_media == nullptr || input_media->get_id() != telegram_api::inputMediaUploadedDocument::ID) { +bool FileManager::extract_was_thumbnail_uploaded( + const telegram_api::object_ptr<telegram_api::InputMedia> &input_media) { + if (input_media == nullptr) { return false; } - - return static_cast<const telegram_api::inputMediaUploadedDocument *>(input_media.get())->thumb_ != nullptr; + switch (input_media->get_id()) { + case telegram_api::inputMediaUploadedDocument::ID: + return static_cast<const telegram_api::inputMediaUploadedDocument *>(input_media.get())->thumb_ != nullptr; + case telegram_api::inputMediaPaidMedia::ID: { + auto &extended_media = static_cast<const telegram_api::inputMediaPaidMedia *>(input_media.get())->extended_media_; + if (extended_media.size() > 1u) { + for (auto &media : extended_media) { + CHECK(!extract_was_thumbnail_uploaded(media)); + } + return false; + } + CHECK(extended_media.size() == 1u); + return extract_was_thumbnail_uploaded(extended_media[0]); + } + default: + return false; + } } -string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> &input_media) { +string FileManager::extract_file_reference(const telegram_api::object_ptr<telegram_api::InputMedia> &input_media) { if (input_media == nullptr) { return string(); } @@ -3538,12 +4187,43 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp return extract_file_reference(static_cast<const telegram_api::inputMediaDocument *>(input_media.get())->id_); case telegram_api::inputMediaPhoto::ID: return extract_file_reference(static_cast<const telegram_api::inputMediaPhoto *>(input_media.get())->id_); + case telegram_api::inputMediaPaidMedia::ID: + UNREACHABLE(); + return string(); + case telegram_api::inputMediaUploadedDocument::ID: { + auto uploaded_document = static_cast<const telegram_api::inputMediaUploadedDocument *>(input_media.get()); + if (uploaded_document->file_->get_id() != telegram_api::inputFileStoryDocument::ID) { + return string(); + } + return extract_file_reference( + static_cast<const telegram_api::inputFileStoryDocument *>(uploaded_document->file_.get())->id_); + } default: return string(); } } -string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputDocument> &input_document) { +vector<string> FileManager::extract_file_references( + const telegram_api::object_ptr<telegram_api::InputMedia> &input_media) { + if (input_media == nullptr) { + return {}; + } + switch (input_media->get_id()) { + case telegram_api::inputMediaDocument::ID: + case telegram_api::inputMediaPhoto::ID: + return {extract_file_reference(input_media)}; + case telegram_api::inputMediaPaidMedia::ID: + return transform(static_cast<const telegram_api::inputMediaPaidMedia *>(input_media.get())->extended_media_, + [](const telegram_api::object_ptr<telegram_api::InputMedia> &media) { + return extract_file_reference(media); + }); + default: + return {}; + } +} + +string FileManager::extract_file_reference( + const telegram_api::object_ptr<telegram_api::InputDocument> &input_document) { if (input_document == nullptr || input_document->get_id() != telegram_api::inputDocument::ID) { return string(); } @@ -3551,7 +4231,7 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp return static_cast<const telegram_api::inputDocument *>(input_document.get())->file_reference_.as_slice().str(); } -string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputPhoto> &input_photo) { +string FileManager::extract_file_reference(const telegram_api::object_ptr<telegram_api::InputPhoto> &input_photo) { if (input_photo == nullptr || input_photo->get_id() != telegram_api::inputPhoto::ID) { return string(); } @@ -3559,11 +4239,12 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp return static_cast<const telegram_api::inputPhoto *>(input_photo.get())->file_reference_.as_slice().str(); } -bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) { +bool FileManager::extract_was_uploaded(const telegram_api::object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) { return input_chat_photo != nullptr && input_chat_photo->get_id() == telegram_api::inputChatUploadedPhoto::ID; } -string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) { +string FileManager::extract_file_reference( + const telegram_api::object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) { if (input_chat_photo == nullptr || input_chat_photo->get_id() != telegram_api::inputChatPhoto::ID) { return string(); } @@ -3579,7 +4260,7 @@ FileId FileManager::next_file_id() { } CHECK(file_id_info_.size() <= static_cast<size_t>(std::numeric_limits<int32>::max())); FileId res(static_cast<int32>(file_id_info_.size()), 0); - file_id_info_.push_back({}); + file_id_info_.push_back(make_unique<FileIdInfo>()); return res; } @@ -3590,12 +4271,12 @@ FileManager::FileNodeId FileManager::next_file_node_id() { return res; } -void FileManager::on_start_download(QueryId query_id) { +void FileManager::on_start_download(FileDownloadManager::QueryId query_id) { if (is_closed_) { return; } - auto query = queries_container_.get(query_id); + auto query = download_queries_.get(query_id); CHECK(query != nullptr); auto file_id = query->file_id_; @@ -3612,19 +4293,19 @@ void FileManager::on_start_download(QueryId query_id) { file_node->is_download_started_ = true; } -void FileManager::on_partial_download(QueryId query_id, PartialLocalFileLocation partial_local, int64 ready_size, +void FileManager::on_partial_download(FileDownloadManager::QueryId query_id, PartialLocalFileLocation partial_local, int64 size) { if (is_closed_) { return; } - auto query = queries_container_.get(query_id); + auto query = download_queries_.get(query_id); CHECK(query != nullptr); auto file_id = query->file_id_; auto file_node = get_file_node(file_id); LOG(DEBUG) << "Receive on_partial_download for file " << file_id << " with " << partial_local - << ", ready_size = " << ready_size << " and size = " << size; + << " and size = " << size; if (!file_node) { return; } @@ -3638,16 +4319,16 @@ void FileManager::on_partial_download(QueryId query_id, PartialLocalFileLocation file_node->set_size(size); } } - file_node->set_local_location(LocalFileLocation(std::move(partial_local)), ready_size, -1, -1 /* TODO */); + file_node->set_local_location(LocalFileLocation(std::move(partial_local)), -1, -1 /* TODO */); try_flush_node(file_node, "on_partial_download"); } -void FileManager::on_hash(QueryId query_id, string hash) { +void FileManager::on_hash(FileUploadManager::QueryId query_id, string hash) { if (is_closed_) { return; } - auto query = queries_container_.get(query_id); + auto query = upload_queries_.get(query_id); CHECK(query != nullptr); auto file_id = query->file_id_; @@ -3664,18 +4345,17 @@ void FileManager::on_hash(QueryId query_id, string hash) { file_node->encryption_key_.set_value_hash(secure_storage::ValueHash::create(hash).move_as_ok()); } -void FileManager::on_partial_upload(QueryId query_id, PartialRemoteFileLocation partial_remote, int64 ready_size) { +void FileManager::on_partial_upload(FileUploadManager::QueryId query_id, PartialRemoteFileLocation partial_remote) { if (is_closed_) { return; } - auto query = queries_container_.get(query_id); + auto query = upload_queries_.get(query_id); CHECK(query != nullptr); auto file_id = query->file_id_; auto file_node = get_file_node(file_id); - LOG(DEBUG) << "Receive on_partial_upload for file " << file_id << " with " << partial_remote << " and ready size " - << ready_size; + LOG(DEBUG) << "Receive on_partial_upload for file " << file_id << " with " << partial_remote; if (!file_node) { LOG(ERROR) << "Can't find being uploaded file " << file_id; return; @@ -3686,21 +4366,22 @@ void FileManager::on_partial_upload(QueryId query_id, PartialRemoteFileLocation return; } - file_node->set_partial_remote_location(std::move(partial_remote), ready_size); + file_node->set_partial_remote_location(std::move(partial_remote)); try_flush_node(file_node, "on_partial_upload"); } -void FileManager::on_download_ok(QueryId query_id, FullLocalFileLocation local, int64 size, bool is_new) { +void FileManager::on_download_ok(FileDownloadManager::QueryId query_id, FullLocalFileLocation local, int64 size, + bool is_new) { if (is_closed_) { return; } - Query query; + DownloadQuery query; bool was_active; - std::tie(query, was_active) = finish_query(query_id); + std::tie(query, was_active) = finish_download_query(query_id); auto file_id = query.file_id_; LOG(INFO) << "ON DOWNLOAD OK of " << (is_new ? "new" : "checked") << " file " << file_id << " of size " << size; - auto r_new_file_id = register_local(std::move(local), DialogId(), size, false, false, true, file_id); + auto r_new_file_id = register_local(std::move(local), DialogId(), size, false, true, file_id); Status status = Status::OK(); if (r_new_file_id.is_error()) { status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message()); @@ -3711,19 +4392,19 @@ void FileManager::on_download_ok(QueryId query_id, FullLocalFileLocation local, } if (status.is_error()) { LOG(ERROR) << status.message(); - return on_error_impl(get_file_node(file_id), query.type_, was_active, std::move(status)); + return on_download_error_impl(get_file_node(file_id), query.type_, was_active, std::move(status)); } } -void FileManager::on_upload_ok(QueryId query_id, FileType file_type, PartialRemoteFileLocation partial_remote, - int64 size) { +void FileManager::on_upload_ok(FileUploadManager::QueryId query_id, FileType file_type, + PartialRemoteFileLocation partial_remote) { if (is_closed_) { return; } CHECK(partial_remote.ready_part_count_ == partial_remote.part_count_); - auto some_file_id = finish_query(query_id).first.file_id_; - LOG(INFO) << "ON UPLOAD OK file " << some_file_id << " of size " << size; + auto some_file_id = finish_upload_query(query_id).first.file_id_; + LOG(INFO) << "ON UPLOAD OK file " << some_file_id; auto file_node = get_file_node(some_file_id); if (!file_node) { @@ -3794,37 +4475,37 @@ void FileManager::on_upload_ok(QueryId query_id, FileType file_type, PartialRemo } // for upload by hash -void FileManager::on_upload_full_ok(QueryId query_id, FullRemoteFileLocation remote) { +void FileManager::on_upload_full_ok(FileUploadManager::QueryId query_id, FullRemoteFileLocation remote) { if (is_closed_) { return; } - auto file_id = finish_query(query_id).first.file_id_; + auto file_id = finish_upload_query(query_id).first.file_id_; LOG(INFO) << "ON UPLOAD FULL OK for file " << file_id; auto new_file_id = register_remote(std::move(remote), FileLocationSource::FromServer, DialogId(), 0, 0, ""); LOG_STATUS(merge(new_file_id, file_id)); } -void FileManager::on_partial_generate(QueryId query_id, PartialLocalFileLocation partial_local, int64 expected_size) { +void FileManager::on_partial_generate(FileGenerateManager::QueryId query_id, PartialLocalFileLocation partial_local, + int64 expected_size) { if (is_closed_) { return; } - auto query = queries_container_.get(query_id); + auto query = generate_queries_.get(query_id); CHECK(query != nullptr); auto file_id = query->file_id_; auto file_node = get_file_node(file_id); - auto bitmask = Bitmask(Bitmask::Decode{}, partial_local.ready_bitmask_); - LOG(DEBUG) << "Receive on_partial_generate for file " << file_id << ": " << partial_local.path_ << " " << bitmask; + LOG(DEBUG) << "Receive on_partial_generate for file " << file_id << ": " << partial_local.path_ << " " + << partial_local.ready_size_; if (!file_node) { return; } if (file_node->generate_id_ != query_id) { return; } - auto ready_size = bitmask.get_total_size(partial_local.part_size_, file_node->size_); - file_node->set_local_location(LocalFileLocation(partial_local), ready_size, -1, -1 /* TODO */); + file_node->set_local_location(LocalFileLocation(partial_local), -1, -1 /* TODO */); // TODO check for size and local_size, abort generation if needed if (expected_size > 0) { file_node->set_expected_size(expected_size); @@ -3834,21 +4515,21 @@ void FileManager::on_partial_generate(QueryId query_id, PartialLocalFileLocation run_upload(file_node, {}); } if (file_node->upload_id_ != 0) { - send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_, + send_closure(file_upload_manager_, &FileUploadManager::update_local_file_location, file_node->upload_id_, LocalFileLocation(std::move(partial_local))); } try_flush_node(file_node, "on_partial_generate"); } -void FileManager::on_generate_ok(QueryId query_id, FullLocalFileLocation local) { +void FileManager::on_generate_ok(FileGenerateManager::QueryId query_id, FullLocalFileLocation local) { if (is_closed_) { return; } - Query query; + GenerateQuery query; bool was_active; - std::tie(query, was_active) = finish_query(query_id); + std::tie(query, was_active) = finish_generate_query(query_id); auto generate_file_id = query.file_id_; LOG(INFO) << "Receive on_generate_ok for file " << generate_file_id << ": " << local; @@ -3859,18 +4540,19 @@ void FileManager::on_generate_ok(QueryId query_id, FullLocalFileLocation local) auto old_upload_id = file_node->upload_id_; - auto r_new_file_id = register_local(local, DialogId(), 0, false, false, false, generate_file_id); + auto r_new_file_id = register_local(local, DialogId(), 0, false, false, generate_file_id); file_node = get_file_node(generate_file_id); if (r_new_file_id.is_error()) { - return on_error_impl( - file_node, query.type_, was_active, + return on_generate_error_impl( + file_node, was_active, Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error())); } CHECK(file_node); FileView file_view(file_node); if (context_->need_notify_on_new_files()) { - if (!file_view.has_generate_location() || !begins_with(file_view.generate_location().conversion_, "#file_id#")) { + auto generate_location = file_view.get_generate_location(); + if (generate_location == nullptr || !begins_with(generate_location->conversion_, "#file_id#")) { context_->on_new_file(file_view.size(), file_view.get_allocated_local_size(), 1); } } @@ -3879,66 +4561,75 @@ void FileManager::on_generate_ok(QueryId query_id, FullLocalFileLocation local) if (was_active) { if (old_upload_id != 0 && old_upload_id == file_node->upload_id_) { - send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_, + send_closure(file_upload_manager_, &FileUploadManager::update_local_file_location, file_node->upload_id_, LocalFileLocation(std::move(local))); } } } -void FileManager::on_error(QueryId query_id, Status status) { +void FileManager::on_download_error(FileDownloadManager::QueryId query_id, Status status) { if (is_closed_) { return; } - Query query; + DownloadQuery query; bool was_active; - std::tie(query, was_active) = finish_query(query_id); + std::tie(query, was_active) = finish_download_query(query_id); auto node = get_file_node(query.file_id_); if (!node) { LOG(ERROR) << "Can't find file node for " << query.file_id_ << " " << status; return; } + on_download_error_impl(node, query.type_, was_active, std::move(status)); +} - if (query.type_ == Query::Type::UploadByHash && !G()->close_flag()) { - LOG(INFO) << "Upload By Hash failed: " << status << ", restart upload"; - node->get_by_hash_ = false; - run_upload(node, {}); +void FileManager::on_generate_error(FileGenerateManager::QueryId query_id, Status status) { + if (is_closed_) { return; } - on_error_impl(node, query.type_, was_active, std::move(status)); -} -void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_active, Status status) { - SCOPE_EXIT { - try_flush_node(node, "on_error_impl"); - }; - - if (status.message() == "FILE_PART_INVALID") { - bool has_partial_small_location = node->remote_.partial && !node->remote_.partial->is_big_; - FileView file_view(node); - auto expected_size = file_view.expected_size(true); - bool should_be_big_location = is_file_big(file_view.get_type(), expected_size); + GenerateQuery query; + bool was_active; + std::tie(query, was_active) = finish_generate_query(query_id); + auto node = get_file_node(query.file_id_); + if (!node) { + LOG(ERROR) << "Can't find file node for " << query.file_id_ << " " << status; + return; + } + on_generate_error_impl(node, was_active, std::move(status)); +} - node->delete_partial_remote_location(); - if (has_partial_small_location && should_be_big_location) { - run_upload(node, {}); - return; - } +void FileManager::on_upload_error(FileUploadManager::QueryId query_id, Status status) { + if (is_closed_) { + return; + } - LOG(ERROR) << "Failed to upload file " << node->main_file_id_ << ": unexpected " << status - << ", is_small = " << has_partial_small_location << ", should_be_big = " << should_be_big_location - << ", expected size = " << expected_size; + UploadQuery query; + bool was_active; + std::tie(query, was_active) = finish_upload_query(query_id); + auto node = get_file_node(query.file_id_); + if (!node) { + LOG(ERROR) << "Can't find file node for " << query.file_id_ << " " << status; + return; } - if (begins_with(status.message(), "FILE_GENERATE_LOCATION_INVALID")) { - node->set_generate_location(nullptr); + if (query.type_ == UploadQuery::Type::UploadByHash && !G()->close_flag()) { + LOG(INFO) << "Upload By Hash failed: " << status << ", restart upload"; + node->get_by_hash_ = false; + return run_upload(node, {}); } + on_upload_error_impl(node, query.type_, was_active, std::move(status)); +} + +void FileManager::on_download_error_impl(FileNodePtr node, DownloadQuery::Type type, bool was_active, Status status) { + SCOPE_EXIT { + try_flush_node(node, "on_error_impl"); + }; if ((status.message() == "FILE_ID_INVALID" || status.message() == "LOCATION_INVALID") && FileView(node).may_reload_photo()) { node->need_reload_photo_ = true; - run_download(node, true); - return; + return run_download(node, true); } if (FileReferenceManager::is_file_reference_error(status)) { @@ -3958,39 +4649,26 @@ void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_act } CHECK(!node->file_ids_.empty()); delete_file_reference(node->file_ids_.back(), file_reference); - run_download(node, true); - return; - } - - if (begins_with(status.message(), "FILE_UPLOAD_RESTART")) { - if (ends_with(status.message(), "WITH_FILE_REFERENCE")) { - node->upload_was_update_file_reference_ = true; - } - run_upload(node, {}); - return; + return run_download(node, true); } if (begins_with(status.message(), "FILE_DOWNLOAD_RESTART")) { if (ends_with(status.message(), "WITH_FILE_REFERENCE")) { node->download_was_update_file_reference_ = true; - run_download(node, true); - return; + return run_download(node, true); } else if (ends_with(status.message(), "INCREASE_PART_SIZE")) { if (try_fix_partial_local_location(node)) { - run_download(node, true); - return; + return run_download(node, true); } } else { node->can_search_locally_ = false; - run_download(node, true); - return; + return run_download(node, true); } } if (status.message() == "MTPROTO_CLUSTER_INVALID") { send_closure(G()->config_manager(), &ConfigManager::request_config, true); - run_download(node, true); - return; + return run_download(node, true); } if (!was_active) { @@ -4002,33 +4680,113 @@ void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_act status = Global::request_aborted_error(); } else { if (status.code() != -1) { - if (type == Query::Type::Generate && node->generate_ != nullptr) { - LOG(WARNING) << "Failed to generate file " << node->main_file_id_ << " with " << *node->generate_ << ": " - << status; - } else { - LOG(WARNING) << "Failed to " << type << " file " << node->main_file_id_ << " of type " - << FileView(node).get_type() << ": " << status; + LOG(WARNING) << "Failed to " << type << " file " << node->main_file_id_ << " of type " + << FileView(node).get_type() << ": " << status; + } + if (status.code() == 0 && node->local_.type() == LocalFileLocation::Type::Partial && + !begins_with(status.message(), "FILE_DOWNLOAD_ID_INVALID") && + !begins_with(status.message(), "FILE_DOWNLOAD_LIMIT")) { + // Remove partial location + CSlice path = node->local_.partial().path_; + if (begins_with(path, get_files_temp_dir(FileType::SecureDecrypted)) || + begins_with(path, get_files_temp_dir(FileType::Video))) { + LOG(INFO) << "Unlink file " << path; + send_closure(file_load_manager_, &FileLoadManager::unlink_file, std::move(node->local_.partial().path_), + Promise<Unit>()); + node->drop_local_location(); } } + status = Status::Error(400, status.message()); + } + + on_file_load_error(node, std::move(status)); +} + +void FileManager::on_generate_error_impl(FileNodePtr node, bool was_active, Status status) { + SCOPE_EXIT { + try_flush_node(node, "on_generate_error_impl"); + }; + if (begins_with(status.message(), "FILE_GENERATE_LOCATION_INVALID")) { + node->set_generate_location(nullptr); + } + if (!was_active) { + return; + } + + if (G()->close_flag() && (status.code() < 400 || (status.code() == Global::request_aborted_error().code() && + status.message() == Global::request_aborted_error().message()))) { + status = Global::request_aborted_error(); + } else { + if (status.code() != -1 && node->generate_ != nullptr) { + LOG(WARNING) << "Failed to generate file " << node->main_file_id_ << " with " << *node->generate_ << ": " + << status; + } if (status.code() == 0) { // Remove partial locations - if (node->local_.type() == LocalFileLocation::Type::Partial && - !begins_with(status.message(), "FILE_DOWNLOAD_ID_INVALID") && - !begins_with(status.message(), "FILE_DOWNLOAD_LIMIT")) { - CSlice path = node->local_.partial().path_; - if (begins_with(path, get_files_temp_dir(FileType::SecureDecrypted)) || - begins_with(path, get_files_temp_dir(FileType::Video))) { - LOG(INFO) << "Unlink file " << path; - send_closure(file_load_manager_, &FileLoadManager::unlink_file, std::move(node->local_.partial().path_), - Promise<Unit>()); - node->drop_local_location(); - } + if (node->local_.type() == LocalFileLocation::Type::Partial) { + // the file itself has already been deleted + node->drop_local_location(); } node->delete_partial_remote_location(); } status = Status::Error(400, status.message()); } + on_file_load_error(node, std::move(status)); +} + +void FileManager::on_upload_error_impl(FileNodePtr node, UploadQuery::Type type, bool was_active, Status status) { + SCOPE_EXIT { + try_flush_node(node, "on_upload_error_impl"); + }; + + if (status.message() == "FILE_PART_INVALID") { + bool has_partial_small_location = node->remote_.partial && !node->remote_.partial->is_big_; + FileView file_view(node); + auto expected_size = file_view.expected_size(true); + bool should_be_big_location = is_file_big(file_view.get_type(), expected_size); + + node->delete_partial_remote_location(); + if (has_partial_small_location && should_be_big_location) { + return run_upload(node, {}); + } + + LOG(ERROR) << "Failed to upload file " << node->main_file_id_ << ": unexpected " << status + << ", is_small = " << has_partial_small_location << ", should_be_big = " << should_be_big_location + << ", expected size = " << expected_size; + } + + if (begins_with(status.message(), "FILE_UPLOAD_RESTART")) { + if (ends_with(status.message(), "WITH_FILE_REFERENCE")) { + node->upload_was_update_file_reference_ = true; + } else { + node->delete_partial_remote_location(); + } + return run_upload(node, {}); + } + + if (!was_active) { + return; + } + + if (G()->close_flag() && (status.code() < 400 || (status.code() == Global::request_aborted_error().code() && + status.message() == Global::request_aborted_error().message()))) { + status = Global::request_aborted_error(); + } else { + if (status.code() != -1) { + LOG(WARNING) << "Failed to " << type << " file " << node->main_file_id_ << " of type " + << FileView(node).get_type() << ": " << status; + } + if (status.code() == 0) { + node->delete_partial_remote_location(); + } + status = Status::Error(400, status.message()); + } + + on_file_load_error(node, std::move(status)); +} + +void FileManager::on_file_load_error(FileNodePtr node, Status status) { // Stop everything on error do_cancel_generate(node); do_cancel_download(node); @@ -4053,39 +4811,53 @@ void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_act } } -std::pair<FileManager::Query, bool> FileManager::finish_query(QueryId query_id) { - SCOPE_EXIT { - queries_container_.erase(query_id); - }; - auto query = queries_container_.get(query_id); +std::pair<FileManager::DownloadQuery, bool> FileManager::finish_download_query(FileDownloadManager::QueryId query_id) { + auto query = download_queries_.get(query_id); CHECK(query != nullptr); - auto res = *query; + download_queries_.erase(query_id); + auto node = get_file_node(res.file_id_); - if (!node) { - return std::make_pair(res, false); - } - bool was_active = false; - if (node->generate_id_ == query_id) { - node->generate_id_ = 0; - node->generate_was_update_ = false; - node->set_generate_priority(0, 0); - was_active = true; - } - if (node->download_id_ == query_id) { + if (node && node->download_id_ == query_id) { node->download_id_ = 0; node->download_was_update_file_reference_ = false; node->is_download_started_ = false; node->set_download_priority(0); - was_active = true; + return std::make_pair(res, true); } - if (node->upload_id_ == query_id) { + return std::make_pair(res, false); +} + +std::pair<FileManager::GenerateQuery, bool> FileManager::finish_generate_query(FileGenerateManager::QueryId query_id) { + auto query = generate_queries_.get(query_id); + CHECK(query != nullptr); + auto res = *query; + generate_queries_.erase(query_id); + + auto node = get_file_node(res.file_id_); + if (node && node->generate_id_ == query_id) { + node->generate_id_ = 0; + node->generate_was_update_ = false; + node->set_generate_priority(0, 0); + return std::make_pair(res, true); + } + return std::make_pair(res, false); +} + +std::pair<FileManager::UploadQuery, bool> FileManager::finish_upload_query(FileUploadManager::QueryId query_id) { + auto query = upload_queries_.get(query_id); + CHECK(query != nullptr); + auto res = *query; + upload_queries_.erase(query_id); + + auto node = get_file_node(res.file_id_); + if (node && node->upload_id_ == query_id) { node->upload_id_ = 0; node->upload_was_update_file_reference_ = false; node->set_upload_priority(0); - was_active = true; + return std::make_pair(res, true); } - return std::make_pair(res, was_active); + return std::make_pair(res, false); } FullRemoteFileLocation *FileManager::get_remote(int32 key) { @@ -4095,6 +4867,48 @@ FullRemoteFileLocation *FileManager::get_remote(int32 key) { return &remote_location_info_.get(key).remote_; } +class FileManager::PreliminaryUploadFileCallback final : public UploadCallback { + public: + void on_upload_ok(FileId file_id, tl_object_ptr<telegram_api::InputFile> input_file) final { + // cancel file upload of the file to allow next upload with the same file to succeed + send_closure(G()->file_manager(), &FileManager::cancel_upload, file_id); + } + + void on_upload_encrypted_ok(FileId file_id, tl_object_ptr<telegram_api::InputEncryptedFile> input_file) final { + // cancel file upload of the file to allow next upload with the same file to succeed + send_closure(G()->file_manager(), &FileManager::cancel_upload, file_id); + } + + void on_upload_secure_ok(FileId file_id, tl_object_ptr<telegram_api::InputSecureFile> input_file) final { + // cancel file upload of the file to allow next upload with the same file to succeed + send_closure(G()->file_manager(), &FileManager::cancel_upload, file_id); + } + + void on_upload_error(FileId file_id, Status error) final { + } +}; + +void FileManager::preliminary_upload_file(const td_api::object_ptr<td_api::InputFile> &input_file, FileType file_type, + int32 priority, Promise<td_api::object_ptr<td_api::file>> &&promise) { + if (!(1 <= priority && priority <= 32)) { + return promise.set_error(Status::Error(400, "Upload priority must be between 1 and 32")); + } + + bool is_secret = file_type == FileType::Encrypted || file_type == FileType::EncryptedThumbnail; + bool is_secure = file_type == FileType::SecureEncrypted; + auto r_file_id = + get_input_file_id(file_type, input_file, DialogId(), false, is_secret, !is_secure && !is_secret, is_secure); + if (r_file_id.is_error()) { + return promise.set_error(Status::Error(r_file_id.error().code(), r_file_id.error().message())); + } + auto file_id = r_file_id.ok(); + auto upload_file_id = dup_file_id(file_id, "preliminary_upload_file"); + + upload(upload_file_id, std::make_shared<PreliminaryUploadFileCallback>(), priority, 0); + + promise.set_value(get_file_object(upload_file_id, false)); +} + Result<string> FileManager::get_suggested_file_name(FileId file_id, const string &directory) { if (!file_id.is_valid()) { return Status::Error(400, "Invalid file identifier"); @@ -4110,11 +4924,42 @@ Result<string> FileManager::get_suggested_file_name(FileId file_id, const string void FileManager::hangup() { file_db_.reset(); file_generate_manager_.reset(); - file_load_manager_.reset(); - while (!queries_container_.empty()) { - auto query_ids = queries_container_.ids(); + file_download_manager_.reset(); + file_upload_manager_.reset(); + while (!download_queries_.empty()) { + auto query_ids = download_queries_.ids(); for (auto query_id : query_ids) { - on_error(query_id, Global::request_aborted_error()); + DownloadQuery query; + bool was_active; + std::tie(query, was_active) = finish_download_query(static_cast<FileDownloadManager::QueryId>(query_id)); + auto node = get_file_node(query.file_id_); + if (node) { + on_download_error_impl(node, query.type_, was_active, Global::request_aborted_error()); + } + } + } + while (!generate_queries_.empty()) { + auto query_ids = generate_queries_.ids(); + for (auto query_id : query_ids) { + GenerateQuery query; + bool was_active; + std::tie(query, was_active) = finish_generate_query(static_cast<FileGenerateManager::QueryId>(query_id)); + auto node = get_file_node(query.file_id_); + if (node) { + on_generate_error_impl(node, was_active, Global::request_aborted_error()); + } + } + } + while (!upload_queries_.empty()) { + auto query_ids = upload_queries_.ids(); + for (auto query_id : query_ids) { + UploadQuery query; + bool was_active; + std::tie(query, was_active) = finish_upload_query(static_cast<FileUploadManager::QueryId>(query_id)); + auto node = get_file_node(query.file_id_); + if (node) { + on_upload_error_impl(node, query.type_, was_active, Global::request_aborted_error()); + } } } is_closed_ = true; |