diff options
Diffstat (limited to 'protocols/Telegram/tdlib/td/benchmark')
21 files changed, 2743 insertions, 767 deletions
diff --git a/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt b/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt index 90f294fbc9..53fbdede52 100644 --- a/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt +++ b/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt @@ -1,4 +1,6 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +if ((CMAKE_MAJOR_VERSION LESS 3) OR (CMAKE_VERSION VERSION_LESS "3.0.2")) + message(FATAL_ERROR "CMake >= 3.0.2 is required") +endif() if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -7,7 +9,14 @@ endif() #TODO: all benchmarks in one file add_executable(bench_crypto bench_crypto.cpp) -target_link_libraries(bench_crypto PRIVATE tdcore tdutils ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) +target_link_libraries(bench_crypto PRIVATE tdutils ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) +if (WIN32) + if (MINGW) + target_link_libraries(bench_crypto PRIVATE ws2_32 mswsock crypt32) + else() + target_link_libraries(bench_crypto PRIVATE ws2_32 Mswsock Crypt32) + endif() +endif() target_include_directories(bench_crypto SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) add_executable(bench_actor bench_actor.cpp) @@ -40,6 +49,12 @@ target_link_libraries(bench_tddb PRIVATE tdcore tddb tdutils) add_executable(bench_misc bench_misc.cpp) target_link_libraries(bench_misc PRIVATE tdcore tdutils) +add_executable(check_proxy check_proxy.cpp) +target_link_libraries(check_proxy PRIVATE tdclient tdutils) + +add_executable(check_tls check_tls.cpp) +target_link_libraries(check_tls PRIVATE tdutils) + add_executable(rmdir rmdir.cpp) target_link_libraries(rmdir PRIVATE tdutils) @@ -57,3 +72,22 @@ if (NOT WIN32 AND NOT CYGWIN) add_executable(bench_queue bench_queue.cpp) target_link_libraries(bench_queue PRIVATE tdutils) endif() + +if (TD_TEST_FOLLY AND TD_WITH_ABSEIL) + find_package(ABSL QUIET) + find_package(folly QUIET) + find_package(gflags QUIET) + + if (ABSL_FOUND AND folly_FOUND) + add_executable(memory-hashset-memprof EXCLUDE_FROM_ALL hashset_memory.cpp) + target_compile_definitions(memory-hashset-memprof PRIVATE USE_MEMPROF=1) + target_link_libraries(memory-hashset-memprof PRIVATE tdutils memprof_stat Folly::folly absl::flat_hash_map absl::hash) + + add_executable(memory-hashset-os hashset_memory.cpp) + target_compile_definitions(memory-hashset-os PRIVATE USE_MEMPROF=0) + target_link_libraries(memory-hashset-os PRIVATE tdutils Folly::folly absl::flat_hash_map absl::hash) + + add_executable(hashmap-build hashmap_build.cpp) + target_link_libraries(hashmap-build PRIVATE tdutils Folly::folly absl::flat_hash_map absl::hash) + endif() +endif() diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp index a966d601cf..afe94ca3c1 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp @@ -1,47 +1,101 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/utils/benchmark.h" - #include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" #include "td/actor/PromiseFuture.h" +#include "td/utils/benchmark.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" #include "td/utils/logging.h" - -#include <algorithm> +#include "td/utils/Promise.h" +#include "td/utils/SliceBuilder.h" #if TD_MSVC #pragma comment(linker, "/STACK:16777216") #endif +struct TestActor final : public td::Actor { + static td::int32 actor_count_; + + void start_up() final { + actor_count_++; + stop(); + } + + void tear_down() final { + if (--actor_count_ == 0) { + td::Scheduler::instance()->finish(); + } + } +}; + +td::int32 TestActor::actor_count_; + +namespace td { +template <> +class ActorTraits<TestActor> { + public: + static constexpr bool need_context = false; + static constexpr bool need_start_up = true; +}; +} // namespace td + +class CreateActorBench final : public td::Benchmark { + td::ConcurrentScheduler scheduler_{0, 0}; + + void start_up() final { + scheduler_.start(); + } + + void tear_down() final { + scheduler_.finish(); + } + + public: + td::string get_description() const final { + return "CreateActor"; + } + + void run(int n) final { + for (int i = 0; i < n; i++) { + scheduler_.create_actor_unsafe<TestActor>(0, "TestActor").release(); + } + while (scheduler_.run_main(10)) { + // empty + } + } +}; + template <int type> -class RingBench : public td::Benchmark { +class RingBench final : public td::Benchmark { public: struct PassActor; private: - int actor_n_; - int thread_n_; - std::vector<td::ActorId<PassActor>> actor_array_; - td::ConcurrentScheduler *scheduler_; + int actor_n_ = -1; + int thread_n_ = -1; + td::vector<td::ActorId<PassActor>> actor_array_; + td::ConcurrentScheduler *scheduler_ = nullptr; public: - std::string get_description() const override { + td::string get_description() const final { static const char *types[] = {"later", "immediate", "raw", "tail", "lambda"}; static_assert(0 <= type && type < 5, ""); return PSTRING() << "Ring (send_" << types[type] << ") (threads_n = " << thread_n_ << ")"; } - struct PassActor : public td::Actor { - int id; + struct PassActor final : public td::Actor { + int id = -1; td::ActorId<PassActor> next_actor; int start_n = 0; void pass(int n) { - // LOG(INFO) << "pass: " << n; + // LOG(INFO) << "Pass: " << n; if (n == 0) { td::Scheduler::instance()->finish(); } else { @@ -56,24 +110,23 @@ class RingBench : public td::Benchmark { send_closure_later(next_actor, &PassActor::pass, n - 1); } else { // TODO: it is three times faster than send_event - // may be send event could be further optimized? - ::td::Scheduler::instance()->hack(static_cast<td::ActorId<Actor>>(next_actor), - td::Event::raw(static_cast<td::uint32>(n - 1))); + // maybe send event could be further optimized? + next_actor.get_actor_unsafe()->raw_event(td::Event::raw(static_cast<td::uint32>(n - 1)).data); } } else if (type == 4) { - send_lambda(next_actor, [=, ptr = next_actor.get_actor_unsafe()] { ptr->pass(n - 1); }); + send_lambda(next_actor, [n, ptr = next_actor.get_actor_unsafe()] { ptr->pass(n - 1); }); } } } - void raw_event(const td::Event::Raw &raw) override { + void raw_event(const td::Event::Raw &raw) final { pass(static_cast<int>(raw.u32)); } - void start_up() override { + void start_up() final { yield(); } - void wakeup() override { + void wakeup() final { if (start_n != 0) { int n = start_n; start_n = 0; @@ -85,11 +138,10 @@ class RingBench : public td::Benchmark { RingBench(int actor_n, int thread_n) : actor_n_(actor_n), thread_n_(thread_n) { } - void start_up() override { - scheduler_ = new td::ConcurrentScheduler(); - scheduler_->init(thread_n_); + void start_up() final { + scheduler_ = new td::ConcurrentScheduler(thread_n_, 0); - actor_array_ = std::vector<td::ActorId<PassActor>>(actor_n_); + actor_array_ = td::vector<td::ActorId<PassActor>>(actor_n_); for (int i = 0; i < actor_n_; i++) { actor_array_[i] = scheduler_->create_actor_unsafe<PassActor>(thread_n_ ? i % thread_n_ : 0, "PassActor").release(); @@ -101,30 +153,30 @@ class RingBench : public td::Benchmark { scheduler_->start(); } - void run(int n) override { + void run(int n) final { // first actor is on main_thread - actor_array_[0].get_actor_unsafe()->start_n = std::max(n, 100); + actor_array_[0].get_actor_unsafe()->start_n = td::max(n, 100); while (scheduler_->run_main(10)) { // empty } } - void tear_down() override { + void tear_down() final { scheduler_->finish(); delete scheduler_; } }; template <int type> -class QueryBench : public td::Benchmark { +class QueryBench final : public td::Benchmark { public: - std::string get_description() const override { + td::string get_description() const final { static const char *types[] = {"callback", "immediate future", "delayed future", "dummy", "lambda", "lambda_future"}; static_assert(0 <= type && type < 6, ""); return PSTRING() << "QueryBench: " << types[type]; } - class ClientActor : public td::Actor { + class ClientActor final : public td::Actor { public: class Callback { public: @@ -136,7 +188,7 @@ class QueryBench : public td::Benchmark { virtual ~Callback() = default; virtual void on_result(int x) = 0; }; - explicit ClientActor(std::unique_ptr<Callback> callback) : callback_(std::move(callback)) { + explicit ClientActor(td::unique_ptr<Callback> callback) : callback_(std::move(callback)) { } void f(int x) { callback_->on_result(x * x); @@ -152,23 +204,23 @@ class QueryBench : public td::Benchmark { } private: - std::unique_ptr<Callback> callback_; + td::unique_ptr<Callback> callback_; }; - class ServerActor : public td::Actor { + class ServerActor final : public td::Actor { public: - class ClientCallback : public ClientActor::Callback { + class ClientCallback final : public ClientActor::Callback { public: explicit ClientCallback(td::ActorId<ServerActor> server) : server_(server) { } - void on_result(int x) override { + void on_result(int x) final { send_closure(server_, &ServerActor::on_result, x); } private: td::ActorId<ServerActor> server_; }; - void start_up() override { + void start_up() final { client_ = td::create_actor<ClientActor>("Client", td::make_unique<ClientCallback>(actor_id(this))).release(); } @@ -177,7 +229,7 @@ class QueryBench : public td::Benchmark { wakeup(); } - void wakeup() override { + void wakeup() final { while (true) { if (n_ < 0) { td::Scheduler::instance()->finish(); @@ -206,10 +258,12 @@ class QueryBench : public td::Benchmark { } else if (type == 4) { int val = 0; send_lambda(client_, [&] { val = n_ * n_; }); + CHECK(val == n_ * n_); } else if (type == 5) { send_closure(client_, &ClientActor::f_promise, - td::PromiseCreator::lambda( - [id = actor_id(this), n = n_](td::Unit) { send_closure(id, &ServerActor::result, n * n); })); + td::PromiseCreator::lambda([actor_id = actor_id(this), n = n_](td::Unit) { + send_closure(actor_id, &ServerActor::result, n * n); + })); return; } } @@ -220,7 +274,7 @@ class QueryBench : public td::Benchmark { wakeup(); } - void raw_event(const td::Event::Raw &event) override { + void raw_event(const td::Event::Raw &event) final { int val = future_.move_as_ok(); CHECK(val == n_ * n_); wakeup(); @@ -232,22 +286,21 @@ class QueryBench : public td::Benchmark { private: td::ActorId<ClientActor> client_; - int n_; + int n_ = 0; td::FutureActor<int> future_; }; - void start_up() override { - scheduler_ = new td::ConcurrentScheduler(); - scheduler_->init(0); + void start_up() final { + scheduler_ = new td::ConcurrentScheduler(0, 0); server_ = scheduler_->create_actor_unsafe<ServerActor>(0, "Server"); scheduler_->start(); } - void run(int n) override { + void run(int n) final { // first actor is on main_thread { - auto guard = scheduler_->get_current_guard(); + auto guard = scheduler_->get_main_guard(); send_closure(server_, &ServerActor::run, n); } while (scheduler_->run_main(10)) { @@ -255,19 +308,21 @@ class QueryBench : public td::Benchmark { } } - void tear_down() override { + void tear_down() final { server_.release(); scheduler_->finish(); delete scheduler_; } private: - td::ConcurrentScheduler *scheduler_; + td::ConcurrentScheduler *scheduler_ = nullptr; td::ActorOwn<ServerActor> server_; }; int main() { - SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); + td::init_openssl_threads(); + + bench(CreateActorBench()); bench(RingBench<4>(504, 0)); bench(RingBench<3>(504, 0)); bench(RingBench<0>(504, 0)); @@ -286,5 +341,4 @@ int main() { bench(RingBench<0>(504, 2)); bench(RingBench<1>(504, 2)); bench(RingBench<2>(504, 2)); - return 0; } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp index 44d309ef11..40d7602d2e 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp @@ -1,75 +1,374 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include "td/utils/benchmark.h" +#include "td/utils/common.h" #include "td/utils/crypto.h" -#include "td/utils/int_types.h" -#include "td/utils/logging.h" #include "td/utils/port/thread.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" +#include "td/utils/SliceBuilder.h" +#include "td/utils/UInt.h" +#include <openssl/evp.h> #include <openssl/sha.h> +#include <algorithm> #include <array> #include <atomic> #include <cstdint> #include <cstdlib> +#include <iterator> #include <random> #include <string> #include <vector> static constexpr int DATA_SIZE = 8 << 10; +static constexpr int SHORT_DATA_SIZE = 64; -class SHA1Bench : public td::Benchmark { +#if OPENSSL_VERSION_NUMBER <= 0x10100000L +class SHA1Bench final : public td::Benchmark { public: alignas(64) unsigned char data[DATA_SIZE]; - std::string get_description() const override { + std::string get_description() const final { return PSTRING() << "SHA1 OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; } - void start_up() override { - for (int i = 0; i < DATA_SIZE; i++) { - data[i] = 123; - data[i] = 0; - } + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { unsigned char md[20]; SHA1(data, DATA_SIZE, md); } } }; +#endif + +class SHA1ShortBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[SHORT_DATA_SIZE]; + + std::string get_description() const final { + return PSTRING() << "SHA1 [" << SHORT_DATA_SIZE << "B]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + } + + void run(int n) final { + unsigned char md[20]; + for (int i = 0; i < n; i++) { + td::sha1(td::Slice(data, SHORT_DATA_SIZE), md); + } + } +}; + +class SHA256ShortBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[SHORT_DATA_SIZE]; + + std::string get_description() const final { + return PSTRING() << "SHA256 [" << SHORT_DATA_SIZE << "B]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + } + + void run(int n) final { + unsigned char md[32]; + for (int i = 0; i < n; i++) { + td::sha256(td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 32)); + } + } +}; + +class SHA512ShortBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[SHORT_DATA_SIZE]; + + std::string get_description() const final { + return PSTRING() << "SHA512 [" << SHORT_DATA_SIZE << "B]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + } + + void run(int n) final { + unsigned char md[64]; + for (int i = 0; i < n; i++) { + td::sha512(td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 64)); + } + } +}; -class AESBench : public td::Benchmark { +class HmacSha256ShortBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[SHORT_DATA_SIZE]; + + std::string get_description() const final { + return PSTRING() << "HMAC-SHA256 [" << SHORT_DATA_SIZE << "B]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + } + + void run(int n) final { + unsigned char md[32]; + for (int i = 0; i < n; i++) { + td::hmac_sha256(td::Slice(data, SHORT_DATA_SIZE), td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 32)); + } + } +}; + +class HmacSha512ShortBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[SHORT_DATA_SIZE]; + + std::string get_description() const final { + return PSTRING() << "HMAC-SHA512 [" << SHORT_DATA_SIZE << "B]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + } + + void run(int n) final { + unsigned char md[32]; + for (int i = 0; i < n; i++) { + td::hmac_sha256(td::Slice(data, SHORT_DATA_SIZE), td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 32)); + } + } +}; + +class AesEcbBench final : public td::Benchmark { public: alignas(64) unsigned char data[DATA_SIZE]; td::UInt256 key; td::UInt256 iv; - std::string get_description() const override { - return PSTRING() << "AES OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; + std::string get_description() const final { + return PSTRING() << "AES ECB OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + td::Random::secure_bytes(key.raw, sizeof(key)); + td::Random::secure_bytes(iv.raw, sizeof(iv)); } - void start_up() override { - for (int i = 0; i < DATA_SIZE; i++) { - data[i] = 123; + void run(int n) final { + td::AesState state; + state.init(td::as_slice(key), true); + td::MutableSlice data_slice(data, DATA_SIZE); + for (int i = 0; i <= n; i++) { + size_t step = 16; + for (size_t offset = 0; offset + step <= data_slice.size(); offset += step) { + state.encrypt(data_slice.ubegin() + offset, data_slice.ubegin() + offset, static_cast<int>(step)); + } } + } +}; + +class AesIgeEncryptBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[DATA_SIZE]; + td::UInt256 key; + td::UInt256 iv; + + std::string get_description() const final { + return PSTRING() << "AES IGE OpenSSL encrypt [" << (DATA_SIZE >> 10) << "KB]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); td::Random::secure_bytes(key.raw, sizeof(key)); td::Random::secure_bytes(iv.raw, sizeof(iv)); } - void run(int n) override { + void run(int n) final { td::MutableSlice data_slice(data, DATA_SIZE); + td::AesIgeState state; + state.init(as_slice(key), as_slice(iv), true); for (int i = 0; i < n; i++) { - td::aes_ige_encrypt(key, &iv, data_slice, data_slice); + state.encrypt(data_slice, data_slice); + } + } +}; + +class AesIgeDecryptBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[DATA_SIZE]; + td::UInt256 key; + td::UInt256 iv; + + std::string get_description() const final { + return PSTRING() << "AES IGE OpenSSL decrypt [" << (DATA_SIZE >> 10) << "KB]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + td::Random::secure_bytes(key.raw, sizeof(key)); + td::Random::secure_bytes(iv.raw, sizeof(iv)); + } + + void run(int n) final { + td::MutableSlice data_slice(data, DATA_SIZE); + td::AesIgeState state; + state.init(as_slice(key), as_slice(iv), false); + for (int i = 0; i < n; i++) { + state.decrypt(data_slice, data_slice); + } + } +}; + +class AesCtrBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[DATA_SIZE]; + td::UInt256 key; + td::UInt128 iv; + + std::string get_description() const final { + return PSTRING() << "AES CTR OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + td::Random::secure_bytes(key.raw, sizeof(key)); + td::Random::secure_bytes(iv.raw, sizeof(iv)); + } + + void run(int n) final { + td::MutableSlice data_slice(data, DATA_SIZE); + td::AesCtrState state; + state.init(as_slice(key), as_slice(iv)); + for (int i = 0; i < n; i++) { + state.encrypt(data_slice, data_slice); + } + } +}; + +#if OPENSSL_VERSION_NUMBER >= 0x10100000L +class AesCtrOpenSSLBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[DATA_SIZE]; + td::UInt256 key; + td::UInt128 iv; + + std::string get_description() const final { + return PSTRING() << "AES CTR RAW OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + td::Random::secure_bytes(key.raw, sizeof(key)); + td::Random::secure_bytes(iv.raw, sizeof(iv)); + } + + void run(int n) final { + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(ctx, EVP_aes_256_ctr(), nullptr, key.raw, iv.raw); + + td::MutableSlice data_slice(data, DATA_SIZE); + td::AesCtrState state; + state.init(as_slice(key), as_slice(iv)); + for (int i = 0; i < n; i++) { + int len = 0; + EVP_EncryptUpdate(ctx, data_slice.ubegin(), &len, data_slice.ubegin(), DATA_SIZE); + CHECK(len == DATA_SIZE); + } + + EVP_CIPHER_CTX_free(ctx); + } +}; +#endif + +class AesCbcDecryptBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[DATA_SIZE]; + td::UInt256 key; + td::UInt128 iv; + + std::string get_description() const final { + return PSTRING() << "AES CBC Decrypt OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + td::Random::secure_bytes(as_slice(key)); + td::Random::secure_bytes(as_slice(iv)); + } + + void run(int n) final { + td::MutableSlice data_slice(data, DATA_SIZE); + for (int i = 0; i < n; i++) { + td::aes_cbc_decrypt(as_slice(key), as_slice(iv), data_slice, data_slice); + } + } +}; + +class AesCbcEncryptBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[DATA_SIZE]; + td::UInt256 key; + td::UInt128 iv; + + std::string get_description() const final { + return PSTRING() << "AES CBC Encrypt OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + td::Random::secure_bytes(as_slice(key)); + td::Random::secure_bytes(as_slice(iv)); + } + + void run(int n) final { + td::MutableSlice data_slice(data, DATA_SIZE); + for (int i = 0; i < n; i++) { + td::aes_cbc_encrypt(as_slice(key), as_slice(iv), data_slice, data_slice); + } + } +}; + +template <bool use_state> +class AesIgeShortBench final : public td::Benchmark { + public: + alignas(64) unsigned char data[SHORT_DATA_SIZE]; + td::UInt256 key; + td::UInt256 iv; + + std::string get_description() const final { + return PSTRING() << "AES IGE OpenSSL " << (use_state ? "EVP" : "C ") << "[" << SHORT_DATA_SIZE << "B]"; + } + + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); + td::Random::secure_bytes(as_slice(key)); + td::Random::secure_bytes(as_slice(iv)); + } + + void run(int n) final { + td::MutableSlice data_slice(data, SHORT_DATA_SIZE); + for (int i = 0; i < n; i++) { + if (use_state) { + td::AesIgeState ige; + ige.init(as_slice(key), as_slice(iv), false); + ige.decrypt(data_slice, data_slice); + } else { + td::aes_ige_decrypt(as_slice(key), as_slice(iv), data_slice, data_slice); + } } } }; @@ -110,15 +409,15 @@ BENCH(TdRandFast, "td_rand_fast") { #if !TD_THREAD_UNSUPPORTED BENCH(SslRand, "ssl_rand_int32") { std::vector<td::thread> v; - std::atomic<td::uint32> sum; + std::atomic<td::uint32> sum{0}; for (int i = 0; i < 3; i++) { - v.push_back(td::thread([&] { + v.emplace_back([&sum, n] { td::int32 res = 0; for (int j = 0; j < n; j++) { res ^= td::Random::secure_int32(); } sum += res; - })); + }); } for (auto &x : v) { x.join(); @@ -147,22 +446,19 @@ BENCH(Pbkdf2, "pbkdf2") { td::pbkdf2_sha256(password, salt, n, key); } -class Crc32Bench : public td::Benchmark { +class Crc32Bench final : public td::Benchmark { public: alignas(64) unsigned char data[DATA_SIZE]; - std::string get_description() const override { + std::string get_description() const final { return PSTRING() << "Crc32 zlib [" << (DATA_SIZE >> 10) << "KB]"; } - void start_up() override { - for (int i = 0; i < DATA_SIZE; i++) { - data[i] = 123; - data[i] = 0; - } + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); } - void run(int n) override { + void run(int n) final { td::uint64 res = 0; for (int i = 0; i < n; i++) { res += td::crc32(td::Slice(data, DATA_SIZE)); @@ -171,22 +467,19 @@ class Crc32Bench : public td::Benchmark { } }; -class Crc64Bench : public td::Benchmark { +class Crc64Bench final : public td::Benchmark { public: alignas(64) unsigned char data[DATA_SIZE]; - std::string get_description() const override { + std::string get_description() const final { return PSTRING() << "Crc64 Anton [" << (DATA_SIZE >> 10) << "KB]"; } - void start_up() override { - for (int i = 0; i < DATA_SIZE; i++) { - data[i] = 123; - data[i] = 0; - } + void start_up() final { + std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); } - void run(int n) override { + void run(int n) final { td::uint64 res = 0; for (int i = 0; i < n; i++) { res += td::crc64(td::Slice(data, DATA_SIZE)); @@ -196,6 +489,20 @@ class Crc64Bench : public td::Benchmark { }; int main() { + td::init_openssl_threads(); + td::bench(AesCtrBench()); +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + td::bench(AesCtrOpenSSLBench()); +#endif + + td::bench(AesCbcDecryptBench()); + td::bench(AesCbcEncryptBench()); + td::bench(AesIgeShortBench<true>()); + td::bench(AesIgeShortBench<false>()); + td::bench(AesIgeEncryptBench()); + td::bench(AesIgeDecryptBench()); + td::bench(AesEcbBench()); + td::bench(Pbkdf2Bench()); td::bench(RandBench()); td::bench(CppRandBench()); @@ -205,9 +512,14 @@ int main() { td::bench(SslRandBench()); #endif td::bench(SslRandBufBench()); +#if OPENSSL_VERSION_NUMBER <= 0x10100000L td::bench(SHA1Bench()); - td::bench(AESBench()); +#endif + td::bench(SHA1ShortBench()); + td::bench(SHA256ShortBench()); + td::bench(SHA512ShortBench()); + td::bench(HmacSha256ShortBench()); + td::bench(HmacSha512ShortBench()); td::bench(Crc32Bench()); td::bench(Crc64Bench()); - return 0; } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp index dc768e9d9d..822a915938 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp @@ -1,28 +1,35 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/actor/actor.h" - #include "td/db/binlog/Binlog.h" +#include "td/db/binlog/ConcurrentBinlog.h" #include "td/db/BinlogKeyValue.h" +#include "td/db/DbKey.h" #include "td/db/SeqKeyValue.h" +#include "td/db/SqliteConnectionSafe.h" #include "td/db/SqliteDb.h" #include "td/db/SqliteKeyValueAsync.h" +#include "td/db/SqliteKeyValueSafe.h" + +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" #include "td/utils/benchmark.h" +#include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" +#include "td/utils/SliceBuilder.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" #include <memory> template <class KeyValueT> -class TdKvBench : public td::Benchmark { - td::ConcurrentScheduler sched; +class TdKvBench final : public td::Benchmark { + td::ConcurrentScheduler sched{1, 0}; td::string name_; public: @@ -30,27 +37,27 @@ class TdKvBench : public td::Benchmark { name_ = std::move(name); } - td::string get_description() const override { + td::string get_description() const final { return name_; } - class Main : public td::Actor { + class Main final : public td::Actor { public: explicit Main(int n) : n_(n) { } private: - void loop() override { + void loop() final { KeyValueT::destroy("test_tddb").ignore(); - class Worker : public Actor { + class Worker final : public Actor { public: Worker(int n, td::string db_name) : n_(n) { kv_.init(db_name).ensure(); } private: - void loop() override { + void loop() final { for (int i = 0; i < n_; i++) { kv_.set(td::to_string(i % 10), td::to_string(i)); } @@ -64,12 +71,11 @@ class TdKvBench : public td::Benchmark { int n_; }; - void start_up_n(int n) override { - sched.init(1); + void start_up_n(int n) final { sched.create_actor_unsafe<Main>(1, "Main", n).release(); } - void run(int n) override { + void run(int n) final { sched.start(); while (sched.run_main(10)) { // empty @@ -77,24 +83,23 @@ class TdKvBench : public td::Benchmark { sched.finish(); } - void tear_down() override { + void tear_down() final { } }; template <bool is_encrypted = false> -class SqliteKVBench : public td::Benchmark { +class SqliteKVBench final : public td::Benchmark { td::SqliteDb db; - td::string get_description() const override { + td::string get_description() const final { return PSTRING() << "SqliteKV " << td::tag("is_encrypted", is_encrypted); } - void start_up() override { + void start_up() final { td::string path = "testdb.sqlite"; td::SqliteDb::destroy(path).ignore(); if (is_encrypted) { - td::SqliteDb::change_key(path, td::DbKey::password("cucumber"), td::DbKey::empty()); - db = td::SqliteDb::open_with_key(path, td::DbKey::password("cucumber")).move_as_ok(); + db = td::SqliteDb::change_key(path, true, td::DbKey::password("cucumber"), td::DbKey::empty()).move_as_ok(); } else { - db = td::SqliteDb::open_with_key(path, td::DbKey::empty()).move_as_ok(); + db = td::SqliteDb::open_with_key(path, true, td::DbKey::empty()).move_as_ok(); } db.exec("PRAGMA encoding=\"UTF-8\"").ensure(); db.exec("PRAGMA synchronous=NORMAL").ensure(); @@ -103,7 +108,7 @@ class SqliteKVBench : public td::Benchmark { db.exec("DROP TABLE IF EXISTS KV").ensure(); db.exec("CREATE TABLE IF NOT EXISTS KV (k BLOB PRIMARY KEY, v BLOB)").ensure(); } - void run(int n) override { + void run(int n) final { auto stmt = db.get_statement("REPLACE INTO KV (k, v) VALUES(?1, ?2)").move_as_ok(); db.exec("BEGIN TRANSACTION").ensure(); for (int i = 0; i < n; i++) { @@ -135,17 +140,17 @@ static td::Status init_db(td::SqliteDb &db) { return td::Status::OK(); } -class SqliteKeyValueAsyncBench : public td::Benchmark { +class SqliteKeyValueAsyncBench final : public td::Benchmark { public: - td::string get_description() const override { + td::string get_description() const final { return "SqliteKeyValueAsync"; } - void start_up() override { + void start_up() final { do_start_up().ensure(); scheduler_->start(); } - void run(int n) override { - auto guard = scheduler_->get_current_guard(); + void run(int n) final { + auto guard = scheduler_->get_main_guard(); for (int i = 0; i < n; i++) { auto key = td::to_string(i % 10); @@ -153,10 +158,10 @@ class SqliteKeyValueAsyncBench : public td::Benchmark { sqlite_kv_async_->set(key, value, td::Auto()); } } - void tear_down() override { + void tear_down() final { scheduler_->run_main(0.1); { - auto guard = scheduler_->get_current_guard(); + auto guard = scheduler_->get_main_guard(); sqlite_kv_async_.reset(); sqlite_kv_safe_.reset(); sql_connection_->close_and_destroy(); @@ -167,21 +172,20 @@ class SqliteKeyValueAsyncBench : public td::Benchmark { } private: - std::unique_ptr<td::ConcurrentScheduler> scheduler_; + td::unique_ptr<td::ConcurrentScheduler> scheduler_; std::shared_ptr<td::SqliteConnectionSafe> sql_connection_; std::shared_ptr<td::SqliteKeyValueSafe> sqlite_kv_safe_; - std::unique_ptr<td::SqliteKeyValueAsyncInterface> sqlite_kv_async_; + td::unique_ptr<td::SqliteKeyValueAsyncInterface> sqlite_kv_async_; td::Status do_start_up() { - scheduler_ = std::make_unique<td::ConcurrentScheduler>(); - scheduler_->init(1); + scheduler_ = td::make_unique<td::ConcurrentScheduler>(1, 0); - auto guard = scheduler_->get_current_guard(); + auto guard = scheduler_->get_main_guard(); td::string sql_db_name = "testdb.sqlite"; td::SqliteDb::destroy(sql_db_name).ignore(); - sql_connection_ = std::make_shared<td::SqliteConnectionSafe>(sql_db_name); + sql_connection_ = std::make_shared<td::SqliteConnectionSafe>(sql_db_name, td::DbKey::empty()); auto &db = sql_connection_->get(); TRY_STATUS(init_db(db)); @@ -192,31 +196,31 @@ class SqliteKeyValueAsyncBench : public td::Benchmark { } }; -class SeqKvBench : public td::Benchmark { - td::string get_description() const override { +class SeqKvBench final : public td::Benchmark { + td::string get_description() const final { return "SeqKvBench"; } td::SeqKeyValue kv; - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { - kv.set(td::to_string(i % 10), td::to_string(i)); + kv.set(PSLICE() << i % 10, PSLICE() << i); } } }; template <bool is_encrypted = false> -class BinlogKeyValueBench : public td::Benchmark { - td::string get_description() const override { +class BinlogKeyValueBench final : public td::Benchmark { + td::string get_description() const final { return PSTRING() << "BinlogKeyValue " << td::tag("is_encrypted", is_encrypted); } td::BinlogKeyValue<td::Binlog> kv; - void start_up() override { + void start_up() final { td::SqliteDb::destroy("test_binlog").ignore(); kv.init("test_binlog", is_encrypted ? td::DbKey::password("cucumber") : td::DbKey::empty()).ensure(); } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { kv.set(td::to_string(i % 10), td::to_string(i)); } @@ -233,5 +237,4 @@ int main() { bench(TdKvBench<td::BinlogKeyValue<td::Binlog>>("BinlogKeyValue<Binlog>")); bench(TdKvBench<td::BinlogKeyValue<td::ConcurrentBinlog>>("BinlogKeyValue<ConcurrentBinlog>")); bench(SeqKvBench()); - return 0; } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp index f6718152b4..ae2068c050 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp @@ -1,9 +1,8 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // int main() { - return 0; } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp index 08d04f009c..f518fe03b0 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp @@ -1,15 +1,15 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/utils/benchmark.h" // for bench, do_not_optimize_away, etc - -#include "td/mtproto/crypto.h" +#include "td/mtproto/DhCallback.h" +#include "td/mtproto/DhHandshake.h" #include "td/utils/base64.h" -#include "td/utils/logging.h" +#include "td/utils/benchmark.h" +#include "td/utils/common.h" #include "td/utils/Slice.h" #include <map> @@ -18,59 +18,55 @@ #include <semaphore.h> #endif -namespace td { - -static int32 g = 3; -static string prime_base64 = +static td::int32 g = 3; +static td::string prime_base64 = "xxyuucaxyQSObFIvcPE_c5gNQCOOPiHBSTTQN1Y9kw9IGYoKp8FAWCKUk9IlMPTb-jNvbgrJJROVQ67UTM58NyD9UfaUWHBaxozU_mtrE6vcl0ZRKW" "kyhFTxj6-MWV9kJHf-lrsqlB1bzR1KyMxJiAcI-ps3jjxPOpBgvuZ8-aSkppWBEFGQfhYnU7VrD2tBDbp02KhLKhSzFE4O8ShHVP0X7ZUNWWW0ud1G" "WC2xF40WnGvEZbDW_5yjko_vW5rk5Bj8Feg-vqD4f6n_Xu1wBQ3tKEn0e_lZ2VaFDOkphR8NgRX2NbEF7i5OFdBLJFS_b0-t8DSxBAMRnNjjuS_MW" "w"; -class HandshakeBench : public Benchmark { - std::string get_description() const override { +class HandshakeBench final : public td::Benchmark { + td::string get_description() const final { return "Handshake"; } - class FakeDhCallback : public DhCallback { + class FakeDhCallback final : public td::mtproto::DhCallback { public: - int is_good_prime(Slice prime_str) const override { + int is_good_prime(td::Slice prime_str) const final { auto it = cache.find(prime_str.str()); if (it == cache.end()) { return -1; } return it->second; } - void add_good_prime(Slice prime_str) const override { + void add_good_prime(td::Slice prime_str) const final { cache[prime_str.str()] = 1; } - void add_bad_prime(Slice prime_str) const override { + void add_bad_prime(td::Slice prime_str) const final { cache[prime_str.str()] = 0; } - mutable std::map<string, int> cache; + mutable std::map<td::string, int> cache; } dh_callback; - void run(int n) override { - DhHandshake a; - DhHandshake b; - auto prime = base64url_decode(prime_base64).move_as_ok(); + void run(int n) final { + td::mtproto::DhHandshake a; + td::mtproto::DhHandshake b; + auto prime = td::base64url_decode(prime_base64).move_as_ok(); + td::mtproto::DhHandshake::check_config(g, prime, &dh_callback).ensure(); for (int i = 0; i < n; i += 2) { a.set_config(g, prime); b.set_config(g, prime); b.set_g_a(a.get_g_b()); a.set_g_a(b.get_g_b()); - a.run_checks(&dh_callback).ensure(); - b.run_checks(&dh_callback).ensure(); + a.run_checks(true, &dh_callback).ensure(); + b.run_checks(true, &dh_callback).ensure(); auto a_key = a.gen_key(); auto b_key = b.gen_key(); CHECK(a_key.first == b_key.first); } } }; -} // namespace td int main() { - SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); - td::bench(td::HandshakeBench()); - return 0; + td::bench(HandshakeBench()); } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp index 6958a5b313..70fdb909c2 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp @@ -1,15 +1,18 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/actor/actor.h" - #include "td/net/HttpOutboundConnection.h" #include "td/net/HttpQuery.h" +#include "td/net/SslStream.h" + +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" #include "td/utils/buffer.h" +#include "td/utils/BufferedFd.h" #include "td/utils/logging.h" #include "td/utils/port/IPAddress.h" #include "td/utils/port/SocketFd.h" @@ -18,50 +21,52 @@ #include <atomic> #include <limits> -namespace td { - std::atomic<int> counter; -class HttpClient : public HttpOutboundConnection::Callback { - void start_up() override { - IPAddress addr; + +class HttpClient final : public td::HttpOutboundConnection::Callback { + void start_up() final { + td::IPAddress addr; addr.init_ipv4_port("127.0.0.1", 8082).ensure(); - auto fd = SocketFd::open(addr); - CHECK(fd.is_ok()) << fd.error(); - connection_ = - create_actor<HttpOutboundConnection>("Connect", fd.move_as_ok(), std::numeric_limits<size_t>::max(), 0, 0, - ActorOwn<HttpOutboundConnection::Callback>(actor_id(this))); + auto fd = td::SocketFd::open(addr); + LOG_CHECK(fd.is_ok()) << fd.error(); + connection_ = td::create_actor<td::HttpOutboundConnection>( + "Connect", td::BufferedFd<td::SocketFd>(fd.move_as_ok()), td::SslStream{}, std::numeric_limits<size_t>::max(), + 0, 0, td::ActorOwn<td::HttpOutboundConnection::Callback>(actor_id(this))); yield(); cnt_ = 100000; counter++; } - void tear_down() override { + + void tear_down() final { if (--counter == 0) { - Scheduler::instance()->finish(); + td::Scheduler::instance()->finish(); } } - void loop() override { + + void loop() final { if (cnt_-- < 0) { return stop(); } - send_closure(connection_, &HttpOutboundConnection::write_next, BufferSlice("GET / HTTP/1.1\r\n\r\n")); - send_closure(connection_, &HttpOutboundConnection::write_ok); + send_closure(connection_, &td::HttpOutboundConnection::write_next, td::BufferSlice("GET / HTTP/1.1\r\n\r\n")); + send_closure(connection_, &td::HttpOutboundConnection::write_ok); LOG(INFO) << "SEND"; } - void handle(HttpQueryPtr result) override { + + void handle(td::unique_ptr<td::HttpQuery> result) final { loop(); } - void on_connection_error(Status error) override { + + void on_connection_error(td::Status error) final { LOG(ERROR) << "ERROR: " << error; } - ActorOwn<HttpOutboundConnection> connection_; - int cnt_; + td::ActorOwn<td::HttpOutboundConnection> connection_; + int cnt_ = 0; }; int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); - auto scheduler = make_unique<ConcurrentScheduler>(); - scheduler->init(0); + auto scheduler = td::make_unique<td::ConcurrentScheduler>(0, 0); scheduler->create_actor_unsafe<HttpClient>(0, "Client1").release(); scheduler->create_actor_unsafe<HttpClient>(0, "Client2").release(); scheduler->start(); @@ -69,10 +74,4 @@ int main() { // empty } scheduler->finish(); - return 0; -} -} // namespace td - -int main() { - return td::main(); } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp index 2afe2d73ff..403af06bec 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -9,19 +9,20 @@ #include "td/utils/benchmark.h" #include "td/utils/buffer.h" +#include "td/utils/common.h" #include "td/utils/find_boundary.h" #include "td/utils/logging.h" static std::string http_query = "GET / HTTP/1.1\r\nConnection:keep-alive\r\nhost:127.0.0.1:8080\r\n\r\n"; static const size_t block_size = 2500; -class HttpReaderBench : public td::Benchmark { - std::string get_description() const override { +class HttpReaderBench final : public td::Benchmark { + std::string get_description() const final { return "HttpReaderBench"; } - void run(int n) override { - int cnt = static_cast<int>(block_size / http_query.size()); + void run(int n) final { + auto cnt = static_cast<int>(block_size / http_query.size()); td::HttpQuery q; int parsed = 0; int sent = 0; @@ -45,27 +46,26 @@ class HttpReaderBench : public td::Benchmark { td::ChainBufferReader reader_; td::HttpReader http_reader_; - void start_up() override { - writer_ = td::ChainBufferWriter::create_empty(); + void start_up() final { reader_ = writer_.extract_reader(); http_reader_.init(&reader_, 10000, 0); } }; -class BufferBench : public td::Benchmark { - std::string get_description() const override { +class BufferBench final : public td::Benchmark { + std::string get_description() const final { return "BufferBench"; } - void run(int n) override { - int cnt = static_cast<int>(block_size / http_query.size()); + void run(int n) final { + auto cnt = static_cast<int>(block_size / http_query.size()); for (int i = 0; i < n; i += cnt) { for (int j = 0; j < cnt; j++) { writer_.append(http_query); } reader_.sync_with_writer(); for (int j = 0; j < cnt; j++) { - reader_.cut_head(http_query.size()); + auto result = reader_.cut_head(http_query.size()); } } } @@ -73,19 +73,18 @@ class BufferBench : public td::Benchmark { td::ChainBufferReader reader_; td::HttpReader http_reader_; - void start_up() override { - writer_ = td::ChainBufferWriter::create_empty(); + void start_up() final { reader_ = writer_.extract_reader(); } }; -class FindBoundaryBench : public td::Benchmark { - std::string get_description() const override { +class FindBoundaryBench final : public td::Benchmark { + std::string get_description() const final { return "FindBoundaryBench"; } - void run(int n) override { - int cnt = static_cast<int>(block_size / http_query.size()); + void run(int n) final { + auto cnt = static_cast<int>(block_size / http_query.size()); for (int i = 0; i < n; i += cnt) { for (int j = 0; j < cnt; j++) { writer_.append(http_query); @@ -95,7 +94,7 @@ class FindBoundaryBench : public td::Benchmark { size_t len = 0; find_boundary(reader_.clone(), "\r\n\r\n", len); CHECK(size_t(len) + 4 == http_query.size()); - reader_.cut_head(len + 2); + auto result = reader_.cut_head(len + 2); reader_.advance(2); } } @@ -104,8 +103,7 @@ class FindBoundaryBench : public td::Benchmark { td::ChainBufferReader reader_; td::HttpReader http_reader_; - void start_up() override { - writer_ = td::ChainBufferWriter::create_empty(); + void start_up() final { reader_ = writer_.extract_reader(); } }; diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp index c48e8b4a67..33cde9ef8c 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp @@ -1,31 +1,32 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/actor/actor.h" - #include "td/net/HttpHeaderCreator.h" #include "td/net/HttpInboundConnection.h" #include "td/net/HttpQuery.h" #include "td/net/TcpListener.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" + #include "td/utils/buffer.h" +#include "td/utils/BufferedFd.h" #include "td/utils/logging.h" #include "td/utils/port/SocketFd.h" #include "td/utils/Slice.h" -namespace td { - static int cnt = 0; -class HelloWorld : public HttpInboundConnection::Callback { + +class HelloWorld final : public td::HttpInboundConnection::Callback { public: - void handle(HttpQueryPtr query, ActorOwn<HttpInboundConnection> connection) override { + void handle(td::unique_ptr<td::HttpQuery> query, td::ActorOwn<td::HttpInboundConnection> connection) final { // LOG(ERROR) << *query; - HttpHeaderCreator hc; - Slice content = "hello world"; - //auto content = BufferSlice("hello world"); + td::HttpHeaderCreator hc; + td::Slice content = "hello world"; + //auto content = td::BufferSlice("hello world"); hc.init_ok(); hc.set_keep_alive(); hc.set_content_size(content.size()); @@ -35,55 +36,49 @@ class HelloWorld : public HttpInboundConnection::Callback { auto res = hc.finish(content); LOG_IF(FATAL, res.is_error()) << res.error(); - send_closure(connection, &HttpInboundConnection::write_next, BufferSlice(res.ok())); - send_closure(connection.release(), &HttpInboundConnection::write_ok); + send_closure(connection, &td::HttpInboundConnection::write_next, td::BufferSlice(res.ok())); + send_closure(connection.release(), &td::HttpInboundConnection::write_ok); } - void hangup() override { + void hangup() final { LOG(ERROR) << "CLOSE " << cnt--; stop(); } }; const int N = 0; -class Server : public TcpListener::Callback { +class Server final : public td::TcpListener::Callback { public: - void start_up() override { - listener_ = create_actor<TcpListener>("Listener", 8082, ActorOwn<TcpListener::Callback>(actor_id(this))); + void start_up() final { + listener_ = + td::create_actor<td::TcpListener>("Listener", 8082, td::ActorOwn<td::TcpListener::Callback>(actor_id(this))); } - void accept(SocketFd fd) override { + void accept(td::SocketFd fd) final { LOG(ERROR) << "ACCEPT " << cnt++; pos_++; auto scheduler_id = pos_ % (N != 0 ? N : 1) + (N != 0); - create_actor_on_scheduler<HttpInboundConnection>("HttpInboundConnection", scheduler_id, std::move(fd), 1024 * 1024, - 0, 0, - create_actor_on_scheduler<HelloWorld>("HelloWorld", scheduler_id)) + td::create_actor_on_scheduler<td::HttpInboundConnection>( + "HttpInboundConnection", scheduler_id, td::BufferedFd<td::SocketFd>(std::move(fd)), 1024 * 1024, 0, 0, + td::create_actor_on_scheduler<HelloWorld>("HelloWorld", scheduler_id)) .release(); } - void hangup() override { + void hangup() final { // may be it should be default?.. - LOG(ERROR) << "hangup.."; + LOG(ERROR) << "Hanging up.."; stop(); } private: - ActorOwn<TcpListener> listener_; + td::ActorOwn<td::TcpListener> listener_; int pos_{0}; }; int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); - auto scheduler = make_unique<ConcurrentScheduler>(); - scheduler->init(N); + auto scheduler = td::make_unique<td::ConcurrentScheduler>(N, 0); scheduler->create_actor_unsafe<Server>(0, "Server").release(); scheduler->start(); while (scheduler->run_main(10)) { // empty } scheduler->finish(); - return 0; -} -} // namespace td - -int main() { - return td::main(); } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp index da6fbbd713..8bbd768b40 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp @@ -1,49 +1,46 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/actor/actor.h" - #include "td/net/HttpHeaderCreator.h" -#include "td/net/HttpInboundConnection.h" #include "td/net/TcpListener.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" + #include "td/utils/buffer.h" #include "td/utils/logging.h" -#include "td/utils/port/Fd.h" +#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/SocketFd.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" #include <array> -namespace td { - -// HttpInboundConnection header static int cnt = 0; -class HelloWorld : public Actor { + +class HelloWorld final : public td::Actor { public: - explicit HelloWorld(SocketFd socket_fd) : socket_fd_(std::move(socket_fd)) { + explicit HelloWorld(td::SocketFd socket_fd) : socket_fd_(std::move(socket_fd)) { } private: - SocketFd socket_fd_; + td::SocketFd socket_fd_; std::array<char, 1024> read_buf; size_t read_new_lines{0}; - std::string hello_; - std::string write_buf_; + td::string hello_; + td::string write_buf_; size_t write_pos_{0}; - void start_up() override { - socket_fd_.get_fd().set_observer(this); - subscribe(socket_fd_.get_fd()); - HttpHeaderCreator hc; - Slice content = "hello world"; - //auto content = BufferSlice("hello world"); + void start_up() final { + td::Scheduler::subscribe(socket_fd_.get_poll_info().extract_pollable_fd(this)); + td::HttpHeaderCreator hc; + td::Slice content = "hello world"; + //auto content = td::BufferSlice("hello world"); hc.init_ok(); hc.set_keep_alive(); hc.set_content_size(content.size()); @@ -53,36 +50,37 @@ class HelloWorld : public Actor { hello_ = hc.finish(content).ok().str(); } - void loop() override { + void loop() final { auto status = do_loop(); if (status.is_error()) { - unsubscribe(socket_fd_.get_fd()); + td::Scheduler::unsubscribe(socket_fd_.get_poll_info().get_pollable_fd_ref()); stop(); LOG(ERROR) << "CLOSE: " << status; } } - Status do_loop() { + td::Status do_loop() { + sync_with_poll(socket_fd_); TRY_STATUS(read_loop()); TRY_STATUS(write_loop()); - if (can_close(socket_fd_)) { - return Status::Error("CLOSE"); + if (can_close_local(socket_fd_)) { + return td::Status::Error("CLOSE"); } - return Status::OK(); + return td::Status::OK(); } - Status write_loop() { - while (can_write(socket_fd_) && write_pos_ < write_buf_.size()) { - TRY_RESULT(written, socket_fd_.write(Slice(write_buf_).substr(write_pos_))); + td::Status write_loop() { + while (can_write_local(socket_fd_) && write_pos_ < write_buf_.size()) { + TRY_RESULT(written, socket_fd_.write(td::Slice(write_buf_).substr(write_pos_))); write_pos_ += written; if (write_pos_ == write_buf_.size()) { write_pos_ = 0; write_buf_.clear(); } } - return Status::OK(); + return td::Status::OK(); } - Status read_loop() { - while (can_read(socket_fd_)) { - TRY_RESULT(read_size, socket_fd_.read(MutableSlice(read_buf.data(), read_buf.size()))); + td::Status read_loop() { + while (can_read_local(socket_fd_)) { + TRY_RESULT(read_size, socket_fd_.read(td::MutableSlice(read_buf.data(), read_buf.size()))); for (size_t i = 0; i < read_size; i++) { if (read_buf[i] == '\n') { read_new_lines++; @@ -93,46 +91,41 @@ class HelloWorld : public Actor { } } } - return Status::OK(); + return td::Status::OK(); } }; + const int N = 0; -class Server : public TcpListener::Callback { +class Server final : public td::TcpListener::Callback { public: - void start_up() override { - listener_ = create_actor<TcpListener>("Listener", 8082, ActorOwn<TcpListener::Callback>(actor_id(this))); + void start_up() final { + listener_ = + td::create_actor<td::TcpListener>("Listener", 8082, td::ActorOwn<td::TcpListener::Callback>(actor_id(this))); } - void accept(SocketFd fd) override { + void accept(td::SocketFd fd) final { LOG(ERROR) << "ACCEPT " << cnt++; pos_++; auto scheduler_id = pos_ % (N != 0 ? N : 1) + (N != 0); - create_actor_on_scheduler<HelloWorld>("HttpInboundConnection", scheduler_id, std::move(fd)).release(); + td::create_actor_on_scheduler<HelloWorld>("HelloWorld", scheduler_id, std::move(fd)).release(); } - void hangup() override { + void hangup() final { // may be it should be default?.. - LOG(ERROR) << "hangup.."; + LOG(ERROR) << "Hanging up.."; stop(); } private: - ActorOwn<TcpListener> listener_; + td::ActorOwn<td::TcpListener> listener_; int pos_{0}; }; int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); - auto scheduler = make_unique<ConcurrentScheduler>(); - scheduler->init(N); + auto scheduler = td::make_unique<td::ConcurrentScheduler>(N, 0); scheduler->create_actor_unsafe<Server>(0, "Server").release(); scheduler->start(); while (scheduler->run_main(10)) { // empty } scheduler->finish(); - return 0; -} -} // namespace td - -int main() { - return td::main(); } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp index fbda47590b..4b140422f1 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp @@ -1,46 +1,48 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/actor/actor.h" - #include "td/net/HttpHeaderCreator.h" #include "td/net/HttpQuery.h" #include "td/net/HttpReader.h" #include "td/net/TcpListener.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" + #include "td/utils/buffer.h" #include "td/utils/BufferedFd.h" #include "td/utils/logging.h" -#include "td/utils/port/Fd.h" +#include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/SocketFd.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" -namespace td { - -class HttpEchoConnection : public Actor { +class HttpEchoConnection final : public td::Actor { public: - explicit HttpEchoConnection(SocketFd fd) : fd_(std::move(fd)) { + explicit HttpEchoConnection(td::SocketFd fd) : fd_(std::move(fd)) { } private: - BufferedFd<SocketFd> fd_; - HttpReader reader_; - HttpQuery query_; - void start_up() override { - fd_.get_fd().set_observer(this); - subscribe(fd_.get_fd()); + td::BufferedFd<td::SocketFd> fd_; + td::HttpReader reader_; + td::HttpQuery query_; + void start_up() final { + td::Scheduler::subscribe(fd_.get_poll_info().extract_pollable_fd(this)); reader_.init(&fd_.input_buffer(), 1024 * 1024, 0); } + void tear_down() final { + td::Scheduler::unsubscribe_before_close(fd_.get_poll_info().get_pollable_fd_ref()); + fd_.close(); + } void handle_query() { - query_ = HttpQuery(); - HttpHeaderCreator hc; - Slice content = "hello world"; - //auto content = BufferSlice("hello world"); + query_ = td::HttpQuery(); + td::HttpHeaderCreator hc; + td::Slice content = "hello world"; + //auto content = td::BufferSlice("hello world"); hc.init_ok(); hc.set_keep_alive(); hc.set_content_size(content.size()); @@ -51,20 +53,19 @@ class HttpEchoConnection : public Actor { fd_.output_buffer().append(res.ok()); } - void loop() override { + void loop() final { + sync_with_poll(fd_); auto status = [&] { TRY_STATUS(loop_read()); TRY_STATUS(loop_write()); - return Status::OK(); + return td::Status::OK(); }(); - if (status.is_error() || can_close(fd_)) { + if (status.is_error() || can_close_local(fd_)) { stop(); } } - Status loop_read() { - if (can_read(fd_)) { - TRY_STATUS(fd_.flush_read()); - } + td::Status loop_read() { + TRY_STATUS(fd_.flush_read()); while (true) { TRY_RESULT(need, reader_.read_next(&query_)); if (need == 0) { @@ -73,49 +74,43 @@ class HttpEchoConnection : public Actor { break; } } - return Status::OK(); + return td::Status::OK(); } - Status loop_write() { + td::Status loop_write() { TRY_STATUS(fd_.flush_write()); - return Status::OK(); + return td::Status::OK(); } }; -const int N = 4; -class Server : public TcpListener::Callback { +const int N = 8; +class Server final : public td::TcpListener::Callback { public: - void start_up() override { - listener_ = create_actor<TcpListener>("Listener", 8082, ActorOwn<TcpListener::Callback>(actor_id(this))); + void start_up() final { + listener_ = + td::create_actor<td::TcpListener>("Listener", 8082, td::ActorOwn<td::TcpListener::Callback>(actor_id(this))); } - void accept(SocketFd fd) override { + void accept(td::SocketFd fd) final { pos_++; auto scheduler_id = pos_ % (N != 0 ? N : 1) + (N != 0); - create_actor_on_scheduler<HttpEchoConnection>("HttpInboundConnection", scheduler_id, std::move(fd)).release(); + td::create_actor_on_scheduler<HttpEchoConnection>("HttpEchoConnection", scheduler_id, std::move(fd)).release(); } - void hangup() override { - LOG(ERROR) << "hangup.."; + void hangup() final { + LOG(ERROR) << "Hanging up.."; stop(); } private: - ActorOwn<TcpListener> listener_; + td::ActorOwn<td::TcpListener> listener_; int pos_{0}; }; int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); - auto scheduler = make_unique<ConcurrentScheduler>(); - scheduler->init(N); + auto scheduler = td::make_unique<td::ConcurrentScheduler>(N, 0); scheduler->create_actor_unsafe<Server>(0, "Server").release(); scheduler->start(); while (scheduler->run_main(10)) { // empty } scheduler->finish(); - return 0; -} -} // namespace td - -int main() { - return td::main(); } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp index a57b1b9b42..8c2628b8a9 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp @@ -1,16 +1,16 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include "td/utils/benchmark.h" +#include "td/utils/common.h" #include "td/utils/logging.h" #include <cstdio> #include <fstream> #include <iostream> -#include <mutex> #include <ostream> #include <streambuf> #include <string> @@ -35,63 +35,63 @@ std::string create_tmp_file() { #endif } -class IostreamWriteBench : public td::Benchmark { +class IostreamWriteBench final : public td::Benchmark { protected: std::string file_name_; std::ofstream stream; - enum { buffer_size = 1 << 20 }; - char buffer[buffer_size]; + static constexpr std::size_t BUFFER_SIZE = 1 << 20; + char buffer[BUFFER_SIZE]; public: - std::string get_description() const override { + std::string get_description() const final { return "ostream (to file, no buf, no flush)"; } - void start_up() override { + void start_up() final { file_name_ = create_tmp_file(); stream.open(file_name_.c_str()); CHECK(stream.is_open()); - // stream.rdbuf()->pubsetbuf(buffer, buffer_size); + // stream.rdbuf()->pubsetbuf(buffer, BUFFER_SIZE); } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { stream << "This is just for test" << 987654321 << '\n'; } } - void tear_down() override { + void tear_down() final { stream.close(); unlink(file_name_.c_str()); } }; -class FILEWriteBench : public td::Benchmark { +class FILEWriteBench final : public td::Benchmark { protected: std::string file_name_; FILE *file; - enum { buffer_size = 1 << 20 }; - char buffer[buffer_size]; + static constexpr std::size_t BUFFER_SIZE = 1 << 20; + char buffer[BUFFER_SIZE]; public: - std::string get_description() const override { + std::string get_description() const final { return "std::fprintf (to file, no buf, no flush)"; } - void start_up() override { + void start_up() final { file_name_ = create_tmp_file(); file = fopen(file_name_.c_str(), "w"); - // setvbuf(file, buffer, _IOFBF, buffer_size); + // setvbuf(file, buffer, _IOFBF, BUFFER_SIZE); } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { std::fprintf(file, "This is just for test%d\n", 987654321); // std::fflush(file); } } - void tear_down() override { + void tear_down() final { std::fclose(file); unlink(file_name_.c_str()); } @@ -100,58 +100,56 @@ class FILEWriteBench : public td::Benchmark { #if TD_ANDROID #include <android/log.h> #define ALOG(...) __android_log_print(ANDROID_LOG_VERBOSE, "XXX", __VA_ARGS__) -class ALogWriteBench : public td::Benchmark { +class ALogWriteBench final : public td::Benchmark { public: - std::string get_description() const override { + std::string get_description() const final { return "android_log"; } - void start_up() override { + void start_up() final { } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { ALOG("This is just for test%d\n", 987654321); } } - void tear_down() override { + void tear_down() final { } }; #endif -class LogWriteBench : public td::Benchmark { +class LogWriteBench final : public td::Benchmark { protected: std::string file_name_; std::ofstream stream; std::streambuf *old_buf; - enum { buffer_size = 1 << 20 }; - char buffer[buffer_size]; + static constexpr std::size_t BUFFER_SIZE = 1 << 20; + char buffer[BUFFER_SIZE]; public: - std::string get_description() const override { + std::string get_description() const final { return "td_log (slow in debug mode)"; } - void start_up() override { + void start_up() final { file_name_ = create_tmp_file(); stream.open(file_name_.c_str()); CHECK(stream.is_open()); old_buf = std::cerr.rdbuf(stream.rdbuf()); } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { LOG(DEBUG) << "This is just for test" << 987654321; } } - void tear_down() override { + void tear_down() final { stream.close(); unlink(file_name_.c_str()); std::cerr.rdbuf(old_buf); } }; -std::mutex mutex; - int main() { td::bench(LogWriteBench()); #if TD_ANDROID @@ -159,5 +157,4 @@ int main() { #endif td::bench(IostreamWriteBench()); td::bench(FILEWriteBench()); - return 0; } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp index bfbcea438b..56a6117bb4 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp @@ -1,11 +1,15 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // +#include "td/telegram/telegram_api.h" +#include "td/telegram/telegram_api.hpp" + #include "td/utils/benchmark.h" #include "td/utils/common.h" +#include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/port/Clocks.h" #include "td/utils/port/EventFd.h" @@ -15,9 +19,9 @@ #include "td/utils/port/Stat.h" #include "td/utils/port/thread.h" #include "td/utils/Slice.h" - -#include "td/telegram/telegram_api.h" -#include "td/telegram/telegram_api.hpp" +#include "td/utils/SliceBuilder.h" +#include "td/utils/Status.h" +#include "td/utils/ThreadSafeCounter.h" #if !TD_WINDOWS #include <unistd.h> @@ -28,37 +32,38 @@ #include <semaphore.h> #endif +#include <algorithm> +#include <array> #include <atomic> #include <cstdint> - -namespace td { +#include <set> class F { - uint32 ∑ + td::uint32 ∑ public: - explicit F(uint32 &sum) : sum(sum) { + explicit F(td::uint32 &sum) : sum(sum) { } template <class T> void operator()(const T &x) const { - sum += static_cast<uint32>(x.get_id()); + sum += static_cast<td::uint32>(x.get_id()); } }; BENCH(Call, "TL Call") { - tl_object_ptr<telegram_api::Function> x = make_tl_object<telegram_api::account_getWallPapers>(); - uint32 res = 0; + td::tl_object_ptr<td::telegram_api::Function> x = td::make_tl_object<td::telegram_api::account_getWallPapers>(0); + td::uint32 res = 0; F f(res); for (int i = 0; i < n; i++) { downcast_call(*x, f); } - do_not_optimize_away(res); + td::do_not_optimize_away(res); } #if !TD_EVENTFD_UNSUPPORTED BENCH(EventFd, "EventFd") { - EventFd fd; + td::EventFd fd; fd.init(); for (int i = 0; i < n; i++) { fd.release(); @@ -75,15 +80,15 @@ BENCH(NewInt, "new int + delete") { res += reinterpret_cast<std::uintptr_t>(x); delete x; } - do_not_optimize_away(res); + td::do_not_optimize_away(res); } -BENCH(NewObj, "new struct then delete") { +BENCH(NewObj, "new struct, then delete") { struct A { - int32 a = 0; - int32 b = 0; - int32 c = 0; - int32 d = 0; + td::int32 a = 0; + td::int32 b = 0; + td::int32 c = 0; + td::int32 d = 0; }; std::uintptr_t res = 0; A **ptr = new A *[n]; @@ -95,57 +100,57 @@ BENCH(NewObj, "new struct then delete") { delete ptr[i]; } delete[] ptr; - do_not_optimize_away(res); + td::do_not_optimize_away(res); } #if !TD_THREAD_UNSUPPORTED -BENCH(ThreadNew, "new struct then delete in several threads") { - td::NewObjBench a, b; - thread ta([&] { a.run(n / 2); }); - thread tb([&] { b.run(n - n / 2); }); +BENCH(ThreadNew, "new struct, then delete in 2 threads") { + NewObjBench a; + NewObjBench b; + td::thread ta([&] { a.run(n / 2); }); + td::thread tb([&] { b.run(n - n / 2); }); ta.join(); tb.join(); } #endif - -// Too hard for android clang (?) +/* +// Too hard for clang (?) BENCH(Time, "Clocks::monotonic") { double res = 0; for (int i = 0; i < n; i++) { - res += Clocks::monotonic(); + res += td::Clocks::monotonic(); } - do_not_optimize_away(res); + td::do_not_optimize_away(res); } - +*/ #if !TD_WINDOWS -class PipeBench : public Benchmark { +class PipeBench final : public td::Benchmark { public: int p[2]; - PipeBench() { - pipe(p); - } - - string get_description() const override { + td::string get_description() const final { return "pipe write + read int32"; } - void start_up() override { - pipe(p); + void start_up() final { + int res = pipe(p); + CHECK(res == 0); } - void run(int n) override { + void run(int n) final { int res = 0; for (int i = 0; i < n; i++) { int val = 1; - write(p[1], &val, sizeof(val)); - read(p[0], &val, sizeof(val)); + auto write_len = write(p[1], &val, sizeof(val)); + CHECK(write_len == sizeof(val)); + auto read_len = read(p[0], &val, sizeof(val)); + CHECK(read_len == sizeof(val)); res += val; } - do_not_optimize_away(res); + td::do_not_optimize_away(res); } - void tear_down() override { + void tear_down() final { close(p[0]); close(p[1]); } @@ -153,42 +158,42 @@ class PipeBench : public Benchmark { #endif #if TD_LINUX || TD_ANDROID || TD_TIZEN -class SemBench : public Benchmark { +class SemBench final : public td::Benchmark { sem_t sem; public: - string get_description() const override { + td::string get_description() const final { return "sem post + wait"; } - void start_up() override { + void start_up() final { int err = sem_init(&sem, 0, 0); CHECK(err != -1); } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { sem_post(&sem); sem_wait(&sem); } } - void tear_down() override { + void tear_down() final { sem_destroy(&sem); } }; #endif #if !TD_WINDOWS -class UtimeBench : public Benchmark { +class UtimeBench final : public td::Benchmark { public: - void start_up() override { - FileFd::open("test", FileFd::Flags::Create | FileFd::Flags::Write).move_as_ok().close(); + void start_up() final { + td::FileFd::open("test", td::FileFd::Create | td::FileFd::Write).move_as_ok().close(); } - string get_description() const override { + td::string get_description() const final { return "utime"; } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { int err = utime("test", nullptr); CHECK(err >= 0); @@ -203,73 +208,77 @@ class UtimeBench : public Benchmark { #endif BENCH(Pwrite, "pwrite") { - auto fd = FileFd::open("test", FileFd::Flags::Create | FileFd::Flags::Write).move_as_ok(); + auto fd = td::FileFd::open("test", td::FileFd::Create | td::FileFd::Write).move_as_ok(); for (int i = 0; i < n; i++) { fd.pwrite("a", 0).ok(); } fd.close(); } -class CreateFileBench : public Benchmark { - string get_description() const override { +class CreateFileBench final : public td::Benchmark { + td::string get_description() const final { return "create_file"; } - void start_up() override { - mkdir("A").ensure(); + void start_up() final { + td::mkdir("A").ensure(); } - void run(int n) override { + void run(int n) final { for (int i = 0; i < n; i++) { - FileFd::open("A/" + to_string(i), FileFd::Flags::Write | FileFd::Flags::Create).move_as_ok().close(); + td::FileFd::open(PSLICE() << "A/" << i, td::FileFd::Write | td::FileFd::Create).move_as_ok().close(); } } - void tear_down() override { - auto status = td::walk_path("A/", [&](CSlice path, bool is_dir) { - if (is_dir) { - rmdir(path).ignore(); - } else { - unlink(path).ignore(); + void tear_down() final { + td::walk_path("A/", [&](td::CSlice path, auto type) { + if (type == td::WalkPath::Type::ExitDir) { + td::rmdir(path).ignore(); + } else if (type == td::WalkPath::Type::NotDir) { + td::unlink(path).ignore(); } - }); + }).ignore(); } }; -class WalkPathBench : public Benchmark { - string get_description() const override { + +class WalkPathBench final : public td::Benchmark { + td::string get_description() const final { return "walk_path"; } - void start_up_n(int n) override { - mkdir("A").ensure(); + void start_up_n(int n) final { + td::mkdir("A").ensure(); for (int i = 0; i < n; i++) { - FileFd::open("A/" + to_string(i), FileFd::Flags::Write | FileFd::Flags::Create).move_as_ok().close(); + td::FileFd::open(PSLICE() << "A/" << i, td::FileFd::Write | td::FileFd::Create).move_as_ok().close(); } } - void run(int n) override { + void run(int n) final { int cnt = 0; - auto status = td::walk_path("A/", [&](CSlice path, bool is_dir) { - stat(path).ok(); + td::walk_path("A/", [&](td::CSlice path, auto type) { + if (type == td::WalkPath::Type::EnterDir) { + return; + } + td::stat(path).ok(); cnt++; - }); - } - void tear_down() override { - auto status = td::walk_path("A/", [&](CSlice path, bool is_dir) { - if (is_dir) { - rmdir(path).ignore(); - } else { - unlink(path).ignore(); + }).ignore(); + } + void tear_down() final { + td::walk_path("A/", [&](td::CSlice path, auto type) { + if (type == td::WalkPath::Type::ExitDir) { + td::rmdir(path).ignore(); + } else if (type == td::WalkPath::Type::NotDir) { + td::unlink(path).ignore(); } - }); + }).ignore(); } }; #if !TD_THREAD_UNSUPPORTED template <int ThreadN = 2> -class AtomicReleaseIncBench : public Benchmark { - string get_description() const override { +class AtomicReleaseIncBench final : public td::Benchmark { + td::string get_description() const final { return PSTRING() << "AtomicReleaseInc" << ThreadN; } - static std::atomic<uint64> a_; - void run(int n) override { - std::vector<thread> threads; + static std::atomic<td::uint64> a_; + void run(int n) final { + td::vector<td::thread> threads; for (int i = 0; i < ThreadN; i++) { threads.emplace_back([&] { for (int i = 0; i < n / ThreadN; i++) { @@ -283,17 +292,17 @@ class AtomicReleaseIncBench : public Benchmark { } }; template <int ThreadN> -std::atomic<uint64> AtomicReleaseIncBench<ThreadN>::a_; +std::atomic<td::uint64> AtomicReleaseIncBench<ThreadN>::a_; template <int ThreadN = 2> -class AtomicReleaseCasIncBench : public Benchmark { - string get_description() const override { +class AtomicReleaseCasIncBench final : public td::Benchmark { + td::string get_description() const final { return PSTRING() << "AtomicReleaseCasInc" << ThreadN; } - static std::atomic<uint64> a_; - void run(int n) override { - std::vector<thread> threads; + static std::atomic<td::uint64> a_; + void run(int n) final { + td::vector<td::thread> threads; for (int i = 0; i < ThreadN; i++) { threads.emplace_back([&] { for (int i = 0; i < n / ThreadN; i++) { @@ -309,16 +318,16 @@ class AtomicReleaseCasIncBench : public Benchmark { } }; template <int ThreadN> -std::atomic<uint64> AtomicReleaseCasIncBench<ThreadN>::a_; +std::atomic<td::uint64> AtomicReleaseCasIncBench<ThreadN>::a_; -template <int ThreadN = 2> -class RwMutexReadBench : public Benchmark { - string get_description() const override { +template <int ThreadN> +class RwMutexReadBench final : public td::Benchmark { + td::string get_description() const final { return PSTRING() << "RwMutexRead" << ThreadN; } - RwMutex mutex_; - void run(int n) override { - std::vector<thread> threads; + td::RwMutex mutex_; + void run(int n) final { + td::vector<td::thread> threads; for (int i = 0; i < ThreadN; i++) { threads.emplace_back([&] { for (int i = 0; i < n / ThreadN; i++) { @@ -331,14 +340,15 @@ class RwMutexReadBench : public Benchmark { } } }; -template <int ThreadN = 2> -class RwMutexWriteBench : public Benchmark { - string get_description() const override { + +template <int ThreadN> +class RwMutexWriteBench final : public td::Benchmark { + td::string get_description() const final { return PSTRING() << "RwMutexWrite" << ThreadN; } - RwMutex mutex_; - void run(int n) override { - std::vector<thread> threads; + td::RwMutex mutex_; + void run(int n) final { + td::vector<td::thread> threads; for (int i = 0; i < ThreadN; i++) { threads.emplace_back([&] { for (int i = 0; i < n / ThreadN; i++) { @@ -351,42 +361,366 @@ class RwMutexWriteBench : public Benchmark { } } }; + +class ThreadSafeCounterBench final : public td::Benchmark { + static td::ThreadSafeCounter counter_; + int thread_count_; + + td::string get_description() const final { + return PSTRING() << "ThreadSafeCounter" << thread_count_; + } + void run(int n) final { + counter_.clear(); + td::vector<td::thread> threads; + for (int i = 0; i < thread_count_; i++) { + threads.emplace_back([n] { + for (int i = 0; i < n; i++) { + counter_.add(1); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } + CHECK(counter_.sum() == n * thread_count_); + } + + public: + explicit ThreadSafeCounterBench(int thread_count) : thread_count_(thread_count) { + } +}; +td::ThreadSafeCounter ThreadSafeCounterBench::counter_; + +template <bool StrictOrder> +class AtomicCounterBench final : public td::Benchmark { + static std::atomic<td::int64> counter_; + int thread_count_; + + td::string get_description() const final { + return PSTRING() << "AtomicCounter" << thread_count_; + } + void run(int n) final { + counter_.store(0); + td::vector<td::thread> threads; + for (int i = 0; i < thread_count_; i++) { + threads.emplace_back([n] { + for (int i = 0; i < n; i++) { + counter_.fetch_add(1, StrictOrder ? std::memory_order_seq_cst : std::memory_order_relaxed); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } + CHECK(counter_.load() == n * thread_count_); + } + + public: + explicit AtomicCounterBench(int thread_count) : thread_count_(thread_count) { + } +}; +template <bool StrictOrder> +std::atomic<td::int64> AtomicCounterBench<StrictOrder>::counter_; + #endif -} // namespace td + +class IdDuplicateCheckerOld { + public: + static td::string get_description() { + return "Old"; + } + td::Status check(td::int64 message_id) { + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS) { + auto oldest_message_id = *saved_message_ids_.begin(); + if (message_id < oldest_message_id) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", oldest_message_id) + << td::tag("got message_id", message_id)); + } + } + if (saved_message_ids_.count(message_id) != 0) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + + saved_message_ids_.insert(message_id); + if (saved_message_ids_.size() > MAX_SAVED_MESSAGE_IDS) { + saved_message_ids_.erase(saved_message_ids_.begin()); + } + return td::Status::OK(); + } + + private: + static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; + std::set<td::int64> saved_message_ids_; +}; + +template <size_t MAX_SAVED_MESSAGE_IDS> +class IdDuplicateCheckerNew { + public: + static td::string get_description() { + return PSTRING() << "New" << MAX_SAVED_MESSAGE_IDS; + } + td::Status check(td::int64 message_id) { + auto insert_result = saved_message_ids_.insert(message_id); + if (!insert_result.second) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { + auto begin_it = saved_message_ids_.begin(); + bool is_very_old = begin_it == insert_result.first; + saved_message_ids_.erase(begin_it); + if (is_very_old) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", *saved_message_ids_.begin()) + << td::tag("got message_id", message_id)); + } + } + return td::Status::OK(); + } + + private: + std::set<td::int64> saved_message_ids_; +}; + +class IdDuplicateCheckerNewOther { + public: + static td::string get_description() { + return "NewOther"; + } + td::Status check(td::int64 message_id) { + if (!saved_message_ids_.insert(message_id).second) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { + auto begin_it = saved_message_ids_.begin(); + bool is_very_old = *begin_it == message_id; + saved_message_ids_.erase(begin_it); + if (is_very_old) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", *saved_message_ids_.begin()) + << td::tag("got message_id", message_id)); + } + } + return td::Status::OK(); + } + + private: + static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; + std::set<td::int64> saved_message_ids_; +}; + +class IdDuplicateCheckerNewSimple { + public: + static td::string get_description() { + return "NewSimple"; + } + td::Status check(td::int64 message_id) { + auto insert_result = saved_message_ids_.insert(message_id); + if (!insert_result.second) { + return td::Status::Error(1, "Ignore duplicated message_id"); + } + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { + auto begin_it = saved_message_ids_.begin(); + bool is_very_old = begin_it == insert_result.first; + saved_message_ids_.erase(begin_it); + if (is_very_old) { + return td::Status::Error(2, "Ignore very old message_id"); + } + } + return td::Status::OK(); + } + + private: + static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; + std::set<td::int64> saved_message_ids_; +}; + +template <size_t max_size> +class IdDuplicateCheckerArray { + public: + static td::string get_description() { + return PSTRING() << "Array" << max_size; + } + td::Status check(td::int64 message_id) { + if (end_pos_ == 2 * max_size) { + std::copy_n(&saved_message_ids_[max_size], max_size, &saved_message_ids_[0]); + end_pos_ = max_size; + } + if (end_pos_ == 0 || message_id > saved_message_ids_[end_pos_ - 1]) { + // fast path + saved_message_ids_[end_pos_++] = message_id; + return td::Status::OK(); + } + if (end_pos_ >= max_size && message_id < saved_message_ids_[0]) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", saved_message_ids_[0]) + << td::tag("got message_id", message_id)); + } + auto it = std::lower_bound(&saved_message_ids_[0], &saved_message_ids_[end_pos_], message_id); + if (*it == message_id) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + std::copy_backward(it, &saved_message_ids_[end_pos_], &saved_message_ids_[end_pos_ + 1]); + *it = message_id; + ++end_pos_; + return td::Status::OK(); + } + + private: + std::array<td::int64, 2 * max_size> saved_message_ids_; + std::size_t end_pos_ = 0; +}; + +template <class T> +class DuplicateCheckerBench final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBench" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + checker_.check(i).ensure(); + } + } +}; + +template <class T> +class DuplicateCheckerBenchRepeat final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchRepeat" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto iter = i >> 10; + auto pos = i - (iter << 10); + if (pos < 768) { + if (iter >= 3 && pos == 0) { + auto error = checker_.check((iter - 3) * 768 + pos); + CHECK(error.error().code() == 2); + } + checker_.check(iter * 768 + pos).ensure(); + } else { + checker_.check(iter * 768 + pos - 256).ensure_error(); + } + } + } +}; + +template <class T> +class DuplicateCheckerBenchRepeatOnly final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchRepeatOnly" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto result = checker_.check(i & 255); + CHECK(result.is_error() == (i >= 256)); + } + } +}; + +template <class T> +class DuplicateCheckerBenchReverse final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchReverseAdd" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto pos = i & 255; + checker_.check(i - pos + (255 - pos)).ensure(); + } + } +}; + +template <class T> +class DuplicateCheckerBenchEvenOdd final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchEvenOdd" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto pos = i & 255; + checker_.check(i - pos + (pos * 2) % 256 + (pos * 2) / 256).ensure(); + } + } +}; int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); + + td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerNew<1000>>()); + td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerNew<300>>()); + td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerArray<1000>>()); + td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerArray<300>>()); + + td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerNew<1000>>()); + td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerNew<300>>()); + td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerArray<1000>>()); + td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerArray<300>>()); + + td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerNew<1000>>()); + td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerNew<300>>()); + td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerArray<1000>>()); + td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerArray<300>>()); + + td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerOld>()); + td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNew<1000>>()); + td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNewOther>()); + td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNewSimple>()); + td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNew<300>>()); + td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerArray<1000>>()); + td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerArray<300>>()); + + td::bench(DuplicateCheckerBench<IdDuplicateCheckerOld>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<1000>>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerNewOther>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerNewSimple>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<300>>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<100>>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<10>>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerArray<1000>>()); + td::bench(DuplicateCheckerBench<IdDuplicateCheckerArray<300>>()); + #if !TD_THREAD_UNSUPPORTED - td::bench(td::AtomicReleaseIncBench<1>()); - td::bench(td::AtomicReleaseIncBench<2>()); - td::bench(td::AtomicReleaseCasIncBench<1>()); - td::bench(td::AtomicReleaseCasIncBench<2>()); - td::bench(td::RwMutexWriteBench<1>()); - td::bench(td::RwMutexReadBench<1>()); - td::bench(td::RwMutexWriteBench<>()); - td::bench(td::RwMutexReadBench<>()); + for (int i = 1; i <= 16; i *= 2) { + td::bench(ThreadSafeCounterBench(i)); + td::bench(AtomicCounterBench<false>(i)); + td::bench(AtomicCounterBench<true>(i)); + } + + td::bench(AtomicReleaseIncBench<1>()); + td::bench(AtomicReleaseIncBench<2>()); + td::bench(AtomicReleaseCasIncBench<1>()); + td::bench(AtomicReleaseCasIncBench<2>()); + td::bench(RwMutexWriteBench<1>()); + td::bench(RwMutexReadBench<1>()); + td::bench(RwMutexWriteBench<2>()); + td::bench(RwMutexReadBench<2>()); #endif #if !TD_WINDOWS - td::bench(td::UtimeBench()); + td::bench(UtimeBench()); #endif - td::bench(td::WalkPathBench()); - td::bench(td::CreateFileBench()); - td::bench(td::PwriteBench()); + td::bench(WalkPathBench()); + td::bench(CreateFileBench()); + td::bench(PwriteBench()); - td::bench(td::CallBench()); + td::bench(CallBench()); #if !TD_THREAD_UNSUPPORTED - td::bench(td::ThreadNewBench()); + td::bench(ThreadNewBench()); #endif #if !TD_EVENTFD_UNSUPPORTED - td::bench(td::EventFdBench()); + td::bench(EventFdBench()); #endif - td::bench(td::NewObjBench()); - td::bench(td::NewIntBench()); + td::bench(NewObjBench()); + td::bench(NewIntBench()); #if !TD_WINDOWS - td::bench(td::PipeBench()); + td::bench(PipeBench()); #endif #if TD_LINUX || TD_ANDROID || TD_TIZEN - td::bench(td::SemBench()); + td::bench(SemBench()); #endif - return 0; } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp index 13288e6cd7..6f7cf20bc9 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -8,47 +8,56 @@ #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/MpscPollableQueue.h" +#include "td/utils/port/sleep.h" +#include "td/utils/port/thread.h" #include "td/utils/queue.h" +#include "td/utils/Random.h" // TODO: check system calls // TODO: all return values must be checked #include <atomic> -#include <cstdio> -#include <cstdlib> -#include <vector> +#if TD_PORT_POSIX #include <pthread.h> #include <sched.h> #include <semaphore.h> #include <sys/syscall.h> #include <unistd.h> +#endif #if TD_LINUX #include <sys/eventfd.h> #endif -using std::atomic; -using std::vector; - -using td::int32; -using td::uint32; - #define MODE std::memory_order_relaxed // void set_affinity(int mask) { -// int err, syscallres; -// pid_t pid = gettid(); -// syscallres = syscall(__NR_sched_setaffinity, pid, sizeof(mask), &mask); -// if (syscallres) { -// err = errno; -// perror("oppa"); -//} -//} - -// TODO: warnings and asserts. There should be no warnings or debug output in production. +// pid_t pid = gettid(); +// int syscallres = syscall(__NR_sched_setaffinity, pid, sizeof(mask), &mask); +// if (syscallres) { +// perror("Failed to set affinity"); +// } +// } + using qvalue_t = int; +class Backoff { + int cnt = 0; + + public: + bool next() { + cnt++; + if (cnt < 50) { + return true; + } else { + td::usleep_for(1); + return cnt < 500; + } + } +}; + +#if TD_PORT_POSIX // Just for testing, not production class PipeQueue { int input; @@ -57,18 +66,21 @@ class PipeQueue { public: void init() { int new_pipe[2]; - pipe(new_pipe); + int res = pipe(new_pipe); + CHECK(res == 0); output = new_pipe[0]; input = new_pipe[1]; } void put(qvalue_t value) { - write(input, &value, sizeof(value)); + auto len = write(input, &value, sizeof(value)); + CHECK(len == sizeof(value)); } qvalue_t get() { qvalue_t res; - read(output, &res, sizeof(res)); + auto len = read(output, &res, sizeof(res)); + CHECK(len == sizeof(res)); return res; } @@ -78,26 +90,8 @@ class PipeQueue { } }; -class Backoff { - int cnt; - - public: - Backoff() : cnt(0) { - } - - bool next() { - cnt++; - if (cnt < 50) { - return true; - } else { - sched_yield(); - return cnt < 500; - } - } -}; - class VarQueue { - atomic<qvalue_t> data; + std::atomic<qvalue_t> data{0}; public: void init() { @@ -179,6 +173,7 @@ class SemQueue { return get(); } }; +#endif #if TD_LINUX class EventfdQueue { @@ -193,11 +188,14 @@ class EventfdQueue { void put(qvalue_t value) { q.put(value); td::int64 x = 1; - write(fd, &x, sizeof(x)); + auto len = write(fd, &x, sizeof(x)); + CHECK(len == sizeof(x)); } qvalue_t get() { td::int64 x; - read(fd, &x, sizeof(x)); + auto len = read(fd, &x, sizeof(x)); + CHECK(len == sizeof(x)); + CHECK(x == 1); return q.get(); } void destroy() { @@ -212,17 +210,17 @@ const int queue_buf_size = 1 << 10; class BufferQueue { struct node { qvalue_t val; - char pad[64 - sizeof(atomic<qvalue_t>)]; + char pad[64 - sizeof(std::atomic<qvalue_t>)]; }; node q[queue_buf_size]; struct Position { - atomic<uint32> i; - char pad[64 - sizeof(atomic<uint32>)]; + std::atomic<td::uint32> i{0}; + char pad[64 - sizeof(std::atomic<td::uint32>)]; - uint32 local_read_i; - uint32 local_write_i; - char pad2[64 - sizeof(uint32) * 2]; + td::uint32 local_read_i; + td::uint32 local_write_i; + char pad2[64 - sizeof(td::uint32) * 2]; void init() { i = 0; @@ -318,8 +316,7 @@ class BufferQueue { return; } if (!update_writer()) { - std::fprintf(stderr, "put strong failed\n"); - std::exit(0); + LOG(FATAL) << "Put strong failed"; } put_unsafe(val); } @@ -336,7 +333,7 @@ class BufferQueue { #if TD_LINUX class BufferedFdQueue { int fd; - atomic<int> wait_flag; + std::atomic<int> wait_flag{0}; BufferQueue q; char pad[64]; @@ -351,7 +348,8 @@ class BufferedFdQueue { td::int64 x = 1; __sync_synchronize(); if (wait_flag.load(MODE)) { - write(fd, &x, sizeof(x)); + auto len = write(fd, &x, sizeof(x)); + CHECK(len == sizeof(x)); } } void put_noflush(qvalue_t value) { @@ -362,7 +360,8 @@ class BufferedFdQueue { td::int64 x = 1; __sync_synchronize(); if (wait_flag.load(MODE)) { - write(fd, &x, sizeof(x)); + auto len = write(fd, &x, sizeof(x)); + CHECK(len == sizeof(x)); } } void flush_reader() { @@ -393,7 +392,8 @@ class BufferedFdQueue { wait_flag.store(1, MODE); __sync_synchronize(); while (!(res = q.update_reader())) { - read(fd, &x, sizeof(x)); + auto len = read(fd, &x, sizeof(x)); + CHECK(len == sizeof(x)); __sync_synchronize(); } wait_flag.store(0, MODE); @@ -416,7 +416,8 @@ class BufferedFdQueue { wait_flag.store(1, MODE); __sync_synchronize(); while (!q.update_reader()) { - read(fd, &x, sizeof(x)); + auto len = read(fd, &x, sizeof(x)); + CHECK(len == sizeof(x)); __sync_synchronize(); } wait_flag.store(0, MODE); @@ -430,7 +431,7 @@ class BufferedFdQueue { class FdQueue { int fd; - atomic<int> wait_flag; + std::atomic<int> wait_flag{0}; VarQueue q; char pad[64]; @@ -445,12 +446,14 @@ class FdQueue { td::int64 x = 1; __sync_synchronize(); if (wait_flag.load(MODE)) { - write(fd, &x, sizeof(x)); + auto len = write(fd, &x, sizeof(x)); + CHECK(len == sizeof(x)); } } qvalue_t get() { // td::int64 x; - // read(fd, &x, sizeof(x)); + // auto len = read(fd, &x, sizeof(x)); + // CHECK(len == sizeof(x)); // return q.get(); Backoff backoff; @@ -466,14 +469,13 @@ class FdQueue { td::int64 x; wait_flag.store(1, MODE); __sync_synchronize(); - // std::fprintf(stderr, "!\n"); - // while (res == -1 && read(fd, &x, sizeof(x))) { - // res = q.try_get(); - //} + // while (res == -1 && read(fd, &x, sizeof(x)) == sizeof(x)) { + // res = q.try_get(); + // } do { __sync_synchronize(); res = q.try_get(); - } while (res == -1 && read(fd, &x, sizeof(x))); + } while (res == -1 && read(fd, &x, sizeof(x)) == sizeof(x)); q.acquire(); wait_flag.store(0, MODE); return res; @@ -485,6 +487,7 @@ class FdQueue { }; #endif +#if TD_PORT_POSIX class SemBackoffQueue { sem_t sem; VarQueue q; @@ -554,46 +557,40 @@ class SemCheatQueue { }; template <class QueueT> -class QueueBenchmark2 : public td::Benchmark { +class QueueBenchmark2 final : public td::Benchmark { QueueT client, server; int connections_n, queries_n; int server_active_connections; int client_active_connections; - vector<td::int64> server_conn; - vector<td::int64> client_conn; + td::vector<td::int64> server_conn; + td::vector<td::int64> client_conn; + + td::string name; public: - explicit QueueBenchmark2(int connections_n = 1) : connections_n(connections_n) { + QueueBenchmark2(int connections_n, td::string name) : connections_n(connections_n), name(std::move(name)) { } - std::string get_description() const override { - return "QueueBenchmark2"; + td::string get_description() const final { + return name; } - void start_up() override { + void start_up() final { client.init(); server.init(); } - void tear_down() override { + void tear_down() final { client.destroy(); server.destroy(); } void server_process(qvalue_t value) { int no = value & 0x00FFFFFF; - int co = static_cast<int>(static_cast<unsigned int>(value) >> 24); - // std::fprintf(stderr, "-->%d %d\n", co, no); - if (co < 0 || co >= connections_n || no != server_conn[co]++) { - std::fprintf(stderr, "%d %d\n", co, no); - std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(server_conn[co] - 1)); - std::fprintf(stderr, "Server BUG\n"); - while (true) { - } - } - // std::fprintf(stderr, "no = %d/%d\n", no, queries_n); - // std::fprintf(stderr, "answer: %d %d\n", no, co); + auto co = static_cast<int>(static_cast<td::uint32>(value) >> 24); + CHECK(co >= 0 && co < connections_n); + CHECK(no == server_conn[co]++); client.writer_put(value); client.writer_flush(); @@ -603,15 +600,12 @@ class QueueBenchmark2 : public td::Benchmark { } void *server_run(void *) { - server_conn = vector<td::int64>(connections_n); + server_conn = td::vector<td::int64>(connections_n); server_active_connections = connections_n; while (server_active_connections > 0) { int cnt = server.reader_wait(); - if (cnt == 0) { - std::fprintf(stderr, "ERROR!\n"); - std::exit(0); - } + CHECK(cnt != 0); while (cnt-- > 0) { server_process(server.reader_get_unsafe()); server.reader_flush(); @@ -624,18 +618,10 @@ class QueueBenchmark2 : public td::Benchmark { void client_process(qvalue_t value) { int no = value & 0x00FFFFFF; - int co = static_cast<int>(static_cast<unsigned int>(value) >> 24); - // std::fprintf(stderr, "<--%d %d\n", co, no); - if (co < 0 || co >= connections_n || no != client_conn[co]++) { - std::fprintf(stderr, "%d %d\n", co, no); - std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(client_conn[co] - 1)); - std::fprintf(stderr, "BUG\n"); - while (true) { - } - std::exit(0); - } + auto co = static_cast<int>(static_cast<td::uint32>(value) >> 24); + CHECK(co >= 0 && co < connections_n); + CHECK(no == client_conn[co]++); if (no + 1 < queries_n) { - // std::fprintf(stderr, "query: %d %d\n", no + 1, co); server.writer_put(value + 1); server.writer_flush(); } else { @@ -644,12 +630,9 @@ class QueueBenchmark2 : public td::Benchmark { } void *client_run(void *) { - client_conn = vector<td::int64>(connections_n); + client_conn = td::vector<td::int64>(connections_n); client_active_connections = connections_n; - if (queries_n >= (1 << 24)) { - std::fprintf(stderr, "Too big queries_n\n"); - std::exit(0); - } + CHECK(queries_n < (1 << 24)); for (int i = 0; i < connections_n; i++) { server.writer_put(static_cast<qvalue_t>(i) << 24); @@ -658,10 +641,7 @@ class QueueBenchmark2 : public td::Benchmark { while (client_active_connections > 0) { int cnt = client.reader_wait(); - if (cnt == 0) { - std::fprintf(stderr, "ERROR!\n"); - std::exit(0); - } + CHECK(cnt != 0); while (cnt-- > 0) { client_process(client.reader_get_unsafe()); client.reader_flush(); @@ -681,7 +661,7 @@ class QueueBenchmark2 : public td::Benchmark { return static_cast<QueueBenchmark2 *>(arg)->server_run(nullptr); } - void run(int n) override { + void run(int n) final { pthread_t client_thread_id; pthread_t server_thread_id; @@ -696,45 +676,40 @@ class QueueBenchmark2 : public td::Benchmark { }; template <class QueueT> -class QueueBenchmark : public td::Benchmark { +class QueueBenchmark final : public td::Benchmark { QueueT client, server; const int connections_n; int queries_n; + td::string name; + public: - explicit QueueBenchmark(int connections_n = 1) : connections_n(connections_n) { + QueueBenchmark(int connections_n, td::string name) : connections_n(connections_n), name(std::move(name)) { } - std::string get_description() const override { - return "QueueBenchmark"; + td::string get_description() const final { + return name; } - void start_up() override { + void start_up() final { client.init(); server.init(); } - void tear_down() override { + void tear_down() final { client.destroy(); server.destroy(); } void *server_run(void *) { - vector<td::int64> conn(connections_n); + td::vector<td::int64> conn(connections_n); int active_connections = connections_n; while (active_connections > 0) { qvalue_t value = server.get(); int no = value & 0x00FFFFFF; - int co = static_cast<int>(value >> 24); - // std::fprintf(stderr, "-->%d %d\n", co, no); - if (co < 0 || co >= connections_n || no != conn[co]++) { - std::fprintf(stderr, "%d %d\n", co, no); - std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(conn[co] - 1)); - std::fprintf(stderr, "Server BUG\n"); - while (true) { - } - } - // std::fprintf(stderr, "no = %d/%d\n", no, queries_n); + auto co = static_cast<int>(value >> 24); + CHECK(co >= 0 && co < connections_n); + CHECK(no == conn[co]++); client.put(value); if (no + 1 >= queries_n) { active_connections--; @@ -744,11 +719,8 @@ class QueueBenchmark : public td::Benchmark { } void *client_run(void *) { - vector<td::int64> conn(connections_n); - if (queries_n >= (1 << 24)) { - std::fprintf(stderr, "Too big queries_n\n"); - std::exit(0); - } + td::vector<td::int64> conn(connections_n); + CHECK(queries_n < (1 << 24)); for (int i = 0; i < connections_n; i++) { server.put(static_cast<qvalue_t>(i) << 24); } @@ -756,16 +728,9 @@ class QueueBenchmark : public td::Benchmark { while (active_connections > 0) { qvalue_t value = client.get(); int no = value & 0x00FFFFFF; - int co = static_cast<int>(value >> 24); - // std::fprintf(stderr, "<--%d %d\n", co, no); - if (co < 0 || co >= connections_n || no != conn[co]++) { - std::fprintf(stderr, "%d %d\n", co, no); - std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(conn[co] - 1)); - std::fprintf(stderr, "BUG\n"); - while (true) { - } - std::exit(0); - } + auto co = static_cast<int>(value >> 24); + CHECK(co >= 0 && co < connections_n); + CHECK(no == conn[co]++); if (no + 1 < queries_n) { server.put(value + 1); } else { @@ -777,28 +742,18 @@ class QueueBenchmark : public td::Benchmark { } void *client_run2(void *) { - vector<td::int64> conn(connections_n); - if (queries_n >= (1 << 24)) { - std::fprintf(stderr, "Too big queries_n\n"); - std::exit(0); - } - for (int it = 0; it < queries_n; it++) { + td::vector<td::int64> conn(connections_n); + CHECK(queries_n < (1 << 24)); + for (int query = 0; query < queries_n; query++) { for (int i = 0; i < connections_n; i++) { - server.put((static_cast<td::int64>(i) << 24) + it); + server.put((static_cast<td::int64>(i) << 24) + query); } for (int i = 0; i < connections_n; i++) { qvalue_t value = client.get(); int no = value & 0x00FFFFFF; - int co = static_cast<int>(value >> 24); - // std::fprintf(stderr, "<--%d %d\n", co, no); - if (co < 0 || co >= connections_n || no != conn[co]++) { - std::fprintf(stderr, "%d %d\n", co, no); - std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(conn[co] - 1)); - std::fprintf(stderr, "BUG\n"); - while (true) { - } - std::exit(0); - } + auto co = static_cast<int>(value >> 24); + CHECK(co >= 0 && co < connections_n); + CHECK(no == conn[co]++); } } // system("cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"); @@ -813,7 +768,7 @@ class QueueBenchmark : public td::Benchmark { return static_cast<QueueBenchmark *>(arg)->server_run(nullptr); } - void run(int n) override { + void run(int n) final { pthread_t client_thread_id; pthread_t server_thread_id; @@ -828,8 +783,8 @@ class QueueBenchmark : public td::Benchmark { }; template <class QueueT> -class RingBenchmark : public td::Benchmark { - enum { QN = 504 }; +class RingBenchmark final : public td::Benchmark { + static constexpr int QN = 504; struct Thread { int int_id; @@ -840,7 +795,6 @@ class RingBenchmark : public td::Benchmark { void *run() { qvalue_t value; - // std::fprintf(stderr, "start %d\n", int_id); do { int cnt = queue.reader_wait(); CHECK(cnt == 1); @@ -861,7 +815,7 @@ class RingBenchmark : public td::Benchmark { return static_cast<Thread *>(arg)->run(); } - void start_up() override { + void start_up() final { for (int i = 0; i < QN; i++) { q[i].int_id = i; q[i].queue.init(); @@ -869,18 +823,17 @@ class RingBenchmark : public td::Benchmark { } } - void tear_down() override { + void tear_down() final { for (int i = 0; i < QN; i++) { q[i].queue.destroy(); } } - void run(int n) override { + void run(int n) final { for (int i = 0; i < QN; i++) { pthread_create(&q[i].id, nullptr, run_gateway, &q[i]); } - std::fprintf(stderr, "run %d\n", n); if (n < 1000) { n = 1000; } @@ -892,52 +845,89 @@ class RingBenchmark : public td::Benchmark { } } }; +#endif + +/* +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED +static void test_queue() { + td::vector<td::thread> threads; + static constexpr size_t THREAD_COUNT = 100; + td::vector<td::MpscPollableQueue<int>> queues(THREAD_COUNT); + for (auto &q : queues) { + q.init(); + } + for (size_t i = 0; i < THREAD_COUNT; i++) { + threads.emplace_back([&q = queues[i]] { + while (true) { + auto got = q.reader_wait_nonblock(); + while (got-- > 0) { + q.reader_get_unsafe(); + } + q.reader_get_event_fd().wait(1000); + } + }); + } + + for (size_t iter = 0; iter < THREAD_COUNT; iter++) { + td::usleep_for(100); + for (int i = 0; i < 5; i++) { + queues[td::Random::fast(0, THREAD_COUNT - 1)].writer_put(1); + } + } + + for (size_t i = 0; i < THREAD_COUNT; i++) { + threads[i].join(); + } +} +#endif +*/ int main() { - SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); -#define BENCH_Q2(Q, N) \ - std::fprintf(stderr, "!%s %d:\t", #Q, N); \ - td::bench(QueueBenchmark2<Q>(N)); -#define BENCH_Q(Q, N) \ - std::fprintf(stderr, "%s %d:\t", #Q, N); \ - td::bench(QueueBenchmark<Q>(N)); - -#define BENCH_R(Q) \ - std::fprintf(stderr, "%s:\t", #Q); \ - td::bench(RingBenchmark<Q>()); +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED + // test_queue(); +#endif + +#if TD_PORT_POSIX // TODO: yield makes it extremely slow. Yet some backoff may be necessary. - // BENCH_R(SemQueue); - // BENCH_R(td::PollQueue<qvalue_t>); + // td::bench(RingBenchmark<SemQueue>()); + // td::bench(RingBenchmark<td::PollQueue<qvalue_t>>()); - BENCH_Q2(td::PollQueue<qvalue_t>, 1); - BENCH_Q2(td::MpscPollableQueue<qvalue_t>, 1); - BENCH_Q2(td::PollQueue<qvalue_t>, 100); - BENCH_Q2(td::MpscPollableQueue<qvalue_t>, 100); - BENCH_Q2(td::PollQueue<qvalue_t>, 10); - BENCH_Q2(td::MpscPollableQueue<qvalue_t>, 10); +#define BENCH_Q2(Q, N) td::bench(QueueBenchmark2<Q<qvalue_t>>(N, #Q "(" #N ")")) - BENCH_Q(VarQueue, 1); - // BENCH_Q(FdQueue, 1); - // BENCH_Q(BufferedFdQueue, 1); - BENCH_Q(PipeQueue, 1); - BENCH_Q(SemCheatQueue, 1); - BENCH_Q(SemQueue, 1); +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED + BENCH_Q2(td::InfBackoffQueue, 1); + BENCH_Q2(td::MpscPollableQueue, 1); + BENCH_Q2(td::PollQueue, 1); + + BENCH_Q2(td::InfBackoffQueue, 10); + BENCH_Q2(td::MpscPollableQueue, 10); + BENCH_Q2(td::PollQueue, 10); - // BENCH_Q2(td::PollQueue<qvalue_t>, 100); - // BENCH_Q2(td::PollQueue<qvalue_t>, 10); - // BENCH_Q2(td::PollQueue<qvalue_t>, 4); - // BENCH_Q2(td::InfBackoffQueue<qvalue_t>, 100); + BENCH_Q2(td::InfBackoffQueue, 100); + BENCH_Q2(td::MpscPollableQueue, 100); + BENCH_Q2(td::PollQueue, 100); - // BENCH_Q2(td::InfBackoffQueue<qvalue_t>, 1); - // BENCH_Q(SemCheatQueue, 1); + BENCH_Q2(td::PollQueue, 4); + BENCH_Q2(td::PollQueue, 10); + BENCH_Q2(td::PollQueue, 100); +#endif - // BENCH_Q(BufferedFdQueue, 100); - // BENCH_Q(BufferedFdQueue, 10); +#define BENCH_Q(Q, N) td::bench(QueueBenchmark<Q>(N, #Q "(" #N ")")) - // BENCH_Q(BufferQueue, 4); - // BENCH_Q(BufferQueue, 100); - // BENCH_Q(BufferQueue, 10); - // BENCH_Q(BufferQueue, 1); +#if TD_LINUX + BENCH_Q(BufferQueue, 1); + BENCH_Q(BufferedFdQueue, 1); + BENCH_Q(FdQueue, 1); +#endif + BENCH_Q(PipeQueue, 1); + BENCH_Q(SemCheatQueue, 1); + BENCH_Q(SemQueue, 1); + BENCH_Q(VarQueue, 1); - return 0; +#if TD_LINUX + BENCH_Q(BufferQueue, 4); + BENCH_Q(BufferQueue, 10); + BENCH_Q(BufferQueue, 100); +#endif +#endif } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp index 91e957a501..c3298327e5 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp @@ -1,70 +1,78 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include "td/telegram/DialogId.h" +#include "td/telegram/MessageDb.h" #include "td/telegram/MessageId.h" -#include "td/telegram/MessagesDb.h" +#include "td/telegram/NotificationId.h" +#include "td/telegram/ServerMessageId.h" #include "td/telegram/UserId.h" +#include "td/db/DbKey.h" +#include "td/db/SqliteConnectionSafe.h" +#include "td/db/SqliteDb.h" + +#include "td/actor/ConcurrentScheduler.h" + #include "td/utils/benchmark.h" #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/logging.h" +#include "td/utils/Promise.h" #include "td/utils/Random.h" #include "td/utils/Status.h" #include <memory> -namespace td { - -static Status init_db(SqliteDb &db) { +static td::Status init_db(td::SqliteDb &db) { TRY_STATUS(db.exec("PRAGMA encoding=\"UTF-8\"")); TRY_STATUS(db.exec("PRAGMA synchronous=NORMAL")); TRY_STATUS(db.exec("PRAGMA journal_mode=WAL")); TRY_STATUS(db.exec("PRAGMA temp_store=MEMORY")); TRY_STATUS(db.exec("PRAGMA secure_delete=1")); - return Status::OK(); + return td::Status::OK(); } -class MessagesDbBench : public Benchmark { +class MessageDbBench final : public td::Benchmark { public: - string get_description() const override { - return "MessagesDb"; + td::string get_description() const final { + return "MessageDb"; } - void start_up() override { + void start_up() final { LOG(ERROR) << "START UP"; do_start_up().ensure(); scheduler_->start(); } - void run(int n) override { - auto guard = scheduler_->get_current_guard(); + void run(int n) final { + auto guard = scheduler_->get_main_guard(); for (int i = 0; i < n; i += 20) { - auto dialog_id = DialogId{UserId{Random::fast(1, 100)}}; - auto message_id_raw = Random::fast(1, 100000); + auto dialog_id = td::DialogId(td::UserId(static_cast<td::int64>(td::Random::fast(1, 100)))); + auto message_id_raw = td::Random::fast(1, 100000); for (int j = 0; j < 20; j++) { - auto message_id = MessageId{ServerMessageId{message_id_raw + j}}; - auto unique_message_id = ServerMessageId{i + 1}; - auto sender_user_id = UserId{Random::fast(1, 1000)}; + auto message_id = td::MessageId{td::ServerMessageId{message_id_raw + j}}; + auto unique_message_id = td::ServerMessageId{i + 1}; + auto sender_dialog_id = td::DialogId(td::UserId(static_cast<td::int64>(td::Random::fast(1, 1000)))); auto random_id = i + 1; auto ttl_expires_at = 0; - auto data = BufferSlice(Random::fast(100, 299)); + auto data = td::BufferSlice(td::Random::fast(100, 299)); // use async on same thread. - messages_db_async_->add_message({dialog_id, message_id}, unique_message_id, sender_user_id, random_id, - ttl_expires_at, 0, 0, "", std::move(data), Promise<>()); + message_db_async_->add_message({dialog_id, message_id}, unique_message_id, sender_dialog_id, random_id, + ttl_expires_at, 0, 0, "", td::NotificationId(), td::MessageId(), std::move(data), + td::Promise<>()); } } } - void tear_down() override { + void tear_down() final { scheduler_->run_main(0.1); { - auto guard = scheduler_->get_current_guard(); + auto guard = scheduler_->get_main_guard(); sql_connection_.reset(); - messages_db_sync_safe_.reset(); - messages_db_async_.reset(); + message_db_sync_safe_.reset(); + message_db_async_.reset(); } scheduler_->finish(); @@ -73,36 +81,33 @@ class MessagesDbBench : public Benchmark { } private: - std::unique_ptr<td::ConcurrentScheduler> scheduler_; - std::shared_ptr<SqliteConnectionSafe> sql_connection_; - std::shared_ptr<MessagesDbSyncSafeInterface> messages_db_sync_safe_; - std::shared_ptr<MessagesDbAsyncInterface> messages_db_async_; + td::unique_ptr<td::ConcurrentScheduler> scheduler_; + std::shared_ptr<td::SqliteConnectionSafe> sql_connection_; + std::shared_ptr<td::MessageDbSyncSafeInterface> message_db_sync_safe_; + std::shared_ptr<td::MessageDbAsyncInterface> message_db_async_; - Status do_start_up() { - scheduler_ = std::make_unique<ConcurrentScheduler>(); - scheduler_->init(1); + td::Status do_start_up() { + scheduler_ = td::make_unique<td::ConcurrentScheduler>(1, 0); - auto guard = scheduler_->get_current_guard(); + auto guard = scheduler_->get_main_guard(); - string sql_db_name = "testdb.sqlite"; - sql_connection_ = std::make_shared<SqliteConnectionSafe>(sql_db_name); + td::string sql_db_name = "testdb.sqlite"; + sql_connection_ = std::make_shared<td::SqliteConnectionSafe>(sql_db_name, td::DbKey::empty()); auto &db = sql_connection_->get(); TRY_STATUS(init_db(db)); db.exec("BEGIN TRANSACTION").ensure(); // version == 0 ==> db will be destroyed - TRY_STATUS(init_messages_db(db, 0)); + TRY_STATUS(init_message_db(db, 0)); db.exec("COMMIT TRANSACTION").ensure(); - messages_db_sync_safe_ = create_messages_db_sync(sql_connection_); - messages_db_async_ = create_messages_db_async(messages_db_sync_safe_, 0); - return Status::OK(); + message_db_sync_safe_ = td::create_message_db_sync(sql_connection_); + message_db_async_ = td::create_message_db_async(message_db_sync_safe_, 0); + return td::Status::OK(); } }; -} // namespace td int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(WARNING)); - bench(td::MessagesDbBench()); - return 0; + td::bench(MessageDbBench()); } diff --git a/protocols/Telegram/tdlib/td/benchmark/check_proxy.cpp b/protocols/Telegram/tdlib/td/benchmark/check_proxy.cpp new file mode 100644 index 0000000000..0a54c598c0 --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/check_proxy.cpp @@ -0,0 +1,193 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/Client.h" +#include "td/telegram/td_api.h" + +#include "td/utils/base64.h" +#include "td/utils/common.h" +#include "td/utils/filesystem.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/TsCerr.h" + +#include <algorithm> +#include <cstdlib> +#include <iostream> +#include <utility> + +static void usage() { + td::TsCerr() << "Tests specified MTProto-proxies, outputs working proxies to stdout; exits with code 0 if a working " + "proxy was found.\n"; + td::TsCerr() << "Usage: check_proxy [options] server:port:secret [server2:port2:secret2 ...]\n"; + td::TsCerr() << "Options:\n"; + td::TsCerr() << " -v<N>\tSet verbosity level to N\n"; + td::TsCerr() << " -h/--help\tDisplay this information\n"; + td::TsCerr() << " -d/--dc-id\tIdentifier of a datacenter to which try to connect (default is 2)\n"; + td::TsCerr() << " -l/--proxy-list\tName of a file with proxies to check; one proxy per line\n"; + td::TsCerr() << " -t/--timeout\tMaximum overall timeout for the request (default is 10 seconds)\n"; + std::exit(2); +} + +int main(int argc, char **argv) { + int new_verbosity_level = VERBOSITY_NAME(FATAL); + + td::vector<std::pair<td::string, td::td_api::object_ptr<td::td_api::testProxy>>> requests; + + auto add_proxy = [&requests](td::string arg) { + if (arg.empty()) { + return; + } + + std::size_t offset = 0; + if (arg[0] == '[') { + auto end_ipv6_pos = arg.find(']'); + if (end_ipv6_pos == td::string::npos) { + td::TsCerr() << "Error: failed to find end of IPv6 address in \"" << arg << "\"\n"; + usage(); + } + offset = end_ipv6_pos; + } + if (std::count(arg.begin() + offset, arg.end(), ':') == 3) { + auto secret_domain_pos = arg.find(':', arg.find(':', offset) + 1) + 1; + auto domain_pos = arg.find(':', secret_domain_pos); + auto secret = arg.substr(secret_domain_pos, domain_pos - secret_domain_pos); + auto domain = arg.substr(domain_pos + 1); + auto r_decoded_secret = td::hex_decode(secret); + if (r_decoded_secret.is_error()) { + r_decoded_secret = td::base64url_decode(secret); + if (r_decoded_secret.is_error()) { + td::TsCerr() << "Error: failed to find proxy port and secret in \"" << arg << "\"\n"; + usage(); + } + } + arg = arg.substr(0, secret_domain_pos) + td::base64url_encode(r_decoded_secret.ok() + domain); + } + + auto secret_pos = arg.rfind(':'); + if (secret_pos == td::string::npos) { + td::TsCerr() << "Error: failed to find proxy port and secret in \"" << arg << "\"\n"; + usage(); + } + auto secret = arg.substr(secret_pos + 1); + auto port_pos = arg.substr(0, secret_pos).rfind(':'); + if (port_pos == td::string::npos) { + td::TsCerr() << "Error: failed to find proxy secret in \"" << arg << "\"\n"; + usage(); + } + auto r_port = td::to_integer_safe<td::int32>(arg.substr(port_pos + 1, secret_pos - port_pos - 1)); + if (r_port.is_error()) { + td::TsCerr() << "Error: failed to parse proxy port in \"" << arg << "\"\n"; + usage(); + } + auto port = r_port.move_as_ok(); + auto server = arg.substr(0, port_pos); + if (server[0] == '[' && server.back() == ']') { + server = server.substr(1, server.size() - 2); + } + + if (server.empty() || port <= 0 || port > 65536 || secret.empty()) { + td::TsCerr() << "Error: proxy address to check is in wrong format: \"" << arg << "\"\n"; + usage(); + } + + requests.emplace_back(arg, + td::td_api::make_object<td::td_api::testProxy>( + server, port, td::td_api::make_object<td::td_api::proxyTypeMtproto>(secret), -1, -1)); + }; + + td::int32 dc_id = 2; + double timeout = 10.0; + + for (int i = 1; i < argc; i++) { + td::string arg(argv[i]); + + auto get_next_arg = [&i, &arg, argc, argv](bool is_optional = false) { + CHECK(arg.size() >= 2); + if (arg.size() == 2 || arg[1] == '-') { + if (i + 1 < argc && argv[i + 1][0] != '-') { + return td::string(argv[++i]); + } + } else { + if (arg.size() > 2) { + return arg.substr(2); + } + } + if (!is_optional) { + td::TsCerr() << "Error: value is required after " << arg << "\n"; + usage(); + } + return td::string(); + }; + + if (td::begins_with(arg, "-v")) { + arg = get_next_arg(true); + int new_verbosity = 1; + while (arg[0] == 'v') { + new_verbosity++; + arg = arg.substr(1); + } + if (!arg.empty()) { + new_verbosity += td::to_integer<int>(arg) - (new_verbosity == 1); + } + new_verbosity_level = VERBOSITY_NAME(FATAL) + new_verbosity; + } else if (td::begins_with(arg, "-t") || arg == "--timeout") { + timeout = td::to_double(get_next_arg()); + } else if (td::begins_with(arg, "-d") || arg == "--dc-id") { + dc_id = td::to_integer<td::int32>(get_next_arg()); + } else if (td::begins_with(arg, "-l") || arg == "--proxy-list") { + auto r_proxies = td::read_file_str(get_next_arg()); + if (r_proxies.is_error()) { + td::TsCerr() << "Error: wrong file name specified\n"; + usage(); + } + for (auto &proxy : td::full_split(r_proxies.ok(), '\n')) { + add_proxy(td::trim(proxy)); + } + } else if (arg[0] == '-') { + usage(); + } else { + add_proxy(arg); + } + } + + if (requests.empty()) { + td::TsCerr() << "Error: proxy address to check is not specified\n"; + usage(); + } + + SET_VERBOSITY_LEVEL(new_verbosity_level); + + td::ClientManager client_manager; + auto client_id = client_manager.create_client_id(); + for (size_t i = 0; i < requests.size(); i++) { + auto &request = requests[i].second; + request->dc_id_ = dc_id; + request->timeout_ = timeout; + client_manager.send(client_id, i + 1, std::move(request)); + } + size_t successful_requests = 0; + size_t failed_requests = 0; + + while (successful_requests + failed_requests != requests.size()) { + auto response = client_manager.receive(100.0); + CHECK(client_id == response.client_id); + if (1 <= response.request_id && response.request_id <= requests.size()) { + auto &proxy = requests[static_cast<size_t>(response.request_id - 1)].first; + if (response.object->get_id() == td::td_api::error::ID) { + LOG(ERROR) << proxy << ": " << to_string(response.object); + failed_requests++; + } else { + std::cout << proxy << std::endl; + successful_requests++; + } + } + } + + if (successful_requests == 0) { + return 1; + } +} diff --git a/protocols/Telegram/tdlib/td/benchmark/check_tls.cpp b/protocols/Telegram/tdlib/td/benchmark/check_tls.cpp new file mode 100644 index 0000000000..f7fb1fecb7 --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/check_tls.cpp @@ -0,0 +1,336 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/BigNum.h" +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/detail/Iocp.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/port/sleep.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/port/thread.h" +#include "td/utils/Random.h" +#include "td/utils/Slice.h" +#include "td/utils/SliceBuilder.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" + +#include <map> + +static td::BigNumContext context; + +static bool is_quadratic_residue(const td::BigNum &a) { + // 2^255 - 19 + td::BigNum mod = + td::BigNum::from_hex("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed").move_as_ok(); + // (mod - 1) / 2 = 2^254 - 10 + td::BigNum pow = + td::BigNum::from_hex("3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6").move_as_ok(); + + td::BigNum r; + td::BigNum::mod_exp(r, a, pow, mod, context); + td::BigNum one = td::BigNum::from_decimal("1").move_as_ok(); + td::BigNum::mod_add(r, r, one, mod, context); + + td::string result = r.to_decimal(); + CHECK(result == "0" || result == "1" || result == "2"); + return result == "2"; +} + +struct TlsInfo { + td::vector<size_t> extension_list; + td::vector<size_t> encrypted_application_data_length; +}; + +td::Result<TlsInfo> test_tls(const td::string &url) { + td::IPAddress address; + TRY_STATUS(address.init_host_port(url, 443)); + TRY_RESULT(socket, td::SocketFd::open(address)); + + td::string request; + + auto add_string = [&](td::Slice data) { + request.append(data.data(), data.size()); + }; + auto add_random = [&](size_t length) { + while (length-- > 0) { + request += static_cast<char>(td::Random::secure_int32()); + } + }; + auto add_length = [&](size_t length) { + request += static_cast<char>(length / 256); + request += static_cast<char>(length % 256); + }; + auto add_key = [&] { + td::string key(32, '\0'); + td::BigNum mod = + td::BigNum::from_hex("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed").move_as_ok(); + while (true) { + td::Random::secure_bytes(key); + key[31] = static_cast<char>(key[31] & 127); + td::BigNum x = td::BigNum::from_le_binary(key); + if (!is_quadratic_residue(x)) { + continue; + } + + td::BigNum y = x.clone(); + td::BigNum coef = td::BigNum::from_decimal("486662").move_as_ok(); + td::BigNum::mod_add(y, y, coef, mod, context); + td::BigNum::mod_mul(y, y, x, mod, context); + td::BigNum one = td::BigNum::from_decimal("1").move_as_ok(); + td::BigNum::mod_add(y, y, one, mod, context); + td::BigNum::mod_mul(y, y, x, mod, context); + // y = x^3 + 486662 * x^2 + x + if (is_quadratic_residue(y)) { + break; + } + } + request += key; + }; + + const size_t MAX_GREASE = 7; + char greases[MAX_GREASE]; + td::Random::secure_bytes(td::MutableSlice{greases, MAX_GREASE}); + for (auto &grease : greases) { + grease = static_cast<char>((grease & 0xF0) + 0x0A); + } + for (size_t i = 1; i < MAX_GREASE; i += 2) { + if (greases[i] == greases[i - 1]) { + greases[i] = static_cast<char>(0x10 ^ greases[i]); + } + } + auto add_grease = [&](size_t num) { + auto c = greases[num]; + request += c; + request += c; + }; + + add_string("\x16\x03\x01\x02\x00\x01\x00\x01\xfc\x03\x03"); + add_random(32); + add_string("\x20"); + add_random(32); + add_string("\x00\x20"); + add_grease(0); + add_string( + "\x13\x01\x13\x02\x13\x03\xc0\x2b\xc0\x2f\xc0\x2c\xc0\x30\xcc\xa9\xcc\xa8\xc0\x13\xc0\x14\x00\x9c\x00\x9d\x00" + "\x2f\x00\x35\x01\x00\x01\x93"); + add_grease(2); + add_string("\x00\x00\x00\x00"); + add_length(url.size() + 5); + add_length(url.size() + 3); + add_string("\x00"); + add_length(url.size()); + add_string(url); + add_string("\x00\x17\x00\x00\xff\x01\x00\x01\x00\x00\x0a\x00\x0a\x00\x08"); + add_grease(4); + add_string( + "\x00\x1d\x00\x17\x00\x18\x00\x0b\x00\x02\x01\x00\x00\x23\x00\x00\x00\x10\x00\x0e\x00\x0c\x02\x68\x32\x08\x68" + "\x74\x74\x70\x2f\x31\x2e\x31\x00\x05\x00\x05\x01\x00\x00\x00\x00\x00\x0d\x00\x12\x00\x10\x04\x03\x08\x04\x04" + "\x01\x05\x03\x08\x05\x05\x01\x08\x06\x06\x01\x00\x12\x00\x00\x00\x33\x00\x2b\x00\x29"); + add_grease(4); + add_string("\x00\x01\x00\x00\x1d\x00\x20"); + add_key(); + add_string("\x00\x2d\x00\x02\x01\x01\x00\x2b\x00\x0b\x0a"); + add_grease(6); + add_string("\x03\x04\x03\x03\x03\x02\x03\x01\x00\x1b\x00\x03\x02\x00\x02"); + add_grease(3); + add_string("\x00\x01\x00\x00\x15"); + auto padding = 515 - static_cast<int>(request.size()); + CHECK(padding >= 0); + add_length(padding); + request.resize(517); + + // LOG(ERROR) << td::format::as_hex_dump<0>(td::Slice(request)); + + TRY_STATUS(socket.write(request)); + + TlsInfo info; + auto end_time = td::Time::now() + 3; + td::string result; + size_t pos = 0; + size_t server_hello_length = 0; + size_t encrypted_application_data_length_sum = 0; + while (td::Time::now() < end_time) { + static char buf[20000]; + TRY_RESULT(res, socket.read(td::MutableSlice{buf, sizeof(buf)})); + if (res > 0) { + auto read_length = [&]() -> size_t { + CHECK(result.size() >= 2 + pos); + pos += 2; + return static_cast<unsigned char>(result[pos - 2]) * 256 + static_cast<unsigned char>(result[pos - 1]); + }; + + result += td::Slice(buf, res).str(); + while (true) { +#define CHECK_LENGTH(length) \ + if (pos + (length) > result.size()) { \ + break; \ + } +#define EXPECT_STR(pos, str, error) \ + if (!begins_with(td::Slice(result).substr(pos), str)) { \ + return td::Status::Error(error); \ + } + + if (pos == 0) { + CHECK_LENGTH(3); + EXPECT_STR(0, "\x16\x03\x03", "Non-TLS response or TLS <= 1.1"); + pos += 3; + } + if (pos == 3) { + CHECK_LENGTH(2); + server_hello_length = read_length(); + if (server_hello_length <= 39) { + return td::Status::Error("Receive too short server hello"); + } + } + if (server_hello_length > 0) { + if (pos == 5) { + CHECK_LENGTH(server_hello_length); + + EXPECT_STR(5, "\x02\x00", "Non-TLS response 2"); + EXPECT_STR(9, "\x03\x03", "Non-TLS response 3"); + + auto random_id = td::Slice(result.c_str() + 11, 32); + if (random_id == + "\xcf\x21\xad\x74\xe5\x9a\x61\x11\xbe\x1d\x8c\x02\x1e\x65\xb8\x91\xc2\xa2\x11\x16\x7a\xbb\x8c\x5e\x07" + "\x9e\x09\xe2\xc8\xa8\x33\x9c") { + return td::Status::Error("TLS 1.3 servers returning HelloRetryRequest are not supprted"); + } + if (result[43] == '\x00') { + return td::Status::Error("TLS <= 1.2: empty session_id"); + } + EXPECT_STR(43, "\x20", "Non-TLS response 4"); + if (server_hello_length <= 75) { + return td::Status::Error("Receive too short server hello 2"); + } + EXPECT_STR(44, request.substr(44, 32), "TLS <= 1.2: expected mirrored session_id"); + EXPECT_STR(76, "\x13\x01\x00", "TLS <= 1.2: expected 0x1301 as a chosen cipher"); + pos += 74; + size_t extensions_length = read_length(); + if (extensions_length + 76 != server_hello_length) { + return td::Status::Error("Receive wrong extensions length"); + } + while (pos < 5 + server_hello_length - 4) { + info.extension_list.push_back(read_length()); + size_t extension_length = read_length(); + if (pos + extension_length > 5 + server_hello_length) { + return td::Status::Error("Receive wrong extension length"); + } + pos += extension_length; + } + if (pos != 5 + server_hello_length) { + return td::Status::Error("Receive wrong extensions list"); + } + } + if (pos == 5 + server_hello_length) { + CHECK_LENGTH(6); + EXPECT_STR(pos, "\x14\x03\x03\x00\x01\x01", "Expected dummy ChangeCipherSpec"); + pos += 6; + } + if (pos == 11 + server_hello_length + encrypted_application_data_length_sum) { + if (pos == result.size()) { + return info; + } + + CHECK_LENGTH(3); + EXPECT_STR(pos, "\x17\x03\x03", "Expected encrypted application data"); + pos += 3; + } + if (pos == 14 + server_hello_length + encrypted_application_data_length_sum) { + CHECK_LENGTH(2); + size_t encrypted_application_data_length = read_length(); + info.encrypted_application_data_length.push_back(encrypted_application_data_length); + if (encrypted_application_data_length == 0) { + return td::Status::Error("Receive empty encrypted application data"); + } + } + if (pos == 16 + server_hello_length + encrypted_application_data_length_sum) { + CHECK_LENGTH(info.encrypted_application_data_length.back()); + pos += info.encrypted_application_data_length.back(); + encrypted_application_data_length_sum += info.encrypted_application_data_length.back() + 5; + } + } + } + } else { + td::usleep_for(10000); + } + } + + // LOG(ERROR) << url << ":" << td::format::as_hex_dump<0>(td::Slice(result)); + return td::Status::Error("Failed to get response in 3 seconds"); +} + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(WARNING)); + +#if TD_PORT_WINDOWS + auto iocp = td::make_unique<td::detail::Iocp>(); + iocp->init(); + auto iocp_thread = td::thread([&iocp] { iocp->loop(); }); + td::detail::Iocp::Guard iocp_guard(iocp.get()); +#endif + + td::vector<td::string> urls; + for (int i = 1; i < argc; i++) { + urls.emplace_back(argv[i]); + } + for (auto &url : urls) { + const int MAX_TRIES = 100; + td::vector<std::map<size_t, int>> length_count; + td::vector<size_t> extension_list; + for (int i = 0; i < MAX_TRIES; i++) { + auto r_tls_info = test_tls(url); + if (r_tls_info.is_error()) { + LOG(ERROR) << url << ": " << r_tls_info.error(); + break; + } else { + auto tls_info = r_tls_info.move_as_ok(); + if (length_count.size() < tls_info.encrypted_application_data_length.size()) { + length_count.resize(tls_info.encrypted_application_data_length.size()); + } + for (size_t t = 0; t < tls_info.encrypted_application_data_length.size(); t++) { + length_count[t][tls_info.encrypted_application_data_length[t]]++; + } + if (i == 0) { + extension_list = tls_info.extension_list; + } else { + if (extension_list != tls_info.extension_list) { + LOG(ERROR) << url << ": TLS 1.3.0 extension list has changed from " << extension_list << " to " + << tls_info.extension_list; + break; + } + } + } + + if (i == MAX_TRIES - 1) { + if (extension_list != td::vector<size_t>{51, 43} && extension_list != td::vector<size_t>{43, 51}) { + LOG(ERROR) << url << ": TLS 1.3.0 unsupported extension list " << extension_list; + } else { + td::string length_distribution = "|"; + for (size_t t = 0; t < length_count.size(); t++) { + for (auto it : length_count[t]) { + length_distribution += PSTRING() + << it.first << " : " << static_cast<int>(it.second * 100.0 / MAX_TRIES) << "%|"; + } + if (t + 1 != length_count.size()) { + length_distribution += " + |"; + } + } + LOG(ERROR) << url << ": TLS 1.3.0 with extensions " << extension_list << " and " + << (length_count.size() != 1 ? "unsupported " : "") + << "encrypted application data length distribution " << length_distribution; + } + } + } + } + +#if TD_PORT_WINDOWS + iocp->interrupt_loop(); + iocp_thread.join(); +#endif +} diff --git a/protocols/Telegram/tdlib/td/benchmark/hashmap_build.cpp b/protocols/Telegram/tdlib/td/benchmark/hashmap_build.cpp new file mode 100644 index 0000000000..f6e71b748a --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/hashmap_build.cpp @@ -0,0 +1,545 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/FlatHashMap.h" + +#ifdef SCOPE_EXIT +#undef SCOPE_EXIT +#endif + +#include <absl/container/flat_hash_map.h> +#include <array> +#include <folly/container/F14Map.h> +#include <map> +#include <unordered_map> + +#define test_map td::FlatHashMap +//#define test_map folly::F14FastMap +//#define test_map absl::flat_hash_map +//#define test_map std::map +//#define test_map std::unordered_map + +//#define CREATE_MAP(num) CREATE_MAP_IMPL(num) +#define CREATE_MAP(num) + +#define CREATE_MAP_IMPL(num) \ + int f_##num() { \ + test_map<td::int32, std::array<char, num>> m; \ + m.emplace(1, std::array<char, num>{}); \ + int sum = 0; \ + for (auto &it : m) { \ + sum += it.first; \ + } \ + auto it = m.find(1); \ + sum += it->first; \ + m.erase(it); \ + return sum; \ + } \ + int x_##num = f_##num() + +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); + +int main() { +} diff --git a/protocols/Telegram/tdlib/td/benchmark/hashset_memory.cpp b/protocols/Telegram/tdlib/td/benchmark/hashset_memory.cpp new file mode 100644 index 0000000000..8f31b52114 --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/hashset_memory.cpp @@ -0,0 +1,193 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#if USE_MEMPROF +#include "memprof/memprof_stat.h" +#endif + +#include "td/utils/common.h" +#include "td/utils/FlatHashMap.h" +#include "td/utils/FlatHashMapChunks.h" +#include "td/utils/FlatHashTable.h" +#include "td/utils/HashTableUtils.h" +#include "td/utils/logging.h" +#include "td/utils/MapNode.h" +#include "td/utils/misc.h" +#include "td/utils/port/Stat.h" +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" + +#ifdef SCOPE_EXIT +#undef SCOPE_EXIT +#endif + +#include <absl/container/flat_hash_map.h> +#include <array> +#include <folly/container/F14Map.h> +#include <functional> +#include <map> +#include <unordered_map> + +static int mem_stat_i = -1; +static int mem_stat_cur = 0; + +static bool use_memprof() { +#if USE_MEMPROF + return mem_stat_i < 0 && is_memprof_on(); +#else + return mem_stat_i < 0; +#endif +} + +static td::uint64 get_memory() { +#if USE_MEMPROF + if (use_memprof()) { + return get_used_memory_size(); + } +#endif + CHECK(!use_memprof()); + return td::mem_stat().ok().resident_size_; +} + +template <class T> +class Generator { + public: + T next() { + UNREACHABLE(); + } + static size_t dyn_size() { + UNREACHABLE(); + } +}; + +template <class T> +class IntGenerator { + public: + T next() { + return ++value; + } + static size_t dyn_size() { + return 0; + } + + private: + T value{}; +}; + +template <> +class Generator<td::int32> final : public IntGenerator<td::int32> {}; +template <> +class Generator<td::int64> final : public IntGenerator<td::int64> {}; + +template <class T> +class Generator<td::unique_ptr<T>> { + public: + td::unique_ptr<T> next() { + return td::make_unique<T>(); + } + static std::size_t dyn_size() { + return sizeof(T); + } +}; + +template <class T, class KeyT, class ValueT> +static void measure(td::StringBuilder &sb, td::Slice name, td::Slice key_name, td::Slice value_name) { + mem_stat_cur++; + if (mem_stat_i >= 0 && mem_stat_cur != mem_stat_i) { + return; + } + sb << name << "<" << key_name << "," << value_name << "> " << (use_memprof() ? "memprof" : "os") << "\n"; + std::size_t ideal_size = sizeof(KeyT) + sizeof(ValueT) + Generator<ValueT>::dyn_size(); + + sb << "\tempty:" << sizeof(T); + struct Stat { + int pi; + double min_ratio; + double max_ratio; + }; + td::vector<Stat> stat; + stat.reserve(1024); + for (std::size_t size : {1000000u}) { + Generator<KeyT> key_generator; + Generator<ValueT> value_generator; + auto start_mem = get_memory(); + T ht; + auto ratio = [&] { + auto end_mem = get_memory(); + auto used_mem = end_mem - start_mem; + return static_cast<double>(used_mem) / (static_cast<double>(ideal_size) * static_cast<double>(ht.size())); + }; + double min_ratio; + double max_ratio; + auto reset = [&] { + min_ratio = 1e100; + max_ratio = 0; + }; + auto update = [&] { + auto x = ratio(); + min_ratio = td::min(min_ratio, x); + max_ratio = td::max(max_ratio, x); + }; + reset(); + + int p = 10; + int pi = 1; + for (std::size_t i = 0; i < size; i++) { + ht.emplace(key_generator.next(), value_generator.next()); + update(); + if ((i + 1) % p == 0) { + stat.push_back(Stat{pi, min_ratio, max_ratio}); + reset(); + pi++; + p *= 10; + } + } + } + for (auto &s : stat) { + sb << " 10^" << s.pi << ":" << s.min_ratio << "->" << s.max_ratio; + } + sb << '\n'; +} + +template <std::size_t size> +using Bytes = std::array<char, size>; + +template <template <typename... Args> class T> +void print_memory_stats(td::Slice name) { + td::string big_buff(1 << 16, '\0'); + td::StringBuilder sb(big_buff, false); +#define MEASURE(KeyT, ValueT) measure<T<KeyT, ValueT>, KeyT, ValueT>(sb, name, #KeyT, #ValueT); + MEASURE(td::int32, td::int32); + MEASURE(td::int64, td::unique_ptr<Bytes<360>>); + if (!sb.as_cslice().empty()) { + LOG(PLAIN) << '\n' << sb.as_cslice() << '\n'; + } +} + +template <class KeyT, class ValueT, class HashT = td::Hash<KeyT>, class EqT = std::equal_to<KeyT>> +using FlatHashMapImpl = td::FlatHashTable<td::MapNode<KeyT, ValueT>, HashT, EqT>; + +#define FOR_EACH_TABLE(F) \ + F(FlatHashMapImpl) \ + F(folly::F14FastMap) \ + F(absl::flat_hash_map) \ + F(std::unordered_map) \ + F(std::map) +#define BENCHMARK_MEMORY(T) print_memory_stats<T>(#T); + +int main(int argc, const char *argv[]) { + // Usage: + // % benchmark/memory-hashset-os 0 + // Number of benchmarks = 10 + // % for i in {1..10}; do ./benchmark/memory-hashset-os $i; done + if (argc > 1) { + mem_stat_i = td::to_integer<td::int32>(td::Slice(argv[1])); + } + FOR_EACH_TABLE(BENCHMARK_MEMORY); + if (mem_stat_i <= 0) { + LOG(PLAIN) << "Number of benchmarks = " << mem_stat_cur << "\n"; + } +} diff --git a/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp b/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp index f1676baa63..d61baa0a02 100644 --- a/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -14,15 +14,16 @@ int main(int argc, char *argv[]) { } td::CSlice dir(argv[1]); int cnt = 0; - auto status = td::walk_path(dir, [&](td::CSlice path, bool is_dir) { - cnt++; - LOG(INFO) << path << " " << is_dir; - // if (is_dir) { + auto status = td::walk_path(dir, [&](td::CSlice path, auto type) { + if (type != td::WalkPath::Type::EnterDir) { + cnt++; + LOG(INFO) << path << " " << (type == td::WalkPath::Type::ExitDir); + } + //if (is_dir) { // td::rmdir(path); //} else { // td::unlink(path); //} }); LOG(INFO) << status << ": " << cnt; - return 0; } diff --git a/protocols/Telegram/tdlib/td/benchmark/wget.cpp b/protocols/Telegram/tdlib/td/benchmark/wget.cpp index dba997e61f..5145f317f1 100644 --- a/protocols/Telegram/tdlib/td/benchmark/wget.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/wget.cpp @@ -1,39 +1,43 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/actor/actor.h" -#include "td/actor/PromiseFuture.h" - #include "td/net/HttpQuery.h" #include "td/net/Wget.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" + +#include "td/utils/common.h" #include "td/utils/logging.h" +#include "td/utils/Promise.h" #include "td/utils/Status.h" -#include <memory> -#include <string> - int main(int argc, char *argv[]) { - SET_VERBOSITY_LEVEL(VERBOSITY_NAME(INFO)); + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); td::VERBOSITY_NAME(fd) = VERBOSITY_NAME(INFO); - std::string url = (argc > 1 ? argv[1] : "https://telegram.org"); - auto scheduler = std::make_unique<td::ConcurrentScheduler>(); - scheduler->init(0); + td::string url = (argc > 1 ? argv[1] : "https://telegram.org"); + auto timeout = 10; + auto ttl = 3; + auto prefer_ipv6 = (argc > 2 && td::string(argv[2]) == "-6"); + auto scheduler = td::make_unique<td::ConcurrentScheduler>(0, 0); scheduler - ->create_actor_unsafe<td::Wget>(0, "Client", td::PromiseCreator::lambda([](td::Result<td::HttpQueryPtr> res) { + ->create_actor_unsafe<td::Wget>(0, "Client", + td::PromiseCreator::lambda([](td::Result<td::unique_ptr<td::HttpQuery>> res) { + if (res.is_error()) { + LOG(FATAL) << res.error(); + } LOG(ERROR) << *res.ok(); td::Scheduler::instance()->finish(); }), - url) + url, td::Auto(), timeout, ttl, prefer_ipv6) .release(); scheduler->start(); while (scheduler->run_main(10)) { // empty } scheduler->finish(); - return 0; } |