diff options
| author | George Hazan <ghazan@miranda.im> | 2022-11-30 17:48:47 +0300 | 
|---|---|---|
| committer | George Hazan <ghazan@miranda.im> | 2022-11-30 17:48:47 +0300 | 
| commit | 0ece30dc7c0e34b4c5911969b8fa99c33c6d023c (patch) | |
| tree | 671325d3fec09b999411e4e3ab84ef8259261818 /protocols/Telegram/tdlib/td/benchmark | |
| parent | 46c53ffc6809c67e4607e99951a2846c382b63b2 (diff) | |
Telegram: update for TDLIB
Diffstat (limited to 'protocols/Telegram/tdlib/td/benchmark')
21 files changed, 2743 insertions, 767 deletions
diff --git a/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt b/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt index 90f294fbc9..53fbdede52 100644 --- a/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt +++ b/protocols/Telegram/tdlib/td/benchmark/CMakeLists.txt @@ -1,4 +1,6 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +if ((CMAKE_MAJOR_VERSION LESS 3) OR (CMAKE_VERSION VERSION_LESS "3.0.2")) +  message(FATAL_ERROR "CMake >= 3.0.2 is required") +endif()  if (NOT OPENSSL_FOUND)    find_package(OpenSSL REQUIRED) @@ -7,7 +9,14 @@ endif()  #TODO: all benchmarks in one file  add_executable(bench_crypto bench_crypto.cpp) -target_link_libraries(bench_crypto PRIVATE tdcore tdutils ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) +target_link_libraries(bench_crypto PRIVATE tdutils ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) +if (WIN32) +  if (MINGW) +    target_link_libraries(bench_crypto PRIVATE ws2_32 mswsock crypt32) +  else() +    target_link_libraries(bench_crypto PRIVATE ws2_32 Mswsock Crypt32) +  endif() +endif()  target_include_directories(bench_crypto SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})  add_executable(bench_actor bench_actor.cpp) @@ -40,6 +49,12 @@ target_link_libraries(bench_tddb PRIVATE tdcore tddb tdutils)  add_executable(bench_misc bench_misc.cpp)  target_link_libraries(bench_misc PRIVATE tdcore tdutils) +add_executable(check_proxy check_proxy.cpp) +target_link_libraries(check_proxy PRIVATE tdclient tdutils) + +add_executable(check_tls check_tls.cpp) +target_link_libraries(check_tls PRIVATE tdutils) +  add_executable(rmdir rmdir.cpp)  target_link_libraries(rmdir PRIVATE tdutils) @@ -57,3 +72,22 @@ if (NOT WIN32 AND NOT CYGWIN)    add_executable(bench_queue bench_queue.cpp)    target_link_libraries(bench_queue PRIVATE tdutils)  endif() + +if (TD_TEST_FOLLY AND TD_WITH_ABSEIL) +  find_package(ABSL QUIET) +  find_package(folly QUIET) +  find_package(gflags QUIET) + +  if (ABSL_FOUND AND folly_FOUND) +    add_executable(memory-hashset-memprof EXCLUDE_FROM_ALL hashset_memory.cpp) +    target_compile_definitions(memory-hashset-memprof PRIVATE USE_MEMPROF=1) +    target_link_libraries(memory-hashset-memprof PRIVATE tdutils memprof_stat Folly::folly absl::flat_hash_map absl::hash) + +    add_executable(memory-hashset-os hashset_memory.cpp) +    target_compile_definitions(memory-hashset-os PRIVATE USE_MEMPROF=0) +    target_link_libraries(memory-hashset-os PRIVATE tdutils Folly::folly absl::flat_hash_map absl::hash) + +    add_executable(hashmap-build hashmap_build.cpp) +    target_link_libraries(hashmap-build PRIVATE tdutils Folly::folly absl::flat_hash_map absl::hash) +  endif() +endif() diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp index a966d601cf..afe94ca3c1 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_actor.cpp @@ -1,47 +1,101 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/utils/benchmark.h" -  #include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h"  #include "td/actor/PromiseFuture.h" +#include "td/utils/benchmark.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h"  #include "td/utils/logging.h" - -#include <algorithm> +#include "td/utils/Promise.h" +#include "td/utils/SliceBuilder.h"  #if TD_MSVC  #pragma comment(linker, "/STACK:16777216")  #endif +struct TestActor final : public td::Actor { +  static td::int32 actor_count_; + +  void start_up() final { +    actor_count_++; +    stop(); +  } + +  void tear_down() final { +    if (--actor_count_ == 0) { +      td::Scheduler::instance()->finish(); +    } +  } +}; + +td::int32 TestActor::actor_count_; + +namespace td { +template <> +class ActorTraits<TestActor> { + public: +  static constexpr bool need_context = false; +  static constexpr bool need_start_up = true; +}; +}  // namespace td + +class CreateActorBench final : public td::Benchmark { +  td::ConcurrentScheduler scheduler_{0, 0}; + +  void start_up() final { +    scheduler_.start(); +  } + +  void tear_down() final { +    scheduler_.finish(); +  } + + public: +  td::string get_description() const final { +    return "CreateActor"; +  } + +  void run(int n) final { +    for (int i = 0; i < n; i++) { +      scheduler_.create_actor_unsafe<TestActor>(0, "TestActor").release(); +    } +    while (scheduler_.run_main(10)) { +      // empty +    } +  } +}; +  template <int type> -class RingBench : public td::Benchmark { +class RingBench final : public td::Benchmark {   public:    struct PassActor;   private: -  int actor_n_; -  int thread_n_; -  std::vector<td::ActorId<PassActor>> actor_array_; -  td::ConcurrentScheduler *scheduler_; +  int actor_n_ = -1; +  int thread_n_ = -1; +  td::vector<td::ActorId<PassActor>> actor_array_; +  td::ConcurrentScheduler *scheduler_ = nullptr;   public: -  std::string get_description() const override { +  td::string get_description() const final {      static const char *types[] = {"later", "immediate", "raw", "tail", "lambda"};      static_assert(0 <= type && type < 5, "");      return PSTRING() << "Ring (send_" << types[type] << ") (threads_n = " << thread_n_ << ")";    } -  struct PassActor : public td::Actor { -    int id; +  struct PassActor final : public td::Actor { +    int id = -1;      td::ActorId<PassActor> next_actor;      int start_n = 0;      void pass(int n) { -      // LOG(INFO) << "pass: " << n; +      // LOG(INFO) << "Pass: " << n;        if (n == 0) {          td::Scheduler::instance()->finish();        } else { @@ -56,24 +110,23 @@ class RingBench : public td::Benchmark {              send_closure_later(next_actor, &PassActor::pass, n - 1);            } else {              // TODO: it is three times faster than send_event -            // may be send event could be further optimized? -            ::td::Scheduler::instance()->hack(static_cast<td::ActorId<Actor>>(next_actor), -                                              td::Event::raw(static_cast<td::uint32>(n - 1))); +            // maybe send event could be further optimized? +            next_actor.get_actor_unsafe()->raw_event(td::Event::raw(static_cast<td::uint32>(n - 1)).data);            }          } else if (type == 4) { -          send_lambda(next_actor, [=, ptr = next_actor.get_actor_unsafe()] { ptr->pass(n - 1); }); +          send_lambda(next_actor, [n, ptr = next_actor.get_actor_unsafe()] { ptr->pass(n - 1); });          }        }      } -    void raw_event(const td::Event::Raw &raw) override { +    void raw_event(const td::Event::Raw &raw) final {        pass(static_cast<int>(raw.u32));      } -    void start_up() override { +    void start_up() final {        yield();      } -    void wakeup() override { +    void wakeup() final {        if (start_n != 0) {          int n = start_n;          start_n = 0; @@ -85,11 +138,10 @@ class RingBench : public td::Benchmark {    RingBench(int actor_n, int thread_n) : actor_n_(actor_n), thread_n_(thread_n) {    } -  void start_up() override { -    scheduler_ = new td::ConcurrentScheduler(); -    scheduler_->init(thread_n_); +  void start_up() final { +    scheduler_ = new td::ConcurrentScheduler(thread_n_, 0); -    actor_array_ = std::vector<td::ActorId<PassActor>>(actor_n_); +    actor_array_ = td::vector<td::ActorId<PassActor>>(actor_n_);      for (int i = 0; i < actor_n_; i++) {        actor_array_[i] =            scheduler_->create_actor_unsafe<PassActor>(thread_n_ ? i % thread_n_ : 0, "PassActor").release(); @@ -101,30 +153,30 @@ class RingBench : public td::Benchmark {      scheduler_->start();    } -  void run(int n) override { +  void run(int n) final {      // first actor is on main_thread -    actor_array_[0].get_actor_unsafe()->start_n = std::max(n, 100); +    actor_array_[0].get_actor_unsafe()->start_n = td::max(n, 100);      while (scheduler_->run_main(10)) {        // empty      }    } -  void tear_down() override { +  void tear_down() final {      scheduler_->finish();      delete scheduler_;    }  };  template <int type> -class QueryBench : public td::Benchmark { +class QueryBench final : public td::Benchmark {   public: -  std::string get_description() const override { +  td::string get_description() const final {      static const char *types[] = {"callback", "immediate future", "delayed future", "dummy", "lambda", "lambda_future"};      static_assert(0 <= type && type < 6, "");      return PSTRING() << "QueryBench: " << types[type];    } -  class ClientActor : public td::Actor { +  class ClientActor final : public td::Actor {     public:      class Callback {       public: @@ -136,7 +188,7 @@ class QueryBench : public td::Benchmark {        virtual ~Callback() = default;        virtual void on_result(int x) = 0;      }; -    explicit ClientActor(std::unique_ptr<Callback> callback) : callback_(std::move(callback)) { +    explicit ClientActor(td::unique_ptr<Callback> callback) : callback_(std::move(callback)) {      }      void f(int x) {        callback_->on_result(x * x); @@ -152,23 +204,23 @@ class QueryBench : public td::Benchmark {      }     private: -    std::unique_ptr<Callback> callback_; +    td::unique_ptr<Callback> callback_;    }; -  class ServerActor : public td::Actor { +  class ServerActor final : public td::Actor {     public: -    class ClientCallback : public ClientActor::Callback { +    class ClientCallback final : public ClientActor::Callback {       public:        explicit ClientCallback(td::ActorId<ServerActor> server) : server_(server) {        } -      void on_result(int x) override { +      void on_result(int x) final {          send_closure(server_, &ServerActor::on_result, x);        }       private:        td::ActorId<ServerActor> server_;      }; -    void start_up() override { +    void start_up() final {        client_ = td::create_actor<ClientActor>("Client", td::make_unique<ClientCallback>(actor_id(this))).release();      } @@ -177,7 +229,7 @@ class QueryBench : public td::Benchmark {        wakeup();      } -    void wakeup() override { +    void wakeup() final {        while (true) {          if (n_ < 0) {            td::Scheduler::instance()->finish(); @@ -206,10 +258,12 @@ class QueryBench : public td::Benchmark {          } else if (type == 4) {            int val = 0;            send_lambda(client_, [&] { val = n_ * n_; }); +          CHECK(val == n_ * n_);          } else if (type == 5) {            send_closure(client_, &ClientActor::f_promise, -                       td::PromiseCreator::lambda( -                           [id = actor_id(this), n = n_](td::Unit) { send_closure(id, &ServerActor::result, n * n); })); +                       td::PromiseCreator::lambda([actor_id = actor_id(this), n = n_](td::Unit) { +                         send_closure(actor_id, &ServerActor::result, n * n); +                       }));            return;          }        } @@ -220,7 +274,7 @@ class QueryBench : public td::Benchmark {        wakeup();      } -    void raw_event(const td::Event::Raw &event) override { +    void raw_event(const td::Event::Raw &event) final {        int val = future_.move_as_ok();        CHECK(val == n_ * n_);        wakeup(); @@ -232,22 +286,21 @@ class QueryBench : public td::Benchmark {     private:      td::ActorId<ClientActor> client_; -    int n_; +    int n_ = 0;      td::FutureActor<int> future_;    }; -  void start_up() override { -    scheduler_ = new td::ConcurrentScheduler(); -    scheduler_->init(0); +  void start_up() final { +    scheduler_ = new td::ConcurrentScheduler(0, 0);      server_ = scheduler_->create_actor_unsafe<ServerActor>(0, "Server");      scheduler_->start();    } -  void run(int n) override { +  void run(int n) final {      // first actor is on main_thread      { -      auto guard = scheduler_->get_current_guard(); +      auto guard = scheduler_->get_main_guard();        send_closure(server_, &ServerActor::run, n);      }      while (scheduler_->run_main(10)) { @@ -255,19 +308,21 @@ class QueryBench : public td::Benchmark {      }    } -  void tear_down() override { +  void tear_down() final {      server_.release();      scheduler_->finish();      delete scheduler_;    }   private: -  td::ConcurrentScheduler *scheduler_; +  td::ConcurrentScheduler *scheduler_ = nullptr;    td::ActorOwn<ServerActor> server_;  };  int main() { -  SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); +  td::init_openssl_threads(); + +  bench(CreateActorBench());    bench(RingBench<4>(504, 0));    bench(RingBench<3>(504, 0));    bench(RingBench<0>(504, 0)); @@ -286,5 +341,4 @@ int main() {    bench(RingBench<0>(504, 2));    bench(RingBench<1>(504, 2));    bench(RingBench<2>(504, 2)); -  return 0;  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp index 44d309ef11..40d7602d2e 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_crypto.cpp @@ -1,75 +1,374 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  //  #include "td/utils/benchmark.h" +#include "td/utils/common.h"  #include "td/utils/crypto.h" -#include "td/utils/int_types.h" -#include "td/utils/logging.h"  #include "td/utils/port/thread.h"  #include "td/utils/Random.h"  #include "td/utils/Slice.h" +#include "td/utils/SliceBuilder.h" +#include "td/utils/UInt.h" +#include <openssl/evp.h>  #include <openssl/sha.h> +#include <algorithm>  #include <array>  #include <atomic>  #include <cstdint>  #include <cstdlib> +#include <iterator>  #include <random>  #include <string>  #include <vector>  static constexpr int DATA_SIZE = 8 << 10; +static constexpr int SHORT_DATA_SIZE = 64; -class SHA1Bench : public td::Benchmark { +#if OPENSSL_VERSION_NUMBER <= 0x10100000L +class SHA1Bench final : public td::Benchmark {   public:    alignas(64) unsigned char data[DATA_SIZE]; -  std::string get_description() const override { +  std::string get_description() const final {      return PSTRING() << "SHA1 OpenSSL [" << (DATA_SIZE >> 10) << "KB]";    } -  void start_up() override { -    for (int i = 0; i < DATA_SIZE; i++) { -      data[i] = 123; -      data[i] = 0; -    } +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123));    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        unsigned char md[20];        SHA1(data, DATA_SIZE, md);      }    }  }; +#endif + +class SHA1ShortBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[SHORT_DATA_SIZE]; + +  std::string get_description() const final { +    return PSTRING() << "SHA1 [" << SHORT_DATA_SIZE << "B]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +  } + +  void run(int n) final { +    unsigned char md[20]; +    for (int i = 0; i < n; i++) { +      td::sha1(td::Slice(data, SHORT_DATA_SIZE), md); +    } +  } +}; + +class SHA256ShortBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[SHORT_DATA_SIZE]; + +  std::string get_description() const final { +    return PSTRING() << "SHA256 [" << SHORT_DATA_SIZE << "B]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +  } + +  void run(int n) final { +    unsigned char md[32]; +    for (int i = 0; i < n; i++) { +      td::sha256(td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 32)); +    } +  } +}; + +class SHA512ShortBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[SHORT_DATA_SIZE]; + +  std::string get_description() const final { +    return PSTRING() << "SHA512 [" << SHORT_DATA_SIZE << "B]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +  } + +  void run(int n) final { +    unsigned char md[64]; +    for (int i = 0; i < n; i++) { +      td::sha512(td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 64)); +    } +  } +}; -class AESBench : public td::Benchmark { +class HmacSha256ShortBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[SHORT_DATA_SIZE]; + +  std::string get_description() const final { +    return PSTRING() << "HMAC-SHA256 [" << SHORT_DATA_SIZE << "B]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +  } + +  void run(int n) final { +    unsigned char md[32]; +    for (int i = 0; i < n; i++) { +      td::hmac_sha256(td::Slice(data, SHORT_DATA_SIZE), td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 32)); +    } +  } +}; + +class HmacSha512ShortBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[SHORT_DATA_SIZE]; + +  std::string get_description() const final { +    return PSTRING() << "HMAC-SHA512 [" << SHORT_DATA_SIZE << "B]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +  } + +  void run(int n) final { +    unsigned char md[32]; +    for (int i = 0; i < n; i++) { +      td::hmac_sha256(td::Slice(data, SHORT_DATA_SIZE), td::Slice(data, SHORT_DATA_SIZE), td::MutableSlice(md, 32)); +    } +  } +}; + +class AesEcbBench final : public td::Benchmark {   public:    alignas(64) unsigned char data[DATA_SIZE];    td::UInt256 key;    td::UInt256 iv; -  std::string get_description() const override { -    return PSTRING() << "AES OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; +  std::string get_description() const final { +    return PSTRING() << "AES ECB OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +    td::Random::secure_bytes(key.raw, sizeof(key)); +    td::Random::secure_bytes(iv.raw, sizeof(iv));    } -  void start_up() override { -    for (int i = 0; i < DATA_SIZE; i++) { -      data[i] = 123; +  void run(int n) final { +    td::AesState state; +    state.init(td::as_slice(key), true); +    td::MutableSlice data_slice(data, DATA_SIZE); +    for (int i = 0; i <= n; i++) { +      size_t step = 16; +      for (size_t offset = 0; offset + step <= data_slice.size(); offset += step) { +        state.encrypt(data_slice.ubegin() + offset, data_slice.ubegin() + offset, static_cast<int>(step)); +      }      } +  } +}; + +class AesIgeEncryptBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[DATA_SIZE]; +  td::UInt256 key; +  td::UInt256 iv; + +  std::string get_description() const final { +    return PSTRING() << "AES IGE OpenSSL encrypt [" << (DATA_SIZE >> 10) << "KB]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123));      td::Random::secure_bytes(key.raw, sizeof(key));      td::Random::secure_bytes(iv.raw, sizeof(iv));    } -  void run(int n) override { +  void run(int n) final {      td::MutableSlice data_slice(data, DATA_SIZE); +    td::AesIgeState state; +    state.init(as_slice(key), as_slice(iv), true);      for (int i = 0; i < n; i++) { -      td::aes_ige_encrypt(key, &iv, data_slice, data_slice); +      state.encrypt(data_slice, data_slice); +    } +  } +}; + +class AesIgeDecryptBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[DATA_SIZE]; +  td::UInt256 key; +  td::UInt256 iv; + +  std::string get_description() const final { +    return PSTRING() << "AES IGE OpenSSL decrypt [" << (DATA_SIZE >> 10) << "KB]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +    td::Random::secure_bytes(key.raw, sizeof(key)); +    td::Random::secure_bytes(iv.raw, sizeof(iv)); +  } + +  void run(int n) final { +    td::MutableSlice data_slice(data, DATA_SIZE); +    td::AesIgeState state; +    state.init(as_slice(key), as_slice(iv), false); +    for (int i = 0; i < n; i++) { +      state.decrypt(data_slice, data_slice); +    } +  } +}; + +class AesCtrBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[DATA_SIZE]; +  td::UInt256 key; +  td::UInt128 iv; + +  std::string get_description() const final { +    return PSTRING() << "AES CTR OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +    td::Random::secure_bytes(key.raw, sizeof(key)); +    td::Random::secure_bytes(iv.raw, sizeof(iv)); +  } + +  void run(int n) final { +    td::MutableSlice data_slice(data, DATA_SIZE); +    td::AesCtrState state; +    state.init(as_slice(key), as_slice(iv)); +    for (int i = 0; i < n; i++) { +      state.encrypt(data_slice, data_slice); +    } +  } +}; + +#if OPENSSL_VERSION_NUMBER >= 0x10100000L +class AesCtrOpenSSLBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[DATA_SIZE]; +  td::UInt256 key; +  td::UInt128 iv; + +  std::string get_description() const final { +    return PSTRING() << "AES CTR RAW OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +    td::Random::secure_bytes(key.raw, sizeof(key)); +    td::Random::secure_bytes(iv.raw, sizeof(iv)); +  } + +  void run(int n) final { +    EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); +    EVP_EncryptInit_ex(ctx, EVP_aes_256_ctr(), nullptr, key.raw, iv.raw); + +    td::MutableSlice data_slice(data, DATA_SIZE); +    td::AesCtrState state; +    state.init(as_slice(key), as_slice(iv)); +    for (int i = 0; i < n; i++) { +      int len = 0; +      EVP_EncryptUpdate(ctx, data_slice.ubegin(), &len, data_slice.ubegin(), DATA_SIZE); +      CHECK(len == DATA_SIZE); +    } + +    EVP_CIPHER_CTX_free(ctx); +  } +}; +#endif + +class AesCbcDecryptBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[DATA_SIZE]; +  td::UInt256 key; +  td::UInt128 iv; + +  std::string get_description() const final { +    return PSTRING() << "AES CBC Decrypt OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +    td::Random::secure_bytes(as_slice(key)); +    td::Random::secure_bytes(as_slice(iv)); +  } + +  void run(int n) final { +    td::MutableSlice data_slice(data, DATA_SIZE); +    for (int i = 0; i < n; i++) { +      td::aes_cbc_decrypt(as_slice(key), as_slice(iv), data_slice, data_slice); +    } +  } +}; + +class AesCbcEncryptBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[DATA_SIZE]; +  td::UInt256 key; +  td::UInt128 iv; + +  std::string get_description() const final { +    return PSTRING() << "AES CBC Encrypt OpenSSL [" << (DATA_SIZE >> 10) << "KB]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +    td::Random::secure_bytes(as_slice(key)); +    td::Random::secure_bytes(as_slice(iv)); +  } + +  void run(int n) final { +    td::MutableSlice data_slice(data, DATA_SIZE); +    for (int i = 0; i < n; i++) { +      td::aes_cbc_encrypt(as_slice(key), as_slice(iv), data_slice, data_slice); +    } +  } +}; + +template <bool use_state> +class AesIgeShortBench final : public td::Benchmark { + public: +  alignas(64) unsigned char data[SHORT_DATA_SIZE]; +  td::UInt256 key; +  td::UInt256 iv; + +  std::string get_description() const final { +    return PSTRING() << "AES IGE OpenSSL " << (use_state ? "EVP" : "C  ") << "[" << SHORT_DATA_SIZE << "B]"; +  } + +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123)); +    td::Random::secure_bytes(as_slice(key)); +    td::Random::secure_bytes(as_slice(iv)); +  } + +  void run(int n) final { +    td::MutableSlice data_slice(data, SHORT_DATA_SIZE); +    for (int i = 0; i < n; i++) { +      if (use_state) { +        td::AesIgeState ige; +        ige.init(as_slice(key), as_slice(iv), false); +        ige.decrypt(data_slice, data_slice); +      } else { +        td::aes_ige_decrypt(as_slice(key), as_slice(iv), data_slice, data_slice); +      }      }    }  }; @@ -110,15 +409,15 @@ BENCH(TdRandFast, "td_rand_fast") {  #if !TD_THREAD_UNSUPPORTED  BENCH(SslRand, "ssl_rand_int32") {    std::vector<td::thread> v; -  std::atomic<td::uint32> sum; +  std::atomic<td::uint32> sum{0};    for (int i = 0; i < 3; i++) { -    v.push_back(td::thread([&] { +    v.emplace_back([&sum, n] {        td::int32 res = 0;        for (int j = 0; j < n; j++) {          res ^= td::Random::secure_int32();        }        sum += res; -    })); +    });    }    for (auto &x : v) {      x.join(); @@ -147,22 +446,19 @@ BENCH(Pbkdf2, "pbkdf2") {    td::pbkdf2_sha256(password, salt, n, key);  } -class Crc32Bench : public td::Benchmark { +class Crc32Bench final : public td::Benchmark {   public:    alignas(64) unsigned char data[DATA_SIZE]; -  std::string get_description() const override { +  std::string get_description() const final {      return PSTRING() << "Crc32 zlib [" << (DATA_SIZE >> 10) << "KB]";    } -  void start_up() override { -    for (int i = 0; i < DATA_SIZE; i++) { -      data[i] = 123; -      data[i] = 0; -    } +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123));    } -  void run(int n) override { +  void run(int n) final {      td::uint64 res = 0;      for (int i = 0; i < n; i++) {        res += td::crc32(td::Slice(data, DATA_SIZE)); @@ -171,22 +467,19 @@ class Crc32Bench : public td::Benchmark {    }  }; -class Crc64Bench : public td::Benchmark { +class Crc64Bench final : public td::Benchmark {   public:    alignas(64) unsigned char data[DATA_SIZE]; -  std::string get_description() const override { +  std::string get_description() const final {      return PSTRING() << "Crc64 Anton [" << (DATA_SIZE >> 10) << "KB]";    } -  void start_up() override { -    for (int i = 0; i < DATA_SIZE; i++) { -      data[i] = 123; -      data[i] = 0; -    } +  void start_up() final { +    std::fill(std::begin(data), std::end(data), static_cast<unsigned char>(123));    } -  void run(int n) override { +  void run(int n) final {      td::uint64 res = 0;      for (int i = 0; i < n; i++) {        res += td::crc64(td::Slice(data, DATA_SIZE)); @@ -196,6 +489,20 @@ class Crc64Bench : public td::Benchmark {  };  int main() { +  td::init_openssl_threads(); +  td::bench(AesCtrBench()); +#if OPENSSL_VERSION_NUMBER >= 0x10100000L +  td::bench(AesCtrOpenSSLBench()); +#endif + +  td::bench(AesCbcDecryptBench()); +  td::bench(AesCbcEncryptBench()); +  td::bench(AesIgeShortBench<true>()); +  td::bench(AesIgeShortBench<false>()); +  td::bench(AesIgeEncryptBench()); +  td::bench(AesIgeDecryptBench()); +  td::bench(AesEcbBench()); +    td::bench(Pbkdf2Bench());    td::bench(RandBench());    td::bench(CppRandBench()); @@ -205,9 +512,14 @@ int main() {    td::bench(SslRandBench());  #endif    td::bench(SslRandBufBench()); +#if OPENSSL_VERSION_NUMBER <= 0x10100000L    td::bench(SHA1Bench()); -  td::bench(AESBench()); +#endif +  td::bench(SHA1ShortBench()); +  td::bench(SHA256ShortBench()); +  td::bench(SHA512ShortBench()); +  td::bench(HmacSha256ShortBench()); +  td::bench(HmacSha512ShortBench());    td::bench(Crc32Bench());    td::bench(Crc64Bench()); -  return 0;  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp index dc768e9d9d..822a915938 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_db.cpp @@ -1,28 +1,35 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/actor/actor.h" -  #include "td/db/binlog/Binlog.h" +#include "td/db/binlog/ConcurrentBinlog.h"  #include "td/db/BinlogKeyValue.h" +#include "td/db/DbKey.h"  #include "td/db/SeqKeyValue.h" +#include "td/db/SqliteConnectionSafe.h"  #include "td/db/SqliteDb.h"  #include "td/db/SqliteKeyValueAsync.h" +#include "td/db/SqliteKeyValueSafe.h" + +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h"  #include "td/utils/benchmark.h" +#include "td/utils/common.h"  #include "td/utils/format.h"  #include "td/utils/logging.h" +#include "td/utils/SliceBuilder.h"  #include "td/utils/Status.h"  #include "td/utils/StringBuilder.h"  #include <memory>  template <class KeyValueT> -class TdKvBench : public td::Benchmark { -  td::ConcurrentScheduler sched; +class TdKvBench final : public td::Benchmark { +  td::ConcurrentScheduler sched{1, 0};    td::string name_;   public: @@ -30,27 +37,27 @@ class TdKvBench : public td::Benchmark {      name_ = std::move(name);    } -  td::string get_description() const override { +  td::string get_description() const final {      return name_;    } -  class Main : public td::Actor { +  class Main final : public td::Actor {     public:      explicit Main(int n) : n_(n) {      }     private: -    void loop() override { +    void loop() final {        KeyValueT::destroy("test_tddb").ignore(); -      class Worker : public Actor { +      class Worker final : public Actor {         public:          Worker(int n, td::string db_name) : n_(n) {            kv_.init(db_name).ensure();          }         private: -        void loop() override { +        void loop() final {            for (int i = 0; i < n_; i++) {              kv_.set(td::to_string(i % 10), td::to_string(i));            } @@ -64,12 +71,11 @@ class TdKvBench : public td::Benchmark {      int n_;    }; -  void start_up_n(int n) override { -    sched.init(1); +  void start_up_n(int n) final {      sched.create_actor_unsafe<Main>(1, "Main", n).release();    } -  void run(int n) override { +  void run(int n) final {      sched.start();      while (sched.run_main(10)) {        // empty @@ -77,24 +83,23 @@ class TdKvBench : public td::Benchmark {      sched.finish();    } -  void tear_down() override { +  void tear_down() final {    }  };  template <bool is_encrypted = false> -class SqliteKVBench : public td::Benchmark { +class SqliteKVBench final : public td::Benchmark {    td::SqliteDb db; -  td::string get_description() const override { +  td::string get_description() const final {      return PSTRING() << "SqliteKV " << td::tag("is_encrypted", is_encrypted);    } -  void start_up() override { +  void start_up() final {      td::string path = "testdb.sqlite";      td::SqliteDb::destroy(path).ignore();      if (is_encrypted) { -      td::SqliteDb::change_key(path, td::DbKey::password("cucumber"), td::DbKey::empty()); -      db = td::SqliteDb::open_with_key(path, td::DbKey::password("cucumber")).move_as_ok(); +      db = td::SqliteDb::change_key(path, true, td::DbKey::password("cucumber"), td::DbKey::empty()).move_as_ok();      } else { -      db = td::SqliteDb::open_with_key(path, td::DbKey::empty()).move_as_ok(); +      db = td::SqliteDb::open_with_key(path, true, td::DbKey::empty()).move_as_ok();      }      db.exec("PRAGMA encoding=\"UTF-8\"").ensure();      db.exec("PRAGMA synchronous=NORMAL").ensure(); @@ -103,7 +108,7 @@ class SqliteKVBench : public td::Benchmark {      db.exec("DROP TABLE IF EXISTS KV").ensure();      db.exec("CREATE TABLE IF NOT EXISTS KV (k BLOB PRIMARY KEY, v BLOB)").ensure();    } -  void run(int n) override { +  void run(int n) final {      auto stmt = db.get_statement("REPLACE INTO KV (k, v) VALUES(?1, ?2)").move_as_ok();      db.exec("BEGIN TRANSACTION").ensure();      for (int i = 0; i < n; i++) { @@ -135,17 +140,17 @@ static td::Status init_db(td::SqliteDb &db) {    return td::Status::OK();  } -class SqliteKeyValueAsyncBench : public td::Benchmark { +class SqliteKeyValueAsyncBench final : public td::Benchmark {   public: -  td::string get_description() const override { +  td::string get_description() const final {      return "SqliteKeyValueAsync";    } -  void start_up() override { +  void start_up() final {      do_start_up().ensure();      scheduler_->start();    } -  void run(int n) override { -    auto guard = scheduler_->get_current_guard(); +  void run(int n) final { +    auto guard = scheduler_->get_main_guard();      for (int i = 0; i < n; i++) {        auto key = td::to_string(i % 10); @@ -153,10 +158,10 @@ class SqliteKeyValueAsyncBench : public td::Benchmark {        sqlite_kv_async_->set(key, value, td::Auto());      }    } -  void tear_down() override { +  void tear_down() final {      scheduler_->run_main(0.1);      { -      auto guard = scheduler_->get_current_guard(); +      auto guard = scheduler_->get_main_guard();        sqlite_kv_async_.reset();        sqlite_kv_safe_.reset();        sql_connection_->close_and_destroy(); @@ -167,21 +172,20 @@ class SqliteKeyValueAsyncBench : public td::Benchmark {    }   private: -  std::unique_ptr<td::ConcurrentScheduler> scheduler_; +  td::unique_ptr<td::ConcurrentScheduler> scheduler_;    std::shared_ptr<td::SqliteConnectionSafe> sql_connection_;    std::shared_ptr<td::SqliteKeyValueSafe> sqlite_kv_safe_; -  std::unique_ptr<td::SqliteKeyValueAsyncInterface> sqlite_kv_async_; +  td::unique_ptr<td::SqliteKeyValueAsyncInterface> sqlite_kv_async_;    td::Status do_start_up() { -    scheduler_ = std::make_unique<td::ConcurrentScheduler>(); -    scheduler_->init(1); +    scheduler_ = td::make_unique<td::ConcurrentScheduler>(1, 0); -    auto guard = scheduler_->get_current_guard(); +    auto guard = scheduler_->get_main_guard();      td::string sql_db_name = "testdb.sqlite";      td::SqliteDb::destroy(sql_db_name).ignore(); -    sql_connection_ = std::make_shared<td::SqliteConnectionSafe>(sql_db_name); +    sql_connection_ = std::make_shared<td::SqliteConnectionSafe>(sql_db_name, td::DbKey::empty());      auto &db = sql_connection_->get();      TRY_STATUS(init_db(db)); @@ -192,31 +196,31 @@ class SqliteKeyValueAsyncBench : public td::Benchmark {    }  }; -class SeqKvBench : public td::Benchmark { -  td::string get_description() const override { +class SeqKvBench final : public td::Benchmark { +  td::string get_description() const final {      return "SeqKvBench";    }    td::SeqKeyValue kv; -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) { -      kv.set(td::to_string(i % 10), td::to_string(i)); +      kv.set(PSLICE() << i % 10, PSLICE() << i);      }    }  };  template <bool is_encrypted = false> -class BinlogKeyValueBench : public td::Benchmark { -  td::string get_description() const override { +class BinlogKeyValueBench final : public td::Benchmark { +  td::string get_description() const final {      return PSTRING() << "BinlogKeyValue " << td::tag("is_encrypted", is_encrypted);    }    td::BinlogKeyValue<td::Binlog> kv; -  void start_up() override { +  void start_up() final {      td::SqliteDb::destroy("test_binlog").ignore();      kv.init("test_binlog", is_encrypted ? td::DbKey::password("cucumber") : td::DbKey::empty()).ensure();    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        kv.set(td::to_string(i % 10), td::to_string(i));      } @@ -233,5 +237,4 @@ int main() {    bench(TdKvBench<td::BinlogKeyValue<td::Binlog>>("BinlogKeyValue<Binlog>"));    bench(TdKvBench<td::BinlogKeyValue<td::ConcurrentBinlog>>("BinlogKeyValue<ConcurrentBinlog>"));    bench(SeqKvBench()); -  return 0;  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp index f6718152b4..ae2068c050 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_empty.cpp @@ -1,9 +1,8 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  //  int main() { -  return 0;  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp index 08d04f009c..f518fe03b0 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_handshake.cpp @@ -1,15 +1,15 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/utils/benchmark.h"  // for bench, do_not_optimize_away, etc - -#include "td/mtproto/crypto.h" +#include "td/mtproto/DhCallback.h" +#include "td/mtproto/DhHandshake.h"  #include "td/utils/base64.h" -#include "td/utils/logging.h" +#include "td/utils/benchmark.h" +#include "td/utils/common.h"  #include "td/utils/Slice.h"  #include <map> @@ -18,59 +18,55 @@  #include <semaphore.h>  #endif -namespace td { - -static int32 g = 3; -static string prime_base64 = +static td::int32 g = 3; +static td::string prime_base64 =      "xxyuucaxyQSObFIvcPE_c5gNQCOOPiHBSTTQN1Y9kw9IGYoKp8FAWCKUk9IlMPTb-jNvbgrJJROVQ67UTM58NyD9UfaUWHBaxozU_mtrE6vcl0ZRKW"      "kyhFTxj6-MWV9kJHf-lrsqlB1bzR1KyMxJiAcI-ps3jjxPOpBgvuZ8-aSkppWBEFGQfhYnU7VrD2tBDbp02KhLKhSzFE4O8ShHVP0X7ZUNWWW0ud1G"      "WC2xF40WnGvEZbDW_5yjko_vW5rk5Bj8Feg-vqD4f6n_Xu1wBQ3tKEn0e_lZ2VaFDOkphR8NgRX2NbEF7i5OFdBLJFS_b0-t8DSxBAMRnNjjuS_MW"      "w"; -class HandshakeBench : public Benchmark { -  std::string get_description() const override { +class HandshakeBench final : public td::Benchmark { +  td::string get_description() const final {      return "Handshake";    } -  class FakeDhCallback : public DhCallback { +  class FakeDhCallback final : public td::mtproto::DhCallback {     public: -    int is_good_prime(Slice prime_str) const override { +    int is_good_prime(td::Slice prime_str) const final {        auto it = cache.find(prime_str.str());        if (it == cache.end()) {          return -1;        }        return it->second;      } -    void add_good_prime(Slice prime_str) const override { +    void add_good_prime(td::Slice prime_str) const final {        cache[prime_str.str()] = 1;      } -    void add_bad_prime(Slice prime_str) const override { +    void add_bad_prime(td::Slice prime_str) const final {        cache[prime_str.str()] = 0;      } -    mutable std::map<string, int> cache; +    mutable std::map<td::string, int> cache;    } dh_callback; -  void run(int n) override { -    DhHandshake a; -    DhHandshake b; -    auto prime = base64url_decode(prime_base64).move_as_ok(); +  void run(int n) final { +    td::mtproto::DhHandshake a; +    td::mtproto::DhHandshake b; +    auto prime = td::base64url_decode(prime_base64).move_as_ok(); +    td::mtproto::DhHandshake::check_config(g, prime, &dh_callback).ensure();      for (int i = 0; i < n; i += 2) {        a.set_config(g, prime);        b.set_config(g, prime);        b.set_g_a(a.get_g_b());        a.set_g_a(b.get_g_b()); -      a.run_checks(&dh_callback).ensure(); -      b.run_checks(&dh_callback).ensure(); +      a.run_checks(true, &dh_callback).ensure(); +      b.run_checks(true, &dh_callback).ensure();        auto a_key = a.gen_key();        auto b_key = b.gen_key();        CHECK(a_key.first == b_key.first);      }    }  }; -}  // namespace td  int main() { -  SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); -  td::bench(td::HandshakeBench()); -  return 0; +  td::bench(HandshakeBench());  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp index 6958a5b313..70fdb909c2 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http.cpp @@ -1,15 +1,18 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/actor/actor.h" -  #include "td/net/HttpOutboundConnection.h"  #include "td/net/HttpQuery.h" +#include "td/net/SslStream.h" + +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h"  #include "td/utils/buffer.h" +#include "td/utils/BufferedFd.h"  #include "td/utils/logging.h"  #include "td/utils/port/IPAddress.h"  #include "td/utils/port/SocketFd.h" @@ -18,50 +21,52 @@  #include <atomic>  #include <limits> -namespace td { -  std::atomic<int> counter; -class HttpClient : public HttpOutboundConnection::Callback { -  void start_up() override { -    IPAddress addr; + +class HttpClient final : public td::HttpOutboundConnection::Callback { +  void start_up() final { +    td::IPAddress addr;      addr.init_ipv4_port("127.0.0.1", 8082).ensure(); -    auto fd = SocketFd::open(addr); -    CHECK(fd.is_ok()) << fd.error(); -    connection_ = -        create_actor<HttpOutboundConnection>("Connect", fd.move_as_ok(), std::numeric_limits<size_t>::max(), 0, 0, -                                             ActorOwn<HttpOutboundConnection::Callback>(actor_id(this))); +    auto fd = td::SocketFd::open(addr); +    LOG_CHECK(fd.is_ok()) << fd.error(); +    connection_ = td::create_actor<td::HttpOutboundConnection>( +        "Connect", td::BufferedFd<td::SocketFd>(fd.move_as_ok()), td::SslStream{}, std::numeric_limits<size_t>::max(), +        0, 0, td::ActorOwn<td::HttpOutboundConnection::Callback>(actor_id(this)));      yield();      cnt_ = 100000;      counter++;    } -  void tear_down() override { + +  void tear_down() final {      if (--counter == 0) { -      Scheduler::instance()->finish(); +      td::Scheduler::instance()->finish();      }    } -  void loop() override { + +  void loop() final {      if (cnt_-- < 0) {        return stop();      } -    send_closure(connection_, &HttpOutboundConnection::write_next, BufferSlice("GET / HTTP/1.1\r\n\r\n")); -    send_closure(connection_, &HttpOutboundConnection::write_ok); +    send_closure(connection_, &td::HttpOutboundConnection::write_next, td::BufferSlice("GET / HTTP/1.1\r\n\r\n")); +    send_closure(connection_, &td::HttpOutboundConnection::write_ok);      LOG(INFO) << "SEND";    } -  void handle(HttpQueryPtr result) override { + +  void handle(td::unique_ptr<td::HttpQuery> result) final {      loop();    } -  void on_connection_error(Status error) override { + +  void on_connection_error(td::Status error) final {      LOG(ERROR) << "ERROR: " << error;    } -  ActorOwn<HttpOutboundConnection> connection_; -  int cnt_; +  td::ActorOwn<td::HttpOutboundConnection> connection_; +  int cnt_ = 0;  };  int main() {    SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); -  auto scheduler = make_unique<ConcurrentScheduler>(); -  scheduler->init(0); +  auto scheduler = td::make_unique<td::ConcurrentScheduler>(0, 0);    scheduler->create_actor_unsafe<HttpClient>(0, "Client1").release();    scheduler->create_actor_unsafe<HttpClient>(0, "Client2").release();    scheduler->start(); @@ -69,10 +74,4 @@ int main() {      // empty    }    scheduler->finish(); -  return 0; -} -}  // namespace td - -int main() { -  return td::main();  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp index 2afe2d73ff..403af06bec 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_reader.cpp @@ -1,5 +1,5 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -9,19 +9,20 @@  #include "td/utils/benchmark.h"  #include "td/utils/buffer.h" +#include "td/utils/common.h"  #include "td/utils/find_boundary.h"  #include "td/utils/logging.h"  static std::string http_query = "GET / HTTP/1.1\r\nConnection:keep-alive\r\nhost:127.0.0.1:8080\r\n\r\n";  static const size_t block_size = 2500; -class HttpReaderBench : public td::Benchmark { -  std::string get_description() const override { +class HttpReaderBench final : public td::Benchmark { +  std::string get_description() const final {      return "HttpReaderBench";    } -  void run(int n) override { -    int cnt = static_cast<int>(block_size / http_query.size()); +  void run(int n) final { +    auto cnt = static_cast<int>(block_size / http_query.size());      td::HttpQuery q;      int parsed = 0;      int sent = 0; @@ -45,27 +46,26 @@ class HttpReaderBench : public td::Benchmark {    td::ChainBufferReader reader_;    td::HttpReader http_reader_; -  void start_up() override { -    writer_ = td::ChainBufferWriter::create_empty(); +  void start_up() final {      reader_ = writer_.extract_reader();      http_reader_.init(&reader_, 10000, 0);    }  }; -class BufferBench : public td::Benchmark { -  std::string get_description() const override { +class BufferBench final : public td::Benchmark { +  std::string get_description() const final {      return "BufferBench";    } -  void run(int n) override { -    int cnt = static_cast<int>(block_size / http_query.size()); +  void run(int n) final { +    auto cnt = static_cast<int>(block_size / http_query.size());      for (int i = 0; i < n; i += cnt) {        for (int j = 0; j < cnt; j++) {          writer_.append(http_query);        }        reader_.sync_with_writer();        for (int j = 0; j < cnt; j++) { -        reader_.cut_head(http_query.size()); +        auto result = reader_.cut_head(http_query.size());        }      }    } @@ -73,19 +73,18 @@ class BufferBench : public td::Benchmark {    td::ChainBufferReader reader_;    td::HttpReader http_reader_; -  void start_up() override { -    writer_ = td::ChainBufferWriter::create_empty(); +  void start_up() final {      reader_ = writer_.extract_reader();    }  }; -class FindBoundaryBench : public td::Benchmark { -  std::string get_description() const override { +class FindBoundaryBench final : public td::Benchmark { +  std::string get_description() const final {      return "FindBoundaryBench";    } -  void run(int n) override { -    int cnt = static_cast<int>(block_size / http_query.size()); +  void run(int n) final { +    auto cnt = static_cast<int>(block_size / http_query.size());      for (int i = 0; i < n; i += cnt) {        for (int j = 0; j < cnt; j++) {          writer_.append(http_query); @@ -95,7 +94,7 @@ class FindBoundaryBench : public td::Benchmark {          size_t len = 0;          find_boundary(reader_.clone(), "\r\n\r\n", len);          CHECK(size_t(len) + 4 == http_query.size()); -        reader_.cut_head(len + 2); +        auto result = reader_.cut_head(len + 2);          reader_.advance(2);        }      } @@ -104,8 +103,7 @@ class FindBoundaryBench : public td::Benchmark {    td::ChainBufferReader reader_;    td::HttpReader http_reader_; -  void start_up() override { -    writer_ = td::ChainBufferWriter::create_empty(); +  void start_up() final {      reader_ = writer_.extract_reader();    }  }; diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp index c48e8b4a67..33cde9ef8c 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_server.cpp @@ -1,31 +1,32 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/actor/actor.h" -  #include "td/net/HttpHeaderCreator.h"  #include "td/net/HttpInboundConnection.h"  #include "td/net/HttpQuery.h"  #include "td/net/TcpListener.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" +  #include "td/utils/buffer.h" +#include "td/utils/BufferedFd.h"  #include "td/utils/logging.h"  #include "td/utils/port/SocketFd.h"  #include "td/utils/Slice.h" -namespace td { -  static int cnt = 0; -class HelloWorld : public HttpInboundConnection::Callback { + +class HelloWorld final : public td::HttpInboundConnection::Callback {   public: -  void handle(HttpQueryPtr query, ActorOwn<HttpInboundConnection> connection) override { +  void handle(td::unique_ptr<td::HttpQuery> query, td::ActorOwn<td::HttpInboundConnection> connection) final {      // LOG(ERROR) << *query; -    HttpHeaderCreator hc; -    Slice content = "hello world"; -    //auto content = BufferSlice("hello world"); +    td::HttpHeaderCreator hc; +    td::Slice content = "hello world"; +    //auto content = td::BufferSlice("hello world");      hc.init_ok();      hc.set_keep_alive();      hc.set_content_size(content.size()); @@ -35,55 +36,49 @@ class HelloWorld : public HttpInboundConnection::Callback {      auto res = hc.finish(content);      LOG_IF(FATAL, res.is_error()) << res.error(); -    send_closure(connection, &HttpInboundConnection::write_next, BufferSlice(res.ok())); -    send_closure(connection.release(), &HttpInboundConnection::write_ok); +    send_closure(connection, &td::HttpInboundConnection::write_next, td::BufferSlice(res.ok())); +    send_closure(connection.release(), &td::HttpInboundConnection::write_ok);    } -  void hangup() override { +  void hangup() final {      LOG(ERROR) << "CLOSE " << cnt--;      stop();    }  };  const int N = 0; -class Server : public TcpListener::Callback { +class Server final : public td::TcpListener::Callback {   public: -  void start_up() override { -    listener_ = create_actor<TcpListener>("Listener", 8082, ActorOwn<TcpListener::Callback>(actor_id(this))); +  void start_up() final { +    listener_ = +        td::create_actor<td::TcpListener>("Listener", 8082, td::ActorOwn<td::TcpListener::Callback>(actor_id(this)));    } -  void accept(SocketFd fd) override { +  void accept(td::SocketFd fd) final {      LOG(ERROR) << "ACCEPT " << cnt++;      pos_++;      auto scheduler_id = pos_ % (N != 0 ? N : 1) + (N != 0); -    create_actor_on_scheduler<HttpInboundConnection>("HttpInboundConnection", scheduler_id, std::move(fd), 1024 * 1024, -                                                     0, 0, -                                                     create_actor_on_scheduler<HelloWorld>("HelloWorld", scheduler_id)) +    td::create_actor_on_scheduler<td::HttpInboundConnection>( +        "HttpInboundConnection", scheduler_id, td::BufferedFd<td::SocketFd>(std::move(fd)), 1024 * 1024, 0, 0, +        td::create_actor_on_scheduler<HelloWorld>("HelloWorld", scheduler_id))          .release();    } -  void hangup() override { +  void hangup() final {      // may be it should be default?.. -    LOG(ERROR) << "hangup.."; +    LOG(ERROR) << "Hanging up..";      stop();    }   private: -  ActorOwn<TcpListener> listener_; +  td::ActorOwn<td::TcpListener> listener_;    int pos_{0};  };  int main() {    SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); -  auto scheduler = make_unique<ConcurrentScheduler>(); -  scheduler->init(N); +  auto scheduler = td::make_unique<td::ConcurrentScheduler>(N, 0);    scheduler->create_actor_unsafe<Server>(0, "Server").release();    scheduler->start();    while (scheduler->run_main(10)) {      // empty    }    scheduler->finish(); -  return 0; -} -}  // namespace td - -int main() { -  return td::main();  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp index da6fbbd713..8bbd768b40 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_cheat.cpp @@ -1,49 +1,46 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/actor/actor.h" -  #include "td/net/HttpHeaderCreator.h" -#include "td/net/HttpInboundConnection.h"  #include "td/net/TcpListener.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" +  #include "td/utils/buffer.h"  #include "td/utils/logging.h" -#include "td/utils/port/Fd.h" +#include "td/utils/port/detail/PollableFd.h"  #include "td/utils/port/SocketFd.h"  #include "td/utils/Slice.h"  #include "td/utils/Status.h"  #include <array> -namespace td { - -// HttpInboundConnection header  static int cnt = 0; -class HelloWorld : public Actor { + +class HelloWorld final : public td::Actor {   public: -  explicit HelloWorld(SocketFd socket_fd) : socket_fd_(std::move(socket_fd)) { +  explicit HelloWorld(td::SocketFd socket_fd) : socket_fd_(std::move(socket_fd)) {    }   private: -  SocketFd socket_fd_; +  td::SocketFd socket_fd_;    std::array<char, 1024> read_buf;    size_t read_new_lines{0}; -  std::string hello_; -  std::string write_buf_; +  td::string hello_; +  td::string write_buf_;    size_t write_pos_{0}; -  void start_up() override { -    socket_fd_.get_fd().set_observer(this); -    subscribe(socket_fd_.get_fd()); -    HttpHeaderCreator hc; -    Slice content = "hello world"; -    //auto content = BufferSlice("hello world"); +  void start_up() final { +    td::Scheduler::subscribe(socket_fd_.get_poll_info().extract_pollable_fd(this)); +    td::HttpHeaderCreator hc; +    td::Slice content = "hello world"; +    //auto content = td::BufferSlice("hello world");      hc.init_ok();      hc.set_keep_alive();      hc.set_content_size(content.size()); @@ -53,36 +50,37 @@ class HelloWorld : public Actor {      hello_ = hc.finish(content).ok().str();    } -  void loop() override { +  void loop() final {      auto status = do_loop();      if (status.is_error()) { -      unsubscribe(socket_fd_.get_fd()); +      td::Scheduler::unsubscribe(socket_fd_.get_poll_info().get_pollable_fd_ref());        stop();        LOG(ERROR) << "CLOSE: " << status;      }    } -  Status do_loop() { +  td::Status do_loop() { +    sync_with_poll(socket_fd_);      TRY_STATUS(read_loop());      TRY_STATUS(write_loop()); -    if (can_close(socket_fd_)) { -      return Status::Error("CLOSE"); +    if (can_close_local(socket_fd_)) { +      return td::Status::Error("CLOSE");      } -    return Status::OK(); +    return td::Status::OK();    } -  Status write_loop() { -    while (can_write(socket_fd_) && write_pos_ < write_buf_.size()) { -      TRY_RESULT(written, socket_fd_.write(Slice(write_buf_).substr(write_pos_))); +  td::Status write_loop() { +    while (can_write_local(socket_fd_) && write_pos_ < write_buf_.size()) { +      TRY_RESULT(written, socket_fd_.write(td::Slice(write_buf_).substr(write_pos_)));        write_pos_ += written;        if (write_pos_ == write_buf_.size()) {          write_pos_ = 0;          write_buf_.clear();        }      } -    return Status::OK(); +    return td::Status::OK();    } -  Status read_loop() { -    while (can_read(socket_fd_)) { -      TRY_RESULT(read_size, socket_fd_.read(MutableSlice(read_buf.data(), read_buf.size()))); +  td::Status read_loop() { +    while (can_read_local(socket_fd_)) { +      TRY_RESULT(read_size, socket_fd_.read(td::MutableSlice(read_buf.data(), read_buf.size())));        for (size_t i = 0; i < read_size; i++) {          if (read_buf[i] == '\n') {            read_new_lines++; @@ -93,46 +91,41 @@ class HelloWorld : public Actor {          }        }      } -    return Status::OK(); +    return td::Status::OK();    }  }; +  const int N = 0; -class Server : public TcpListener::Callback { +class Server final : public td::TcpListener::Callback {   public: -  void start_up() override { -    listener_ = create_actor<TcpListener>("Listener", 8082, ActorOwn<TcpListener::Callback>(actor_id(this))); +  void start_up() final { +    listener_ = +        td::create_actor<td::TcpListener>("Listener", 8082, td::ActorOwn<td::TcpListener::Callback>(actor_id(this)));    } -  void accept(SocketFd fd) override { +  void accept(td::SocketFd fd) final {      LOG(ERROR) << "ACCEPT " << cnt++;      pos_++;      auto scheduler_id = pos_ % (N != 0 ? N : 1) + (N != 0); -    create_actor_on_scheduler<HelloWorld>("HttpInboundConnection", scheduler_id, std::move(fd)).release(); +    td::create_actor_on_scheduler<HelloWorld>("HelloWorld", scheduler_id, std::move(fd)).release();    } -  void hangup() override { +  void hangup() final {      // may be it should be default?.. -    LOG(ERROR) << "hangup.."; +    LOG(ERROR) << "Hanging up..";      stop();    }   private: -  ActorOwn<TcpListener> listener_; +  td::ActorOwn<td::TcpListener> listener_;    int pos_{0};  };  int main() {    SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); -  auto scheduler = make_unique<ConcurrentScheduler>(); -  scheduler->init(N); +  auto scheduler = td::make_unique<td::ConcurrentScheduler>(N, 0);    scheduler->create_actor_unsafe<Server>(0, "Server").release();    scheduler->start();    while (scheduler->run_main(10)) {      // empty    }    scheduler->finish(); -  return 0; -} -}  // namespace td - -int main() { -  return td::main();  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp index fbda47590b..4b140422f1 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_http_server_fast.cpp @@ -1,46 +1,48 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/actor/actor.h" -  #include "td/net/HttpHeaderCreator.h"  #include "td/net/HttpQuery.h"  #include "td/net/HttpReader.h"  #include "td/net/TcpListener.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" +  #include "td/utils/buffer.h"  #include "td/utils/BufferedFd.h"  #include "td/utils/logging.h" -#include "td/utils/port/Fd.h" +#include "td/utils/port/detail/PollableFd.h"  #include "td/utils/port/SocketFd.h"  #include "td/utils/Slice.h"  #include "td/utils/Status.h" -namespace td { - -class HttpEchoConnection : public Actor { +class HttpEchoConnection final : public td::Actor {   public: -  explicit HttpEchoConnection(SocketFd fd) : fd_(std::move(fd)) { +  explicit HttpEchoConnection(td::SocketFd fd) : fd_(std::move(fd)) {    }   private: -  BufferedFd<SocketFd> fd_; -  HttpReader reader_; -  HttpQuery query_; -  void start_up() override { -    fd_.get_fd().set_observer(this); -    subscribe(fd_.get_fd()); +  td::BufferedFd<td::SocketFd> fd_; +  td::HttpReader reader_; +  td::HttpQuery query_; +  void start_up() final { +    td::Scheduler::subscribe(fd_.get_poll_info().extract_pollable_fd(this));      reader_.init(&fd_.input_buffer(), 1024 * 1024, 0);    } +  void tear_down() final { +    td::Scheduler::unsubscribe_before_close(fd_.get_poll_info().get_pollable_fd_ref()); +    fd_.close(); +  }    void handle_query() { -    query_ = HttpQuery(); -    HttpHeaderCreator hc; -    Slice content = "hello world"; -    //auto content = BufferSlice("hello world"); +    query_ = td::HttpQuery(); +    td::HttpHeaderCreator hc; +    td::Slice content = "hello world"; +    //auto content = td::BufferSlice("hello world");      hc.init_ok();      hc.set_keep_alive();      hc.set_content_size(content.size()); @@ -51,20 +53,19 @@ class HttpEchoConnection : public Actor {      fd_.output_buffer().append(res.ok());    } -  void loop() override { +  void loop() final { +    sync_with_poll(fd_);      auto status = [&] {        TRY_STATUS(loop_read());        TRY_STATUS(loop_write()); -      return Status::OK(); +      return td::Status::OK();      }(); -    if (status.is_error() || can_close(fd_)) { +    if (status.is_error() || can_close_local(fd_)) {        stop();      }    } -  Status loop_read() { -    if (can_read(fd_)) { -      TRY_STATUS(fd_.flush_read()); -    } +  td::Status loop_read() { +    TRY_STATUS(fd_.flush_read());      while (true) {        TRY_RESULT(need, reader_.read_next(&query_));        if (need == 0) { @@ -73,49 +74,43 @@ class HttpEchoConnection : public Actor {          break;        }      } -    return Status::OK(); +    return td::Status::OK();    } -  Status loop_write() { +  td::Status loop_write() {      TRY_STATUS(fd_.flush_write()); -    return Status::OK(); +    return td::Status::OK();    }  }; -const int N = 4; -class Server : public TcpListener::Callback { +const int N = 8; +class Server final : public td::TcpListener::Callback {   public: -  void start_up() override { -    listener_ = create_actor<TcpListener>("Listener", 8082, ActorOwn<TcpListener::Callback>(actor_id(this))); +  void start_up() final { +    listener_ = +        td::create_actor<td::TcpListener>("Listener", 8082, td::ActorOwn<td::TcpListener::Callback>(actor_id(this)));    } -  void accept(SocketFd fd) override { +  void accept(td::SocketFd fd) final {      pos_++;      auto scheduler_id = pos_ % (N != 0 ? N : 1) + (N != 0); -    create_actor_on_scheduler<HttpEchoConnection>("HttpInboundConnection", scheduler_id, std::move(fd)).release(); +    td::create_actor_on_scheduler<HttpEchoConnection>("HttpEchoConnection", scheduler_id, std::move(fd)).release();    } -  void hangup() override { -    LOG(ERROR) << "hangup.."; +  void hangup() final { +    LOG(ERROR) << "Hanging up..";      stop();    }   private: -  ActorOwn<TcpListener> listener_; +  td::ActorOwn<td::TcpListener> listener_;    int pos_{0};  };  int main() {    SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); -  auto scheduler = make_unique<ConcurrentScheduler>(); -  scheduler->init(N); +  auto scheduler = td::make_unique<td::ConcurrentScheduler>(N, 0);    scheduler->create_actor_unsafe<Server>(0, "Server").release();    scheduler->start();    while (scheduler->run_main(10)) {      // empty    }    scheduler->finish(); -  return 0; -} -}  // namespace td - -int main() { -  return td::main();  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp index a57b1b9b42..8c2628b8a9 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_log.cpp @@ -1,16 +1,16 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  //  #include "td/utils/benchmark.h" +#include "td/utils/common.h"  #include "td/utils/logging.h"  #include <cstdio>  #include <fstream>  #include <iostream> -#include <mutex>  #include <ostream>  #include <streambuf>  #include <string> @@ -35,63 +35,63 @@ std::string create_tmp_file() {  #endif  } -class IostreamWriteBench : public td::Benchmark { +class IostreamWriteBench final : public td::Benchmark {   protected:    std::string file_name_;    std::ofstream stream; -  enum { buffer_size = 1 << 20 }; -  char buffer[buffer_size]; +  static constexpr std::size_t BUFFER_SIZE = 1 << 20; +  char buffer[BUFFER_SIZE];   public: -  std::string get_description() const override { +  std::string get_description() const final {      return "ostream (to file, no buf, no flush)";    } -  void start_up() override { +  void start_up() final {      file_name_ = create_tmp_file();      stream.open(file_name_.c_str());      CHECK(stream.is_open()); -    //    stream.rdbuf()->pubsetbuf(buffer, buffer_size); +    //    stream.rdbuf()->pubsetbuf(buffer, BUFFER_SIZE);    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        stream << "This is just for test" << 987654321 << '\n';      }    } -  void tear_down() override { +  void tear_down() final {      stream.close();      unlink(file_name_.c_str());    }  }; -class FILEWriteBench : public td::Benchmark { +class FILEWriteBench final : public td::Benchmark {   protected:    std::string file_name_;    FILE *file; -  enum { buffer_size = 1 << 20 }; -  char buffer[buffer_size]; +  static constexpr std::size_t BUFFER_SIZE = 1 << 20; +  char buffer[BUFFER_SIZE];   public: -  std::string get_description() const override { +  std::string get_description() const final {      return "std::fprintf (to file, no buf, no flush)";    } -  void start_up() override { +  void start_up() final {      file_name_ = create_tmp_file();      file = fopen(file_name_.c_str(), "w"); -    //    setvbuf(file, buffer, _IOFBF, buffer_size); +    //    setvbuf(file, buffer, _IOFBF, BUFFER_SIZE);    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        std::fprintf(file, "This is just for test%d\n", 987654321);        //      std::fflush(file);      }    } -  void tear_down() override { +  void tear_down() final {      std::fclose(file);      unlink(file_name_.c_str());    } @@ -100,58 +100,56 @@ class FILEWriteBench : public td::Benchmark {  #if TD_ANDROID  #include <android/log.h>  #define ALOG(...) __android_log_print(ANDROID_LOG_VERBOSE, "XXX", __VA_ARGS__) -class ALogWriteBench : public td::Benchmark { +class ALogWriteBench final : public td::Benchmark {   public: -  std::string get_description() const override { +  std::string get_description() const final {      return "android_log";    } -  void start_up() override { +  void start_up() final {    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        ALOG("This is just for test%d\n", 987654321);      }    } -  void tear_down() override { +  void tear_down() final {    }  };  #endif -class LogWriteBench : public td::Benchmark { +class LogWriteBench final : public td::Benchmark {   protected:    std::string file_name_;    std::ofstream stream;    std::streambuf *old_buf; -  enum { buffer_size = 1 << 20 }; -  char buffer[buffer_size]; +  static constexpr std::size_t BUFFER_SIZE = 1 << 20; +  char buffer[BUFFER_SIZE];   public: -  std::string get_description() const override { +  std::string get_description() const final {      return "td_log (slow in debug mode)";    } -  void start_up() override { +  void start_up() final {      file_name_ = create_tmp_file();      stream.open(file_name_.c_str());      CHECK(stream.is_open());      old_buf = std::cerr.rdbuf(stream.rdbuf());    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        LOG(DEBUG) << "This is just for test" << 987654321;      }    } -  void tear_down() override { +  void tear_down() final {      stream.close();      unlink(file_name_.c_str());      std::cerr.rdbuf(old_buf);    }  }; -std::mutex mutex; -  int main() {    td::bench(LogWriteBench());  #if TD_ANDROID @@ -159,5 +157,4 @@ int main() {  #endif    td::bench(IostreamWriteBench());    td::bench(FILEWriteBench()); -  return 0;  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp index bfbcea438b..56a6117bb4 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_misc.cpp @@ -1,11 +1,15 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // +#include "td/telegram/telegram_api.h" +#include "td/telegram/telegram_api.hpp" +  #include "td/utils/benchmark.h"  #include "td/utils/common.h" +#include "td/utils/format.h"  #include "td/utils/logging.h"  #include "td/utils/port/Clocks.h"  #include "td/utils/port/EventFd.h" @@ -15,9 +19,9 @@  #include "td/utils/port/Stat.h"  #include "td/utils/port/thread.h"  #include "td/utils/Slice.h" - -#include "td/telegram/telegram_api.h" -#include "td/telegram/telegram_api.hpp" +#include "td/utils/SliceBuilder.h" +#include "td/utils/Status.h" +#include "td/utils/ThreadSafeCounter.h"  #if !TD_WINDOWS  #include <unistd.h> @@ -28,37 +32,38 @@  #include <semaphore.h>  #endif +#include <algorithm> +#include <array>  #include <atomic>  #include <cstdint> - -namespace td { +#include <set>  class F { -  uint32 ∑ +  td::uint32 ∑   public: -  explicit F(uint32 &sum) : sum(sum) { +  explicit F(td::uint32 &sum) : sum(sum) {    }    template <class T>    void operator()(const T &x) const { -    sum += static_cast<uint32>(x.get_id()); +    sum += static_cast<td::uint32>(x.get_id());    }  };  BENCH(Call, "TL Call") { -  tl_object_ptr<telegram_api::Function> x = make_tl_object<telegram_api::account_getWallPapers>(); -  uint32 res = 0; +  td::tl_object_ptr<td::telegram_api::Function> x = td::make_tl_object<td::telegram_api::account_getWallPapers>(0); +  td::uint32 res = 0;    F f(res);    for (int i = 0; i < n; i++) {      downcast_call(*x, f);    } -  do_not_optimize_away(res); +  td::do_not_optimize_away(res);  }  #if !TD_EVENTFD_UNSUPPORTED  BENCH(EventFd, "EventFd") { -  EventFd fd; +  td::EventFd fd;    fd.init();    for (int i = 0; i < n; i++) {      fd.release(); @@ -75,15 +80,15 @@ BENCH(NewInt, "new int + delete") {      res += reinterpret_cast<std::uintptr_t>(x);      delete x;    } -  do_not_optimize_away(res); +  td::do_not_optimize_away(res);  } -BENCH(NewObj, "new struct then delete") { +BENCH(NewObj, "new struct, then delete") {    struct A { -    int32 a = 0; -    int32 b = 0; -    int32 c = 0; -    int32 d = 0; +    td::int32 a = 0; +    td::int32 b = 0; +    td::int32 c = 0; +    td::int32 d = 0;    };    std::uintptr_t res = 0;    A **ptr = new A *[n]; @@ -95,57 +100,57 @@ BENCH(NewObj, "new struct then delete") {      delete ptr[i];    }    delete[] ptr; -  do_not_optimize_away(res); +  td::do_not_optimize_away(res);  }  #if !TD_THREAD_UNSUPPORTED -BENCH(ThreadNew, "new struct then delete in several threads") { -  td::NewObjBench a, b; -  thread ta([&] { a.run(n / 2); }); -  thread tb([&] { b.run(n - n / 2); }); +BENCH(ThreadNew, "new struct, then delete in 2 threads") { +  NewObjBench a; +  NewObjBench b; +  td::thread ta([&] { a.run(n / 2); }); +  td::thread tb([&] { b.run(n - n / 2); });    ta.join();    tb.join();  }  #endif - -// Too hard for android clang (?) +/* +// Too hard for clang (?)  BENCH(Time, "Clocks::monotonic") {    double res = 0;    for (int i = 0; i < n; i++) { -    res += Clocks::monotonic(); +    res += td::Clocks::monotonic();    } -  do_not_optimize_away(res); +  td::do_not_optimize_away(res);  } - +*/  #if !TD_WINDOWS -class PipeBench : public Benchmark { +class PipeBench final : public td::Benchmark {   public:    int p[2]; -  PipeBench() { -    pipe(p); -  } - -  string get_description() const override { +  td::string get_description() const final {      return "pipe write + read int32";    } -  void start_up() override { -    pipe(p); +  void start_up() final { +    int res = pipe(p); +    CHECK(res == 0);    } -  void run(int n) override { +  void run(int n) final {      int res = 0;      for (int i = 0; i < n; i++) {        int val = 1; -      write(p[1], &val, sizeof(val)); -      read(p[0], &val, sizeof(val)); +      auto write_len = write(p[1], &val, sizeof(val)); +      CHECK(write_len == sizeof(val)); +      auto read_len = read(p[0], &val, sizeof(val)); +      CHECK(read_len == sizeof(val));        res += val;      } -    do_not_optimize_away(res); +    td::do_not_optimize_away(res);    } -  void tear_down() override { +  void tear_down() final {      close(p[0]);      close(p[1]);    } @@ -153,42 +158,42 @@ class PipeBench : public Benchmark {  #endif  #if TD_LINUX || TD_ANDROID || TD_TIZEN -class SemBench : public Benchmark { +class SemBench final : public td::Benchmark {    sem_t sem;   public: -  string get_description() const override { +  td::string get_description() const final {      return "sem post + wait";    } -  void start_up() override { +  void start_up() final {      int err = sem_init(&sem, 0, 0);      CHECK(err != -1);    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        sem_post(&sem);        sem_wait(&sem);      }    } -  void tear_down() override { +  void tear_down() final {      sem_destroy(&sem);    }  };  #endif  #if !TD_WINDOWS -class UtimeBench : public Benchmark { +class UtimeBench final : public td::Benchmark {   public: -  void start_up() override { -    FileFd::open("test", FileFd::Flags::Create | FileFd::Flags::Write).move_as_ok().close(); +  void start_up() final { +    td::FileFd::open("test", td::FileFd::Create | td::FileFd::Write).move_as_ok().close();    } -  string get_description() const override { +  td::string get_description() const final {      return "utime";    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) {        int err = utime("test", nullptr);        CHECK(err >= 0); @@ -203,73 +208,77 @@ class UtimeBench : public Benchmark {  #endif  BENCH(Pwrite, "pwrite") { -  auto fd = FileFd::open("test", FileFd::Flags::Create | FileFd::Flags::Write).move_as_ok(); +  auto fd = td::FileFd::open("test", td::FileFd::Create | td::FileFd::Write).move_as_ok();    for (int i = 0; i < n; i++) {      fd.pwrite("a", 0).ok();    }    fd.close();  } -class CreateFileBench : public Benchmark { -  string get_description() const override { +class CreateFileBench final : public td::Benchmark { +  td::string get_description() const final {      return "create_file";    } -  void start_up() override { -    mkdir("A").ensure(); +  void start_up() final { +    td::mkdir("A").ensure();    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < n; i++) { -      FileFd::open("A/" + to_string(i), FileFd::Flags::Write | FileFd::Flags::Create).move_as_ok().close(); +      td::FileFd::open(PSLICE() << "A/" << i, td::FileFd::Write | td::FileFd::Create).move_as_ok().close();      }    } -  void tear_down() override { -    auto status = td::walk_path("A/", [&](CSlice path, bool is_dir) { -      if (is_dir) { -        rmdir(path).ignore(); -      } else { -        unlink(path).ignore(); +  void tear_down() final { +    td::walk_path("A/", [&](td::CSlice path, auto type) { +      if (type == td::WalkPath::Type::ExitDir) { +        td::rmdir(path).ignore(); +      } else if (type == td::WalkPath::Type::NotDir) { +        td::unlink(path).ignore();        } -    }); +    }).ignore();    }  }; -class WalkPathBench : public Benchmark { -  string get_description() const override { + +class WalkPathBench final : public td::Benchmark { +  td::string get_description() const final {      return "walk_path";    } -  void start_up_n(int n) override { -    mkdir("A").ensure(); +  void start_up_n(int n) final { +    td::mkdir("A").ensure();      for (int i = 0; i < n; i++) { -      FileFd::open("A/" + to_string(i), FileFd::Flags::Write | FileFd::Flags::Create).move_as_ok().close(); +      td::FileFd::open(PSLICE() << "A/" << i, td::FileFd::Write | td::FileFd::Create).move_as_ok().close();      }    } -  void run(int n) override { +  void run(int n) final {      int cnt = 0; -    auto status = td::walk_path("A/", [&](CSlice path, bool is_dir) { -      stat(path).ok(); +    td::walk_path("A/", [&](td::CSlice path, auto type) { +      if (type == td::WalkPath::Type::EnterDir) { +        return; +      } +      td::stat(path).ok();        cnt++; -    }); -  } -  void tear_down() override { -    auto status = td::walk_path("A/", [&](CSlice path, bool is_dir) { -      if (is_dir) { -        rmdir(path).ignore(); -      } else { -        unlink(path).ignore(); +    }).ignore(); +  } +  void tear_down() final { +    td::walk_path("A/", [&](td::CSlice path, auto type) { +      if (type == td::WalkPath::Type::ExitDir) { +        td::rmdir(path).ignore(); +      } else if (type == td::WalkPath::Type::NotDir) { +        td::unlink(path).ignore();        } -    }); +    }).ignore();    }  };  #if !TD_THREAD_UNSUPPORTED  template <int ThreadN = 2> -class AtomicReleaseIncBench : public Benchmark { -  string get_description() const override { +class AtomicReleaseIncBench final : public td::Benchmark { +  td::string get_description() const final {      return PSTRING() << "AtomicReleaseInc" << ThreadN;    } -  static std::atomic<uint64> a_; -  void run(int n) override { -    std::vector<thread> threads; +  static std::atomic<td::uint64> a_; +  void run(int n) final { +    td::vector<td::thread> threads;      for (int i = 0; i < ThreadN; i++) {        threads.emplace_back([&] {          for (int i = 0; i < n / ThreadN; i++) { @@ -283,17 +292,17 @@ class AtomicReleaseIncBench : public Benchmark {    }  };  template <int ThreadN> -std::atomic<uint64> AtomicReleaseIncBench<ThreadN>::a_; +std::atomic<td::uint64> AtomicReleaseIncBench<ThreadN>::a_;  template <int ThreadN = 2> -class AtomicReleaseCasIncBench : public Benchmark { -  string get_description() const override { +class AtomicReleaseCasIncBench final : public td::Benchmark { +  td::string get_description() const final {      return PSTRING() << "AtomicReleaseCasInc" << ThreadN;    } -  static std::atomic<uint64> a_; -  void run(int n) override { -    std::vector<thread> threads; +  static std::atomic<td::uint64> a_; +  void run(int n) final { +    td::vector<td::thread> threads;      for (int i = 0; i < ThreadN; i++) {        threads.emplace_back([&] {          for (int i = 0; i < n / ThreadN; i++) { @@ -309,16 +318,16 @@ class AtomicReleaseCasIncBench : public Benchmark {    }  };  template <int ThreadN> -std::atomic<uint64> AtomicReleaseCasIncBench<ThreadN>::a_; +std::atomic<td::uint64> AtomicReleaseCasIncBench<ThreadN>::a_; -template <int ThreadN = 2> -class RwMutexReadBench : public Benchmark { -  string get_description() const override { +template <int ThreadN> +class RwMutexReadBench final : public td::Benchmark { +  td::string get_description() const final {      return PSTRING() << "RwMutexRead" << ThreadN;    } -  RwMutex mutex_; -  void run(int n) override { -    std::vector<thread> threads; +  td::RwMutex mutex_; +  void run(int n) final { +    td::vector<td::thread> threads;      for (int i = 0; i < ThreadN; i++) {        threads.emplace_back([&] {          for (int i = 0; i < n / ThreadN; i++) { @@ -331,14 +340,15 @@ class RwMutexReadBench : public Benchmark {      }    }  }; -template <int ThreadN = 2> -class RwMutexWriteBench : public Benchmark { -  string get_description() const override { + +template <int ThreadN> +class RwMutexWriteBench final : public td::Benchmark { +  td::string get_description() const final {      return PSTRING() << "RwMutexWrite" << ThreadN;    } -  RwMutex mutex_; -  void run(int n) override { -    std::vector<thread> threads; +  td::RwMutex mutex_; +  void run(int n) final { +    td::vector<td::thread> threads;      for (int i = 0; i < ThreadN; i++) {        threads.emplace_back([&] {          for (int i = 0; i < n / ThreadN; i++) { @@ -351,42 +361,366 @@ class RwMutexWriteBench : public Benchmark {      }    }  }; + +class ThreadSafeCounterBench final : public td::Benchmark { +  static td::ThreadSafeCounter counter_; +  int thread_count_; + +  td::string get_description() const final { +    return PSTRING() << "ThreadSafeCounter" << thread_count_; +  } +  void run(int n) final { +    counter_.clear(); +    td::vector<td::thread> threads; +    for (int i = 0; i < thread_count_; i++) { +      threads.emplace_back([n] { +        for (int i = 0; i < n; i++) { +          counter_.add(1); +        } +      }); +    } +    for (auto &thread : threads) { +      thread.join(); +    } +    CHECK(counter_.sum() == n * thread_count_); +  } + + public: +  explicit ThreadSafeCounterBench(int thread_count) : thread_count_(thread_count) { +  } +}; +td::ThreadSafeCounter ThreadSafeCounterBench::counter_; + +template <bool StrictOrder> +class AtomicCounterBench final : public td::Benchmark { +  static std::atomic<td::int64> counter_; +  int thread_count_; + +  td::string get_description() const final { +    return PSTRING() << "AtomicCounter" << thread_count_; +  } +  void run(int n) final { +    counter_.store(0); +    td::vector<td::thread> threads; +    for (int i = 0; i < thread_count_; i++) { +      threads.emplace_back([n] { +        for (int i = 0; i < n; i++) { +          counter_.fetch_add(1, StrictOrder ? std::memory_order_seq_cst : std::memory_order_relaxed); +        } +      }); +    } +    for (auto &thread : threads) { +      thread.join(); +    } +    CHECK(counter_.load() == n * thread_count_); +  } + + public: +  explicit AtomicCounterBench(int thread_count) : thread_count_(thread_count) { +  } +}; +template <bool StrictOrder> +std::atomic<td::int64> AtomicCounterBench<StrictOrder>::counter_; +  #endif -}  // namespace td + +class IdDuplicateCheckerOld { + public: +  static td::string get_description() { +    return "Old"; +  } +  td::Status check(td::int64 message_id) { +    if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS) { +      auto oldest_message_id = *saved_message_ids_.begin(); +      if (message_id < oldest_message_id) { +        return td::Status::Error(2, PSLICE() << "Ignore very old message_id " +                                             << td::tag("oldest message_id", oldest_message_id) +                                             << td::tag("got message_id", message_id)); +      } +    } +    if (saved_message_ids_.count(message_id) != 0) { +      return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); +    } + +    saved_message_ids_.insert(message_id); +    if (saved_message_ids_.size() > MAX_SAVED_MESSAGE_IDS) { +      saved_message_ids_.erase(saved_message_ids_.begin()); +    } +    return td::Status::OK(); +  } + + private: +  static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; +  std::set<td::int64> saved_message_ids_; +}; + +template <size_t MAX_SAVED_MESSAGE_IDS> +class IdDuplicateCheckerNew { + public: +  static td::string get_description() { +    return PSTRING() << "New" << MAX_SAVED_MESSAGE_IDS; +  } +  td::Status check(td::int64 message_id) { +    auto insert_result = saved_message_ids_.insert(message_id); +    if (!insert_result.second) { +      return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); +    } +    if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { +      auto begin_it = saved_message_ids_.begin(); +      bool is_very_old = begin_it == insert_result.first; +      saved_message_ids_.erase(begin_it); +      if (is_very_old) { +        return td::Status::Error(2, PSLICE() << "Ignore very old message_id " +                                             << td::tag("oldest message_id", *saved_message_ids_.begin()) +                                             << td::tag("got message_id", message_id)); +      } +    } +    return td::Status::OK(); +  } + + private: +  std::set<td::int64> saved_message_ids_; +}; + +class IdDuplicateCheckerNewOther { + public: +  static td::string get_description() { +    return "NewOther"; +  } +  td::Status check(td::int64 message_id) { +    if (!saved_message_ids_.insert(message_id).second) { +      return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); +    } +    if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { +      auto begin_it = saved_message_ids_.begin(); +      bool is_very_old = *begin_it == message_id; +      saved_message_ids_.erase(begin_it); +      if (is_very_old) { +        return td::Status::Error(2, PSLICE() << "Ignore very old message_id " +                                             << td::tag("oldest message_id", *saved_message_ids_.begin()) +                                             << td::tag("got message_id", message_id)); +      } +    } +    return td::Status::OK(); +  } + + private: +  static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; +  std::set<td::int64> saved_message_ids_; +}; + +class IdDuplicateCheckerNewSimple { + public: +  static td::string get_description() { +    return "NewSimple"; +  } +  td::Status check(td::int64 message_id) { +    auto insert_result = saved_message_ids_.insert(message_id); +    if (!insert_result.second) { +      return td::Status::Error(1, "Ignore duplicated message_id"); +    } +    if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { +      auto begin_it = saved_message_ids_.begin(); +      bool is_very_old = begin_it == insert_result.first; +      saved_message_ids_.erase(begin_it); +      if (is_very_old) { +        return td::Status::Error(2, "Ignore very old message_id"); +      } +    } +    return td::Status::OK(); +  } + + private: +  static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; +  std::set<td::int64> saved_message_ids_; +}; + +template <size_t max_size> +class IdDuplicateCheckerArray { + public: +  static td::string get_description() { +    return PSTRING() << "Array" << max_size; +  } +  td::Status check(td::int64 message_id) { +    if (end_pos_ == 2 * max_size) { +      std::copy_n(&saved_message_ids_[max_size], max_size, &saved_message_ids_[0]); +      end_pos_ = max_size; +    } +    if (end_pos_ == 0 || message_id > saved_message_ids_[end_pos_ - 1]) { +      // fast path +      saved_message_ids_[end_pos_++] = message_id; +      return td::Status::OK(); +    } +    if (end_pos_ >= max_size && message_id < saved_message_ids_[0]) { +      return td::Status::Error(2, PSLICE() << "Ignore very old message_id " +                                           << td::tag("oldest message_id", saved_message_ids_[0]) +                                           << td::tag("got message_id", message_id)); +    } +    auto it = std::lower_bound(&saved_message_ids_[0], &saved_message_ids_[end_pos_], message_id); +    if (*it == message_id) { +      return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); +    } +    std::copy_backward(it, &saved_message_ids_[end_pos_], &saved_message_ids_[end_pos_ + 1]); +    *it = message_id; +    ++end_pos_; +    return td::Status::OK(); +  } + + private: +  std::array<td::int64, 2 * max_size> saved_message_ids_; +  std::size_t end_pos_ = 0; +}; + +template <class T> +class DuplicateCheckerBench final : public td::Benchmark { +  td::string get_description() const final { +    return PSTRING() << "DuplicateCheckerBench" << T::get_description(); +  } +  void run(int n) final { +    T checker_; +    for (int i = 0; i < n; i++) { +      checker_.check(i).ensure(); +    } +  } +}; + +template <class T> +class DuplicateCheckerBenchRepeat final : public td::Benchmark { +  td::string get_description() const final { +    return PSTRING() << "DuplicateCheckerBenchRepeat" << T::get_description(); +  } +  void run(int n) final { +    T checker_; +    for (int i = 0; i < n; i++) { +      auto iter = i >> 10; +      auto pos = i - (iter << 10); +      if (pos < 768) { +        if (iter >= 3 && pos == 0) { +          auto error = checker_.check((iter - 3) * 768 + pos); +          CHECK(error.error().code() == 2); +        } +        checker_.check(iter * 768 + pos).ensure(); +      } else { +        checker_.check(iter * 768 + pos - 256).ensure_error(); +      } +    } +  } +}; + +template <class T> +class DuplicateCheckerBenchRepeatOnly final : public td::Benchmark { +  td::string get_description() const final { +    return PSTRING() << "DuplicateCheckerBenchRepeatOnly" << T::get_description(); +  } +  void run(int n) final { +    T checker_; +    for (int i = 0; i < n; i++) { +      auto result = checker_.check(i & 255); +      CHECK(result.is_error() == (i >= 256)); +    } +  } +}; + +template <class T> +class DuplicateCheckerBenchReverse final : public td::Benchmark { +  td::string get_description() const final { +    return PSTRING() << "DuplicateCheckerBenchReverseAdd" << T::get_description(); +  } +  void run(int n) final { +    T checker_; +    for (int i = 0; i < n; i++) { +      auto pos = i & 255; +      checker_.check(i - pos + (255 - pos)).ensure(); +    } +  } +}; + +template <class T> +class DuplicateCheckerBenchEvenOdd final : public td::Benchmark { +  td::string get_description() const final { +    return PSTRING() << "DuplicateCheckerBenchEvenOdd" << T::get_description(); +  } +  void run(int n) final { +    T checker_; +    for (int i = 0; i < n; i++) { +      auto pos = i & 255; +      checker_.check(i - pos + (pos * 2) % 256 + (pos * 2) / 256).ensure(); +    } +  } +};  int main() {    SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); + +  td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerNew<1000>>()); +  td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerNew<300>>()); +  td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerArray<1000>>()); +  td::bench(DuplicateCheckerBenchEvenOdd<IdDuplicateCheckerArray<300>>()); + +  td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerNew<1000>>()); +  td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerNew<300>>()); +  td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerArray<1000>>()); +  td::bench(DuplicateCheckerBenchReverse<IdDuplicateCheckerArray<300>>()); + +  td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerNew<1000>>()); +  td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerNew<300>>()); +  td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerArray<1000>>()); +  td::bench(DuplicateCheckerBenchRepeatOnly<IdDuplicateCheckerArray<300>>()); + +  td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerOld>()); +  td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNew<1000>>()); +  td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNewOther>()); +  td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNewSimple>()); +  td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerNew<300>>()); +  td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerArray<1000>>()); +  td::bench(DuplicateCheckerBenchRepeat<IdDuplicateCheckerArray<300>>()); + +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerOld>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<1000>>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerNewOther>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerNewSimple>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<300>>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<100>>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerNew<10>>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerArray<1000>>()); +  td::bench(DuplicateCheckerBench<IdDuplicateCheckerArray<300>>()); +  #if !TD_THREAD_UNSUPPORTED -  td::bench(td::AtomicReleaseIncBench<1>()); -  td::bench(td::AtomicReleaseIncBench<2>()); -  td::bench(td::AtomicReleaseCasIncBench<1>()); -  td::bench(td::AtomicReleaseCasIncBench<2>()); -  td::bench(td::RwMutexWriteBench<1>()); -  td::bench(td::RwMutexReadBench<1>()); -  td::bench(td::RwMutexWriteBench<>()); -  td::bench(td::RwMutexReadBench<>()); +  for (int i = 1; i <= 16; i *= 2) { +    td::bench(ThreadSafeCounterBench(i)); +    td::bench(AtomicCounterBench<false>(i)); +    td::bench(AtomicCounterBench<true>(i)); +  } + +  td::bench(AtomicReleaseIncBench<1>()); +  td::bench(AtomicReleaseIncBench<2>()); +  td::bench(AtomicReleaseCasIncBench<1>()); +  td::bench(AtomicReleaseCasIncBench<2>()); +  td::bench(RwMutexWriteBench<1>()); +  td::bench(RwMutexReadBench<1>()); +  td::bench(RwMutexWriteBench<2>()); +  td::bench(RwMutexReadBench<2>());  #endif  #if !TD_WINDOWS -  td::bench(td::UtimeBench()); +  td::bench(UtimeBench());  #endif -  td::bench(td::WalkPathBench()); -  td::bench(td::CreateFileBench()); -  td::bench(td::PwriteBench()); +  td::bench(WalkPathBench()); +  td::bench(CreateFileBench()); +  td::bench(PwriteBench()); -  td::bench(td::CallBench()); +  td::bench(CallBench());  #if !TD_THREAD_UNSUPPORTED -  td::bench(td::ThreadNewBench()); +  td::bench(ThreadNewBench());  #endif  #if !TD_EVENTFD_UNSUPPORTED -  td::bench(td::EventFdBench()); +  td::bench(EventFdBench());  #endif -  td::bench(td::NewObjBench()); -  td::bench(td::NewIntBench()); +  td::bench(NewObjBench()); +  td::bench(NewIntBench());  #if !TD_WINDOWS -  td::bench(td::PipeBench()); +  td::bench(PipeBench());  #endif  #if TD_LINUX || TD_ANDROID || TD_TIZEN -  td::bench(td::SemBench()); +  td::bench(SemBench());  #endif -  return 0;  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp index 13288e6cd7..6f7cf20bc9 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_queue.cpp @@ -1,5 +1,5 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -8,47 +8,56 @@  #include "td/utils/common.h"  #include "td/utils/logging.h"  #include "td/utils/MpscPollableQueue.h" +#include "td/utils/port/sleep.h" +#include "td/utils/port/thread.h"  #include "td/utils/queue.h" +#include "td/utils/Random.h"  // TODO: check system calls  // TODO: all return values must be checked  #include <atomic> -#include <cstdio> -#include <cstdlib> -#include <vector> +#if TD_PORT_POSIX  #include <pthread.h>  #include <sched.h>  #include <semaphore.h>  #include <sys/syscall.h>  #include <unistd.h> +#endif  #if TD_LINUX  #include <sys/eventfd.h>  #endif -using std::atomic; -using std::vector; - -using td::int32; -using td::uint32; -  #define MODE std::memory_order_relaxed  // void set_affinity(int mask) { -// int err, syscallres; -// pid_t pid = gettid(); -// syscallres = syscall(__NR_sched_setaffinity, pid, sizeof(mask), &mask); -// if (syscallres) { -// err = errno; -// perror("oppa"); -//} -//} - -// TODO: warnings and asserts. There should be no warnings or debug output in production. +//   pid_t pid = gettid(); +//   int syscallres = syscall(__NR_sched_setaffinity, pid, sizeof(mask), &mask); +//   if (syscallres) { +//     perror("Failed to set affinity"); +//   } +// } +  using qvalue_t = int; +class Backoff { +  int cnt = 0; + + public: +  bool next() { +    cnt++; +    if (cnt < 50) { +      return true; +    } else { +      td::usleep_for(1); +      return cnt < 500; +    } +  } +}; + +#if TD_PORT_POSIX  // Just for testing, not production  class PipeQueue {    int input; @@ -57,18 +66,21 @@ class PipeQueue {   public:    void init() {      int new_pipe[2]; -    pipe(new_pipe); +    int res = pipe(new_pipe); +    CHECK(res == 0);      output = new_pipe[0];      input = new_pipe[1];    }    void put(qvalue_t value) { -    write(input, &value, sizeof(value)); +    auto len = write(input, &value, sizeof(value)); +    CHECK(len == sizeof(value));    }    qvalue_t get() {      qvalue_t res; -    read(output, &res, sizeof(res)); +    auto len = read(output, &res, sizeof(res)); +    CHECK(len == sizeof(res));      return res;    } @@ -78,26 +90,8 @@ class PipeQueue {    }  }; -class Backoff { -  int cnt; - - public: -  Backoff() : cnt(0) { -  } - -  bool next() { -    cnt++; -    if (cnt < 50) { -      return true; -    } else { -      sched_yield(); -      return cnt < 500; -    } -  } -}; -  class VarQueue { -  atomic<qvalue_t> data; +  std::atomic<qvalue_t> data{0};   public:    void init() { @@ -179,6 +173,7 @@ class SemQueue {      return get();    }  }; +#endif  #if TD_LINUX  class EventfdQueue { @@ -193,11 +188,14 @@ class EventfdQueue {    void put(qvalue_t value) {      q.put(value);      td::int64 x = 1; -    write(fd, &x, sizeof(x)); +    auto len = write(fd, &x, sizeof(x)); +    CHECK(len == sizeof(x));    }    qvalue_t get() {      td::int64 x; -    read(fd, &x, sizeof(x)); +    auto len = read(fd, &x, sizeof(x)); +    CHECK(len == sizeof(x)); +    CHECK(x == 1);      return q.get();    }    void destroy() { @@ -212,17 +210,17 @@ const int queue_buf_size = 1 << 10;  class BufferQueue {    struct node {      qvalue_t val; -    char pad[64 - sizeof(atomic<qvalue_t>)]; +    char pad[64 - sizeof(std::atomic<qvalue_t>)];    };    node q[queue_buf_size];    struct Position { -    atomic<uint32> i; -    char pad[64 - sizeof(atomic<uint32>)]; +    std::atomic<td::uint32> i{0}; +    char pad[64 - sizeof(std::atomic<td::uint32>)]; -    uint32 local_read_i; -    uint32 local_write_i; -    char pad2[64 - sizeof(uint32) * 2]; +    td::uint32 local_read_i; +    td::uint32 local_write_i; +    char pad2[64 - sizeof(td::uint32) * 2];      void init() {        i = 0; @@ -318,8 +316,7 @@ class BufferQueue {        return;      }      if (!update_writer()) { -      std::fprintf(stderr, "put strong failed\n"); -      std::exit(0); +      LOG(FATAL) << "Put strong failed";      }      put_unsafe(val);    } @@ -336,7 +333,7 @@ class BufferQueue {  #if TD_LINUX  class BufferedFdQueue {    int fd; -  atomic<int> wait_flag; +  std::atomic<int> wait_flag{0};    BufferQueue q;    char pad[64]; @@ -351,7 +348,8 @@ class BufferedFdQueue {      td::int64 x = 1;      __sync_synchronize();      if (wait_flag.load(MODE)) { -      write(fd, &x, sizeof(x)); +      auto len = write(fd, &x, sizeof(x)); +      CHECK(len == sizeof(x));      }    }    void put_noflush(qvalue_t value) { @@ -362,7 +360,8 @@ class BufferedFdQueue {      td::int64 x = 1;      __sync_synchronize();      if (wait_flag.load(MODE)) { -      write(fd, &x, sizeof(x)); +      auto len = write(fd, &x, sizeof(x)); +      CHECK(len == sizeof(x));      }    }    void flush_reader() { @@ -393,7 +392,8 @@ class BufferedFdQueue {      wait_flag.store(1, MODE);      __sync_synchronize();      while (!(res = q.update_reader())) { -      read(fd, &x, sizeof(x)); +      auto len = read(fd, &x, sizeof(x)); +      CHECK(len == sizeof(x));        __sync_synchronize();      }      wait_flag.store(0, MODE); @@ -416,7 +416,8 @@ class BufferedFdQueue {      wait_flag.store(1, MODE);      __sync_synchronize();      while (!q.update_reader()) { -      read(fd, &x, sizeof(x)); +      auto len = read(fd, &x, sizeof(x)); +      CHECK(len == sizeof(x));        __sync_synchronize();      }      wait_flag.store(0, MODE); @@ -430,7 +431,7 @@ class BufferedFdQueue {  class FdQueue {    int fd; -  atomic<int> wait_flag; +  std::atomic<int> wait_flag{0};    VarQueue q;    char pad[64]; @@ -445,12 +446,14 @@ class FdQueue {      td::int64 x = 1;      __sync_synchronize();      if (wait_flag.load(MODE)) { -      write(fd, &x, sizeof(x)); +      auto len = write(fd, &x, sizeof(x)); +      CHECK(len == sizeof(x));      }    }    qvalue_t get() {      // td::int64 x; -    // read(fd, &x, sizeof(x)); +    // auto len = read(fd, &x, sizeof(x)); +    // CHECK(len == sizeof(x));      // return q.get();      Backoff backoff; @@ -466,14 +469,13 @@ class FdQueue {      td::int64 x;      wait_flag.store(1, MODE);      __sync_synchronize(); -    // std::fprintf(stderr, "!\n"); -    // while (res == -1 && read(fd, &x, sizeof(x))) { -    // res = q.try_get(); -    //} +    // while (res == -1 && read(fd, &x, sizeof(x)) == sizeof(x)) { +    //   res = q.try_get(); +    // }      do {        __sync_synchronize();        res = q.try_get(); -    } while (res == -1 && read(fd, &x, sizeof(x))); +    } while (res == -1 && read(fd, &x, sizeof(x)) == sizeof(x));      q.acquire();      wait_flag.store(0, MODE);      return res; @@ -485,6 +487,7 @@ class FdQueue {  };  #endif +#if TD_PORT_POSIX  class SemBackoffQueue {    sem_t sem;    VarQueue q; @@ -554,46 +557,40 @@ class SemCheatQueue {  };  template <class QueueT> -class QueueBenchmark2 : public td::Benchmark { +class QueueBenchmark2 final : public td::Benchmark {    QueueT client, server;    int connections_n, queries_n;    int server_active_connections;    int client_active_connections; -  vector<td::int64> server_conn; -  vector<td::int64> client_conn; +  td::vector<td::int64> server_conn; +  td::vector<td::int64> client_conn; + +  td::string name;   public: -  explicit QueueBenchmark2(int connections_n = 1) : connections_n(connections_n) { +  QueueBenchmark2(int connections_n, td::string name) : connections_n(connections_n), name(std::move(name)) {    } -  std::string get_description() const override { -    return "QueueBenchmark2"; +  td::string get_description() const final { +    return name;    } -  void start_up() override { +  void start_up() final {      client.init();      server.init();    } -  void tear_down() override { +  void tear_down() final {      client.destroy();      server.destroy();    }    void server_process(qvalue_t value) {      int no = value & 0x00FFFFFF; -    int co = static_cast<int>(static_cast<unsigned int>(value) >> 24); -    // std::fprintf(stderr, "-->%d %d\n", co, no); -    if (co < 0 || co >= connections_n || no != server_conn[co]++) { -      std::fprintf(stderr, "%d %d\n", co, no); -      std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(server_conn[co] - 1)); -      std::fprintf(stderr, "Server BUG\n"); -      while (true) { -      } -    } -    // std::fprintf(stderr, "no = %d/%d\n", no, queries_n); -    // std::fprintf(stderr, "answer: %d %d\n", no, co); +    auto co = static_cast<int>(static_cast<td::uint32>(value) >> 24); +    CHECK(co >= 0 && co < connections_n); +    CHECK(no == server_conn[co]++);      client.writer_put(value);      client.writer_flush(); @@ -603,15 +600,12 @@ class QueueBenchmark2 : public td::Benchmark {    }    void *server_run(void *) { -    server_conn = vector<td::int64>(connections_n); +    server_conn = td::vector<td::int64>(connections_n);      server_active_connections = connections_n;      while (server_active_connections > 0) {        int cnt = server.reader_wait(); -      if (cnt == 0) { -        std::fprintf(stderr, "ERROR!\n"); -        std::exit(0); -      } +      CHECK(cnt != 0);        while (cnt-- > 0) {          server_process(server.reader_get_unsafe());          server.reader_flush(); @@ -624,18 +618,10 @@ class QueueBenchmark2 : public td::Benchmark {    void client_process(qvalue_t value) {      int no = value & 0x00FFFFFF; -    int co = static_cast<int>(static_cast<unsigned int>(value) >> 24); -    // std::fprintf(stderr, "<--%d %d\n", co, no); -    if (co < 0 || co >= connections_n || no != client_conn[co]++) { -      std::fprintf(stderr, "%d %d\n", co, no); -      std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(client_conn[co] - 1)); -      std::fprintf(stderr, "BUG\n"); -      while (true) { -      } -      std::exit(0); -    } +    auto co = static_cast<int>(static_cast<td::uint32>(value) >> 24); +    CHECK(co >= 0 && co < connections_n); +    CHECK(no == client_conn[co]++);      if (no + 1 < queries_n) { -      // std::fprintf(stderr, "query: %d %d\n", no + 1, co);        server.writer_put(value + 1);        server.writer_flush();      } else { @@ -644,12 +630,9 @@ class QueueBenchmark2 : public td::Benchmark {    }    void *client_run(void *) { -    client_conn = vector<td::int64>(connections_n); +    client_conn = td::vector<td::int64>(connections_n);      client_active_connections = connections_n; -    if (queries_n >= (1 << 24)) { -      std::fprintf(stderr, "Too big queries_n\n"); -      std::exit(0); -    } +    CHECK(queries_n < (1 << 24));      for (int i = 0; i < connections_n; i++) {        server.writer_put(static_cast<qvalue_t>(i) << 24); @@ -658,10 +641,7 @@ class QueueBenchmark2 : public td::Benchmark {      while (client_active_connections > 0) {        int cnt = client.reader_wait(); -      if (cnt == 0) { -        std::fprintf(stderr, "ERROR!\n"); -        std::exit(0); -      } +      CHECK(cnt != 0);        while (cnt-- > 0) {          client_process(client.reader_get_unsafe());          client.reader_flush(); @@ -681,7 +661,7 @@ class QueueBenchmark2 : public td::Benchmark {      return static_cast<QueueBenchmark2 *>(arg)->server_run(nullptr);    } -  void run(int n) override { +  void run(int n) final {      pthread_t client_thread_id;      pthread_t server_thread_id; @@ -696,45 +676,40 @@ class QueueBenchmark2 : public td::Benchmark {  };  template <class QueueT> -class QueueBenchmark : public td::Benchmark { +class QueueBenchmark final : public td::Benchmark {    QueueT client, server;    const int connections_n;    int queries_n; +  td::string name; +   public: -  explicit QueueBenchmark(int connections_n = 1) : connections_n(connections_n) { +  QueueBenchmark(int connections_n, td::string name) : connections_n(connections_n), name(std::move(name)) {    } -  std::string get_description() const override { -    return "QueueBenchmark"; +  td::string get_description() const final { +    return name;    } -  void start_up() override { +  void start_up() final {      client.init();      server.init();    } -  void tear_down() override { +  void tear_down() final {      client.destroy();      server.destroy();    }    void *server_run(void *) { -    vector<td::int64> conn(connections_n); +    td::vector<td::int64> conn(connections_n);      int active_connections = connections_n;      while (active_connections > 0) {        qvalue_t value = server.get();        int no = value & 0x00FFFFFF; -      int co = static_cast<int>(value >> 24); -      // std::fprintf(stderr, "-->%d %d\n", co, no); -      if (co < 0 || co >= connections_n || no != conn[co]++) { -        std::fprintf(stderr, "%d %d\n", co, no); -        std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(conn[co] - 1)); -        std::fprintf(stderr, "Server BUG\n"); -        while (true) { -        } -      } -      // std::fprintf(stderr, "no = %d/%d\n", no, queries_n); +      auto co = static_cast<int>(value >> 24); +      CHECK(co >= 0 && co < connections_n); +      CHECK(no == conn[co]++);        client.put(value);        if (no + 1 >= queries_n) {          active_connections--; @@ -744,11 +719,8 @@ class QueueBenchmark : public td::Benchmark {    }    void *client_run(void *) { -    vector<td::int64> conn(connections_n); -    if (queries_n >= (1 << 24)) { -      std::fprintf(stderr, "Too big queries_n\n"); -      std::exit(0); -    } +    td::vector<td::int64> conn(connections_n); +    CHECK(queries_n < (1 << 24));      for (int i = 0; i < connections_n; i++) {        server.put(static_cast<qvalue_t>(i) << 24);      } @@ -756,16 +728,9 @@ class QueueBenchmark : public td::Benchmark {      while (active_connections > 0) {        qvalue_t value = client.get();        int no = value & 0x00FFFFFF; -      int co = static_cast<int>(value >> 24); -      // std::fprintf(stderr, "<--%d %d\n", co, no); -      if (co < 0 || co >= connections_n || no != conn[co]++) { -        std::fprintf(stderr, "%d %d\n", co, no); -        std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(conn[co] - 1)); -        std::fprintf(stderr, "BUG\n"); -        while (true) { -        } -        std::exit(0); -      } +      auto co = static_cast<int>(value >> 24); +      CHECK(co >= 0 && co < connections_n); +      CHECK(no == conn[co]++);        if (no + 1 < queries_n) {          server.put(value + 1);        } else { @@ -777,28 +742,18 @@ class QueueBenchmark : public td::Benchmark {    }    void *client_run2(void *) { -    vector<td::int64> conn(connections_n); -    if (queries_n >= (1 << 24)) { -      std::fprintf(stderr, "Too big queries_n\n"); -      std::exit(0); -    } -    for (int it = 0; it < queries_n; it++) { +    td::vector<td::int64> conn(connections_n); +    CHECK(queries_n < (1 << 24)); +    for (int query = 0; query < queries_n; query++) {        for (int i = 0; i < connections_n; i++) { -        server.put((static_cast<td::int64>(i) << 24) + it); +        server.put((static_cast<td::int64>(i) << 24) + query);        }        for (int i = 0; i < connections_n; i++) {          qvalue_t value = client.get();          int no = value & 0x00FFFFFF; -        int co = static_cast<int>(value >> 24); -        // std::fprintf(stderr, "<--%d %d\n", co, no); -        if (co < 0 || co >= connections_n || no != conn[co]++) { -          std::fprintf(stderr, "%d %d\n", co, no); -          std::fprintf(stderr, "expected %d %lld\n", co, static_cast<long long>(conn[co] - 1)); -          std::fprintf(stderr, "BUG\n"); -          while (true) { -          } -          std::exit(0); -        } +        auto co = static_cast<int>(value >> 24); +        CHECK(co >= 0 && co < connections_n); +        CHECK(no == conn[co]++);        }      }      // system("cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"); @@ -813,7 +768,7 @@ class QueueBenchmark : public td::Benchmark {      return static_cast<QueueBenchmark *>(arg)->server_run(nullptr);    } -  void run(int n) override { +  void run(int n) final {      pthread_t client_thread_id;      pthread_t server_thread_id; @@ -828,8 +783,8 @@ class QueueBenchmark : public td::Benchmark {  };  template <class QueueT> -class RingBenchmark : public td::Benchmark { -  enum { QN = 504 }; +class RingBenchmark final : public td::Benchmark { +  static constexpr int QN = 504;    struct Thread {      int int_id; @@ -840,7 +795,6 @@ class RingBenchmark : public td::Benchmark {      void *run() {        qvalue_t value; -      // std::fprintf(stderr, "start %d\n", int_id);        do {          int cnt = queue.reader_wait();          CHECK(cnt == 1); @@ -861,7 +815,7 @@ class RingBenchmark : public td::Benchmark {      return static_cast<Thread *>(arg)->run();    } -  void start_up() override { +  void start_up() final {      for (int i = 0; i < QN; i++) {        q[i].int_id = i;        q[i].queue.init(); @@ -869,18 +823,17 @@ class RingBenchmark : public td::Benchmark {      }    } -  void tear_down() override { +  void tear_down() final {      for (int i = 0; i < QN; i++) {        q[i].queue.destroy();      }    } -  void run(int n) override { +  void run(int n) final {      for (int i = 0; i < QN; i++) {        pthread_create(&q[i].id, nullptr, run_gateway, &q[i]);      } -    std::fprintf(stderr, "run %d\n", n);      if (n < 1000) {        n = 1000;      } @@ -892,52 +845,89 @@ class RingBenchmark : public td::Benchmark {      }    }  }; +#endif + +/* +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED +static void test_queue() { +  td::vector<td::thread> threads; +  static constexpr size_t THREAD_COUNT = 100; +  td::vector<td::MpscPollableQueue<int>> queues(THREAD_COUNT); +  for (auto &q : queues) { +    q.init(); +  } +  for (size_t i = 0; i < THREAD_COUNT; i++) { +    threads.emplace_back([&q = queues[i]] { +      while (true) { +        auto got = q.reader_wait_nonblock(); +        while (got-- > 0) { +          q.reader_get_unsafe(); +        } +        q.reader_get_event_fd().wait(1000); +      } +    }); +  } + +  for (size_t iter = 0; iter < THREAD_COUNT; iter++) { +    td::usleep_for(100); +    for (int i = 0; i < 5; i++) { +      queues[td::Random::fast(0, THREAD_COUNT - 1)].writer_put(1); +    } +  } + +  for (size_t i = 0; i < THREAD_COUNT; i++) { +    threads[i].join(); +  } +} +#endif +*/  int main() { -  SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); -#define BENCH_Q2(Q, N)                      \ -  std::fprintf(stderr, "!%s %d:\t", #Q, N); \ -  td::bench(QueueBenchmark2<Q>(N)); -#define BENCH_Q(Q, N)                      \ -  std::fprintf(stderr, "%s %d:\t", #Q, N); \ -  td::bench(QueueBenchmark<Q>(N)); - -#define BENCH_R(Q)                   \ -  std::fprintf(stderr, "%s:\t", #Q); \ -  td::bench(RingBenchmark<Q>()); +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED +  // test_queue(); +#endif + +#if TD_PORT_POSIX    // TODO: yield makes it extremely slow. Yet some backoff may be necessary. -  //  BENCH_R(SemQueue); -  //  BENCH_R(td::PollQueue<qvalue_t>); +  // td::bench(RingBenchmark<SemQueue>()); +  // td::bench(RingBenchmark<td::PollQueue<qvalue_t>>()); -  BENCH_Q2(td::PollQueue<qvalue_t>, 1); -  BENCH_Q2(td::MpscPollableQueue<qvalue_t>, 1); -  BENCH_Q2(td::PollQueue<qvalue_t>, 100); -  BENCH_Q2(td::MpscPollableQueue<qvalue_t>, 100); -  BENCH_Q2(td::PollQueue<qvalue_t>, 10); -  BENCH_Q2(td::MpscPollableQueue<qvalue_t>, 10); +#define BENCH_Q2(Q, N) td::bench(QueueBenchmark2<Q<qvalue_t>>(N, #Q "(" #N ")")) -  BENCH_Q(VarQueue, 1); -  // BENCH_Q(FdQueue, 1); -  // BENCH_Q(BufferedFdQueue, 1); -  BENCH_Q(PipeQueue, 1); -  BENCH_Q(SemCheatQueue, 1); -  BENCH_Q(SemQueue, 1); +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED +  BENCH_Q2(td::InfBackoffQueue, 1); +  BENCH_Q2(td::MpscPollableQueue, 1); +  BENCH_Q2(td::PollQueue, 1); + +  BENCH_Q2(td::InfBackoffQueue, 10); +  BENCH_Q2(td::MpscPollableQueue, 10); +  BENCH_Q2(td::PollQueue, 10); -  // BENCH_Q2(td::PollQueue<qvalue_t>, 100); -  // BENCH_Q2(td::PollQueue<qvalue_t>, 10); -  // BENCH_Q2(td::PollQueue<qvalue_t>, 4); -  // BENCH_Q2(td::InfBackoffQueue<qvalue_t>, 100); +  BENCH_Q2(td::InfBackoffQueue, 100); +  BENCH_Q2(td::MpscPollableQueue, 100); +  BENCH_Q2(td::PollQueue, 100); -  // BENCH_Q2(td::InfBackoffQueue<qvalue_t>, 1); -  // BENCH_Q(SemCheatQueue, 1); +  BENCH_Q2(td::PollQueue, 4); +  BENCH_Q2(td::PollQueue, 10); +  BENCH_Q2(td::PollQueue, 100); +#endif -  // BENCH_Q(BufferedFdQueue, 100); -  // BENCH_Q(BufferedFdQueue, 10); +#define BENCH_Q(Q, N) td::bench(QueueBenchmark<Q>(N, #Q "(" #N ")")) -  // BENCH_Q(BufferQueue, 4); -  // BENCH_Q(BufferQueue, 100); -  // BENCH_Q(BufferQueue, 10); -  // BENCH_Q(BufferQueue, 1); +#if TD_LINUX +  BENCH_Q(BufferQueue, 1); +  BENCH_Q(BufferedFdQueue, 1); +  BENCH_Q(FdQueue, 1); +#endif +  BENCH_Q(PipeQueue, 1); +  BENCH_Q(SemCheatQueue, 1); +  BENCH_Q(SemQueue, 1); +  BENCH_Q(VarQueue, 1); -  return 0; +#if TD_LINUX +  BENCH_Q(BufferQueue, 4); +  BENCH_Q(BufferQueue, 10); +  BENCH_Q(BufferQueue, 100); +#endif +#endif  } diff --git a/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp b/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp index 91e957a501..c3298327e5 100644 --- a/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/bench_tddb.cpp @@ -1,70 +1,78 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  //  #include "td/telegram/DialogId.h" +#include "td/telegram/MessageDb.h"  #include "td/telegram/MessageId.h" -#include "td/telegram/MessagesDb.h" +#include "td/telegram/NotificationId.h" +#include "td/telegram/ServerMessageId.h"  #include "td/telegram/UserId.h" +#include "td/db/DbKey.h" +#include "td/db/SqliteConnectionSafe.h" +#include "td/db/SqliteDb.h" + +#include "td/actor/ConcurrentScheduler.h" +  #include "td/utils/benchmark.h"  #include "td/utils/buffer.h"  #include "td/utils/common.h"  #include "td/utils/logging.h" +#include "td/utils/Promise.h"  #include "td/utils/Random.h"  #include "td/utils/Status.h"  #include <memory> -namespace td { - -static Status init_db(SqliteDb &db) { +static td::Status init_db(td::SqliteDb &db) {    TRY_STATUS(db.exec("PRAGMA encoding=\"UTF-8\""));    TRY_STATUS(db.exec("PRAGMA synchronous=NORMAL"));    TRY_STATUS(db.exec("PRAGMA journal_mode=WAL"));    TRY_STATUS(db.exec("PRAGMA temp_store=MEMORY"));    TRY_STATUS(db.exec("PRAGMA secure_delete=1")); -  return Status::OK(); +  return td::Status::OK();  } -class MessagesDbBench : public Benchmark { +class MessageDbBench final : public td::Benchmark {   public: -  string get_description() const override { -    return "MessagesDb"; +  td::string get_description() const final { +    return "MessageDb";    } -  void start_up() override { +  void start_up() final {      LOG(ERROR) << "START UP";      do_start_up().ensure();      scheduler_->start();    } -  void run(int n) override { -    auto guard = scheduler_->get_current_guard(); +  void run(int n) final { +    auto guard = scheduler_->get_main_guard();      for (int i = 0; i < n; i += 20) { -      auto dialog_id = DialogId{UserId{Random::fast(1, 100)}}; -      auto message_id_raw = Random::fast(1, 100000); +      auto dialog_id = td::DialogId(td::UserId(static_cast<td::int64>(td::Random::fast(1, 100)))); +      auto message_id_raw = td::Random::fast(1, 100000);        for (int j = 0; j < 20; j++) { -        auto message_id = MessageId{ServerMessageId{message_id_raw + j}}; -        auto unique_message_id = ServerMessageId{i + 1}; -        auto sender_user_id = UserId{Random::fast(1, 1000)}; +        auto message_id = td::MessageId{td::ServerMessageId{message_id_raw + j}}; +        auto unique_message_id = td::ServerMessageId{i + 1}; +        auto sender_dialog_id = td::DialogId(td::UserId(static_cast<td::int64>(td::Random::fast(1, 1000))));          auto random_id = i + 1;          auto ttl_expires_at = 0; -        auto data = BufferSlice(Random::fast(100, 299)); +        auto data = td::BufferSlice(td::Random::fast(100, 299));          // use async on same thread. -        messages_db_async_->add_message({dialog_id, message_id}, unique_message_id, sender_user_id, random_id, -                                        ttl_expires_at, 0, 0, "", std::move(data), Promise<>()); +        message_db_async_->add_message({dialog_id, message_id}, unique_message_id, sender_dialog_id, random_id, +                                       ttl_expires_at, 0, 0, "", td::NotificationId(), td::MessageId(), std::move(data), +                                       td::Promise<>());        }      }    } -  void tear_down() override { +  void tear_down() final {      scheduler_->run_main(0.1);      { -      auto guard = scheduler_->get_current_guard(); +      auto guard = scheduler_->get_main_guard();        sql_connection_.reset(); -      messages_db_sync_safe_.reset(); -      messages_db_async_.reset(); +      message_db_sync_safe_.reset(); +      message_db_async_.reset();      }      scheduler_->finish(); @@ -73,36 +81,33 @@ class MessagesDbBench : public Benchmark {    }   private: -  std::unique_ptr<td::ConcurrentScheduler> scheduler_; -  std::shared_ptr<SqliteConnectionSafe> sql_connection_; -  std::shared_ptr<MessagesDbSyncSafeInterface> messages_db_sync_safe_; -  std::shared_ptr<MessagesDbAsyncInterface> messages_db_async_; +  td::unique_ptr<td::ConcurrentScheduler> scheduler_; +  std::shared_ptr<td::SqliteConnectionSafe> sql_connection_; +  std::shared_ptr<td::MessageDbSyncSafeInterface> message_db_sync_safe_; +  std::shared_ptr<td::MessageDbAsyncInterface> message_db_async_; -  Status do_start_up() { -    scheduler_ = std::make_unique<ConcurrentScheduler>(); -    scheduler_->init(1); +  td::Status do_start_up() { +    scheduler_ = td::make_unique<td::ConcurrentScheduler>(1, 0); -    auto guard = scheduler_->get_current_guard(); +    auto guard = scheduler_->get_main_guard(); -    string sql_db_name = "testdb.sqlite"; -    sql_connection_ = std::make_shared<SqliteConnectionSafe>(sql_db_name); +    td::string sql_db_name = "testdb.sqlite"; +    sql_connection_ = std::make_shared<td::SqliteConnectionSafe>(sql_db_name, td::DbKey::empty());      auto &db = sql_connection_->get();      TRY_STATUS(init_db(db));      db.exec("BEGIN TRANSACTION").ensure();      // version == 0 ==> db will be destroyed -    TRY_STATUS(init_messages_db(db, 0)); +    TRY_STATUS(init_message_db(db, 0));      db.exec("COMMIT TRANSACTION").ensure(); -    messages_db_sync_safe_ = create_messages_db_sync(sql_connection_); -    messages_db_async_ = create_messages_db_async(messages_db_sync_safe_, 0); -    return Status::OK(); +    message_db_sync_safe_ = td::create_message_db_sync(sql_connection_); +    message_db_async_ = td::create_message_db_async(message_db_sync_safe_, 0); +    return td::Status::OK();    }  }; -}  // namespace td  int main() {    SET_VERBOSITY_LEVEL(VERBOSITY_NAME(WARNING)); -  bench(td::MessagesDbBench()); -  return 0; +  td::bench(MessageDbBench());  } diff --git a/protocols/Telegram/tdlib/td/benchmark/check_proxy.cpp b/protocols/Telegram/tdlib/td/benchmark/check_proxy.cpp new file mode 100644 index 0000000000..0a54c598c0 --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/check_proxy.cpp @@ -0,0 +1,193 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/Client.h" +#include "td/telegram/td_api.h" + +#include "td/utils/base64.h" +#include "td/utils/common.h" +#include "td/utils/filesystem.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/TsCerr.h" + +#include <algorithm> +#include <cstdlib> +#include <iostream> +#include <utility> + +static void usage() { +  td::TsCerr() << "Tests specified MTProto-proxies, outputs working proxies to stdout; exits with code 0 if a working " +                  "proxy was found.\n"; +  td::TsCerr() << "Usage: check_proxy [options] server:port:secret [server2:port2:secret2 ...]\n"; +  td::TsCerr() << "Options:\n"; +  td::TsCerr() << "  -v<N>\tSet verbosity level to N\n"; +  td::TsCerr() << "  -h/--help\tDisplay this information\n"; +  td::TsCerr() << "  -d/--dc-id\tIdentifier of a datacenter to which try to connect (default is 2)\n"; +  td::TsCerr() << "  -l/--proxy-list\tName of a file with proxies to check; one proxy per line\n"; +  td::TsCerr() << "  -t/--timeout\tMaximum overall timeout for the request (default is 10 seconds)\n"; +  std::exit(2); +} + +int main(int argc, char **argv) { +  int new_verbosity_level = VERBOSITY_NAME(FATAL); + +  td::vector<std::pair<td::string, td::td_api::object_ptr<td::td_api::testProxy>>> requests; + +  auto add_proxy = [&requests](td::string arg) { +    if (arg.empty()) { +      return; +    } + +    std::size_t offset = 0; +    if (arg[0] == '[') { +      auto end_ipv6_pos = arg.find(']'); +      if (end_ipv6_pos == td::string::npos) { +        td::TsCerr() << "Error: failed to find end of IPv6 address in \"" << arg << "\"\n"; +        usage(); +      } +      offset = end_ipv6_pos; +    } +    if (std::count(arg.begin() + offset, arg.end(), ':') == 3) { +      auto secret_domain_pos = arg.find(':', arg.find(':', offset) + 1) + 1; +      auto domain_pos = arg.find(':', secret_domain_pos); +      auto secret = arg.substr(secret_domain_pos, domain_pos - secret_domain_pos); +      auto domain = arg.substr(domain_pos + 1); +      auto r_decoded_secret = td::hex_decode(secret); +      if (r_decoded_secret.is_error()) { +        r_decoded_secret = td::base64url_decode(secret); +        if (r_decoded_secret.is_error()) { +          td::TsCerr() << "Error: failed to find proxy port and secret in \"" << arg << "\"\n"; +          usage(); +        } +      } +      arg = arg.substr(0, secret_domain_pos) + td::base64url_encode(r_decoded_secret.ok() + domain); +    } + +    auto secret_pos = arg.rfind(':'); +    if (secret_pos == td::string::npos) { +      td::TsCerr() << "Error: failed to find proxy port and secret in \"" << arg << "\"\n"; +      usage(); +    } +    auto secret = arg.substr(secret_pos + 1); +    auto port_pos = arg.substr(0, secret_pos).rfind(':'); +    if (port_pos == td::string::npos) { +      td::TsCerr() << "Error: failed to find proxy secret in \"" << arg << "\"\n"; +      usage(); +    } +    auto r_port = td::to_integer_safe<td::int32>(arg.substr(port_pos + 1, secret_pos - port_pos - 1)); +    if (r_port.is_error()) { +      td::TsCerr() << "Error: failed to parse proxy port in \"" << arg << "\"\n"; +      usage(); +    } +    auto port = r_port.move_as_ok(); +    auto server = arg.substr(0, port_pos); +    if (server[0] == '[' && server.back() == ']') { +      server = server.substr(1, server.size() - 2); +    } + +    if (server.empty() || port <= 0 || port > 65536 || secret.empty()) { +      td::TsCerr() << "Error: proxy address to check is in wrong format: \"" << arg << "\"\n"; +      usage(); +    } + +    requests.emplace_back(arg, +                          td::td_api::make_object<td::td_api::testProxy>( +                              server, port, td::td_api::make_object<td::td_api::proxyTypeMtproto>(secret), -1, -1)); +  }; + +  td::int32 dc_id = 2; +  double timeout = 10.0; + +  for (int i = 1; i < argc; i++) { +    td::string arg(argv[i]); + +    auto get_next_arg = [&i, &arg, argc, argv](bool is_optional = false) { +      CHECK(arg.size() >= 2); +      if (arg.size() == 2 || arg[1] == '-') { +        if (i + 1 < argc && argv[i + 1][0] != '-') { +          return td::string(argv[++i]); +        } +      } else { +        if (arg.size() > 2) { +          return arg.substr(2); +        } +      } +      if (!is_optional) { +        td::TsCerr() << "Error: value is required after " << arg << "\n"; +        usage(); +      } +      return td::string(); +    }; + +    if (td::begins_with(arg, "-v")) { +      arg = get_next_arg(true); +      int new_verbosity = 1; +      while (arg[0] == 'v') { +        new_verbosity++; +        arg = arg.substr(1); +      } +      if (!arg.empty()) { +        new_verbosity += td::to_integer<int>(arg) - (new_verbosity == 1); +      } +      new_verbosity_level = VERBOSITY_NAME(FATAL) + new_verbosity; +    } else if (td::begins_with(arg, "-t") || arg == "--timeout") { +      timeout = td::to_double(get_next_arg()); +    } else if (td::begins_with(arg, "-d") || arg == "--dc-id") { +      dc_id = td::to_integer<td::int32>(get_next_arg()); +    } else if (td::begins_with(arg, "-l") || arg == "--proxy-list") { +      auto r_proxies = td::read_file_str(get_next_arg()); +      if (r_proxies.is_error()) { +        td::TsCerr() << "Error: wrong file name specified\n"; +        usage(); +      } +      for (auto &proxy : td::full_split(r_proxies.ok(), '\n')) { +        add_proxy(td::trim(proxy)); +      } +    } else if (arg[0] == '-') { +      usage(); +    } else { +      add_proxy(arg); +    } +  } + +  if (requests.empty()) { +    td::TsCerr() << "Error: proxy address to check is not specified\n"; +    usage(); +  } + +  SET_VERBOSITY_LEVEL(new_verbosity_level); + +  td::ClientManager client_manager; +  auto client_id = client_manager.create_client_id(); +  for (size_t i = 0; i < requests.size(); i++) { +    auto &request = requests[i].second; +    request->dc_id_ = dc_id; +    request->timeout_ = timeout; +    client_manager.send(client_id, i + 1, std::move(request)); +  } +  size_t successful_requests = 0; +  size_t failed_requests = 0; + +  while (successful_requests + failed_requests != requests.size()) { +    auto response = client_manager.receive(100.0); +    CHECK(client_id == response.client_id); +    if (1 <= response.request_id && response.request_id <= requests.size()) { +      auto &proxy = requests[static_cast<size_t>(response.request_id - 1)].first; +      if (response.object->get_id() == td::td_api::error::ID) { +        LOG(ERROR) << proxy << ": " << to_string(response.object); +        failed_requests++; +      } else { +        std::cout << proxy << std::endl; +        successful_requests++; +      } +    } +  } + +  if (successful_requests == 0) { +    return 1; +  } +} diff --git a/protocols/Telegram/tdlib/td/benchmark/check_tls.cpp b/protocols/Telegram/tdlib/td/benchmark/check_tls.cpp new file mode 100644 index 0000000000..f7fb1fecb7 --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/check_tls.cpp @@ -0,0 +1,336 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/BigNum.h" +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/detail/Iocp.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/port/sleep.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/port/thread.h" +#include "td/utils/Random.h" +#include "td/utils/Slice.h" +#include "td/utils/SliceBuilder.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" + +#include <map> + +static td::BigNumContext context; + +static bool is_quadratic_residue(const td::BigNum &a) { +  // 2^255 - 19 +  td::BigNum mod = +      td::BigNum::from_hex("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed").move_as_ok(); +  // (mod - 1) / 2 = 2^254 - 10 +  td::BigNum pow = +      td::BigNum::from_hex("3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6").move_as_ok(); + +  td::BigNum r; +  td::BigNum::mod_exp(r, a, pow, mod, context); +  td::BigNum one = td::BigNum::from_decimal("1").move_as_ok(); +  td::BigNum::mod_add(r, r, one, mod, context); + +  td::string result = r.to_decimal(); +  CHECK(result == "0" || result == "1" || result == "2"); +  return result == "2"; +} + +struct TlsInfo { +  td::vector<size_t> extension_list; +  td::vector<size_t> encrypted_application_data_length; +}; + +td::Result<TlsInfo> test_tls(const td::string &url) { +  td::IPAddress address; +  TRY_STATUS(address.init_host_port(url, 443)); +  TRY_RESULT(socket, td::SocketFd::open(address)); + +  td::string request; + +  auto add_string = [&](td::Slice data) { +    request.append(data.data(), data.size()); +  }; +  auto add_random = [&](size_t length) { +    while (length-- > 0) { +      request += static_cast<char>(td::Random::secure_int32()); +    } +  }; +  auto add_length = [&](size_t length) { +    request += static_cast<char>(length / 256); +    request += static_cast<char>(length % 256); +  }; +  auto add_key = [&] { +    td::string key(32, '\0'); +    td::BigNum mod = +        td::BigNum::from_hex("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed").move_as_ok(); +    while (true) { +      td::Random::secure_bytes(key); +      key[31] = static_cast<char>(key[31] & 127); +      td::BigNum x = td::BigNum::from_le_binary(key); +      if (!is_quadratic_residue(x)) { +        continue; +      } + +      td::BigNum y = x.clone(); +      td::BigNum coef = td::BigNum::from_decimal("486662").move_as_ok(); +      td::BigNum::mod_add(y, y, coef, mod, context); +      td::BigNum::mod_mul(y, y, x, mod, context); +      td::BigNum one = td::BigNum::from_decimal("1").move_as_ok(); +      td::BigNum::mod_add(y, y, one, mod, context); +      td::BigNum::mod_mul(y, y, x, mod, context); +      // y = x^3 + 486662 * x^2 + x +      if (is_quadratic_residue(y)) { +        break; +      } +    } +    request += key; +  }; + +  const size_t MAX_GREASE = 7; +  char greases[MAX_GREASE]; +  td::Random::secure_bytes(td::MutableSlice{greases, MAX_GREASE}); +  for (auto &grease : greases) { +    grease = static_cast<char>((grease & 0xF0) + 0x0A); +  } +  for (size_t i = 1; i < MAX_GREASE; i += 2) { +    if (greases[i] == greases[i - 1]) { +      greases[i] = static_cast<char>(0x10 ^ greases[i]); +    } +  } +  auto add_grease = [&](size_t num) { +    auto c = greases[num]; +    request += c; +    request += c; +  }; + +  add_string("\x16\x03\x01\x02\x00\x01\x00\x01\xfc\x03\x03"); +  add_random(32); +  add_string("\x20"); +  add_random(32); +  add_string("\x00\x20"); +  add_grease(0); +  add_string( +      "\x13\x01\x13\x02\x13\x03\xc0\x2b\xc0\x2f\xc0\x2c\xc0\x30\xcc\xa9\xcc\xa8\xc0\x13\xc0\x14\x00\x9c\x00\x9d\x00" +      "\x2f\x00\x35\x01\x00\x01\x93"); +  add_grease(2); +  add_string("\x00\x00\x00\x00"); +  add_length(url.size() + 5); +  add_length(url.size() + 3); +  add_string("\x00"); +  add_length(url.size()); +  add_string(url); +  add_string("\x00\x17\x00\x00\xff\x01\x00\x01\x00\x00\x0a\x00\x0a\x00\x08"); +  add_grease(4); +  add_string( +      "\x00\x1d\x00\x17\x00\x18\x00\x0b\x00\x02\x01\x00\x00\x23\x00\x00\x00\x10\x00\x0e\x00\x0c\x02\x68\x32\x08\x68" +      "\x74\x74\x70\x2f\x31\x2e\x31\x00\x05\x00\x05\x01\x00\x00\x00\x00\x00\x0d\x00\x12\x00\x10\x04\x03\x08\x04\x04" +      "\x01\x05\x03\x08\x05\x05\x01\x08\x06\x06\x01\x00\x12\x00\x00\x00\x33\x00\x2b\x00\x29"); +  add_grease(4); +  add_string("\x00\x01\x00\x00\x1d\x00\x20"); +  add_key(); +  add_string("\x00\x2d\x00\x02\x01\x01\x00\x2b\x00\x0b\x0a"); +  add_grease(6); +  add_string("\x03\x04\x03\x03\x03\x02\x03\x01\x00\x1b\x00\x03\x02\x00\x02"); +  add_grease(3); +  add_string("\x00\x01\x00\x00\x15"); +  auto padding = 515 - static_cast<int>(request.size()); +  CHECK(padding >= 0); +  add_length(padding); +  request.resize(517); + +  // LOG(ERROR) << td::format::as_hex_dump<0>(td::Slice(request)); + +  TRY_STATUS(socket.write(request)); + +  TlsInfo info; +  auto end_time = td::Time::now() + 3; +  td::string result; +  size_t pos = 0; +  size_t server_hello_length = 0; +  size_t encrypted_application_data_length_sum = 0; +  while (td::Time::now() < end_time) { +    static char buf[20000]; +    TRY_RESULT(res, socket.read(td::MutableSlice{buf, sizeof(buf)})); +    if (res > 0) { +      auto read_length = [&]() -> size_t { +        CHECK(result.size() >= 2 + pos); +        pos += 2; +        return static_cast<unsigned char>(result[pos - 2]) * 256 + static_cast<unsigned char>(result[pos - 1]); +      }; + +      result += td::Slice(buf, res).str(); +      while (true) { +#define CHECK_LENGTH(length)            \ +  if (pos + (length) > result.size()) { \ +    break;                              \ +  } +#define EXPECT_STR(pos, str, error)                       \ +  if (!begins_with(td::Slice(result).substr(pos), str)) { \ +    return td::Status::Error(error);                      \ +  } + +        if (pos == 0) { +          CHECK_LENGTH(3); +          EXPECT_STR(0, "\x16\x03\x03", "Non-TLS response or TLS <= 1.1"); +          pos += 3; +        } +        if (pos == 3) { +          CHECK_LENGTH(2); +          server_hello_length = read_length(); +          if (server_hello_length <= 39) { +            return td::Status::Error("Receive too short server hello"); +          } +        } +        if (server_hello_length > 0) { +          if (pos == 5) { +            CHECK_LENGTH(server_hello_length); + +            EXPECT_STR(5, "\x02\x00", "Non-TLS response 2"); +            EXPECT_STR(9, "\x03\x03", "Non-TLS response 3"); + +            auto random_id = td::Slice(result.c_str() + 11, 32); +            if (random_id == +                "\xcf\x21\xad\x74\xe5\x9a\x61\x11\xbe\x1d\x8c\x02\x1e\x65\xb8\x91\xc2\xa2\x11\x16\x7a\xbb\x8c\x5e\x07" +                "\x9e\x09\xe2\xc8\xa8\x33\x9c") { +              return td::Status::Error("TLS 1.3 servers returning HelloRetryRequest are not supprted"); +            } +            if (result[43] == '\x00') { +              return td::Status::Error("TLS <= 1.2: empty session_id"); +            } +            EXPECT_STR(43, "\x20", "Non-TLS response 4"); +            if (server_hello_length <= 75) { +              return td::Status::Error("Receive too short server hello 2"); +            } +            EXPECT_STR(44, request.substr(44, 32), "TLS <= 1.2: expected mirrored session_id"); +            EXPECT_STR(76, "\x13\x01\x00", "TLS <= 1.2: expected 0x1301 as a chosen cipher"); +            pos += 74; +            size_t extensions_length = read_length(); +            if (extensions_length + 76 != server_hello_length) { +              return td::Status::Error("Receive wrong extensions length"); +            } +            while (pos < 5 + server_hello_length - 4) { +              info.extension_list.push_back(read_length()); +              size_t extension_length = read_length(); +              if (pos + extension_length > 5 + server_hello_length) { +                return td::Status::Error("Receive wrong extension length"); +              } +              pos += extension_length; +            } +            if (pos != 5 + server_hello_length) { +              return td::Status::Error("Receive wrong extensions list"); +            } +          } +          if (pos == 5 + server_hello_length) { +            CHECK_LENGTH(6); +            EXPECT_STR(pos, "\x14\x03\x03\x00\x01\x01", "Expected dummy ChangeCipherSpec"); +            pos += 6; +          } +          if (pos == 11 + server_hello_length + encrypted_application_data_length_sum) { +            if (pos == result.size()) { +              return info; +            } + +            CHECK_LENGTH(3); +            EXPECT_STR(pos, "\x17\x03\x03", "Expected encrypted application data"); +            pos += 3; +          } +          if (pos == 14 + server_hello_length + encrypted_application_data_length_sum) { +            CHECK_LENGTH(2); +            size_t encrypted_application_data_length = read_length(); +            info.encrypted_application_data_length.push_back(encrypted_application_data_length); +            if (encrypted_application_data_length == 0) { +              return td::Status::Error("Receive empty encrypted application data"); +            } +          } +          if (pos == 16 + server_hello_length + encrypted_application_data_length_sum) { +            CHECK_LENGTH(info.encrypted_application_data_length.back()); +            pos += info.encrypted_application_data_length.back(); +            encrypted_application_data_length_sum += info.encrypted_application_data_length.back() + 5; +          } +        } +      } +    } else { +      td::usleep_for(10000); +    } +  } + +  // LOG(ERROR) << url << ":" << td::format::as_hex_dump<0>(td::Slice(result)); +  return td::Status::Error("Failed to get response in 3 seconds"); +} + +int main(int argc, char *argv[]) { +  SET_VERBOSITY_LEVEL(VERBOSITY_NAME(WARNING)); + +#if TD_PORT_WINDOWS +  auto iocp = td::make_unique<td::detail::Iocp>(); +  iocp->init(); +  auto iocp_thread = td::thread([&iocp] { iocp->loop(); }); +  td::detail::Iocp::Guard iocp_guard(iocp.get()); +#endif + +  td::vector<td::string> urls; +  for (int i = 1; i < argc; i++) { +    urls.emplace_back(argv[i]); +  } +  for (auto &url : urls) { +    const int MAX_TRIES = 100; +    td::vector<std::map<size_t, int>> length_count; +    td::vector<size_t> extension_list; +    for (int i = 0; i < MAX_TRIES; i++) { +      auto r_tls_info = test_tls(url); +      if (r_tls_info.is_error()) { +        LOG(ERROR) << url << ": " << r_tls_info.error(); +        break; +      } else { +        auto tls_info = r_tls_info.move_as_ok(); +        if (length_count.size() < tls_info.encrypted_application_data_length.size()) { +          length_count.resize(tls_info.encrypted_application_data_length.size()); +        } +        for (size_t t = 0; t < tls_info.encrypted_application_data_length.size(); t++) { +          length_count[t][tls_info.encrypted_application_data_length[t]]++; +        } +        if (i == 0) { +          extension_list = tls_info.extension_list; +        } else { +          if (extension_list != tls_info.extension_list) { +            LOG(ERROR) << url << ": TLS 1.3.0 extension list has changed from " << extension_list << " to " +                       << tls_info.extension_list; +            break; +          } +        } +      } + +      if (i == MAX_TRIES - 1) { +        if (extension_list != td::vector<size_t>{51, 43} && extension_list != td::vector<size_t>{43, 51}) { +          LOG(ERROR) << url << ": TLS 1.3.0 unsupported extension list " << extension_list; +        } else { +          td::string length_distribution = "|"; +          for (size_t t = 0; t < length_count.size(); t++) { +            for (auto it : length_count[t]) { +              length_distribution += PSTRING() +                                     << it.first << " : " << static_cast<int>(it.second * 100.0 / MAX_TRIES) << "%|"; +            } +            if (t + 1 != length_count.size()) { +              length_distribution += " + |"; +            } +          } +          LOG(ERROR) << url << ": TLS 1.3.0 with extensions " << extension_list << " and " +                     << (length_count.size() != 1 ? "unsupported " : "") +                     << "encrypted application data length distribution " << length_distribution; +        } +      } +    } +  } + +#if TD_PORT_WINDOWS +  iocp->interrupt_loop(); +  iocp_thread.join(); +#endif +} diff --git a/protocols/Telegram/tdlib/td/benchmark/hashmap_build.cpp b/protocols/Telegram/tdlib/td/benchmark/hashmap_build.cpp new file mode 100644 index 0000000000..f6e71b748a --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/hashmap_build.cpp @@ -0,0 +1,545 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/FlatHashMap.h" + +#ifdef SCOPE_EXIT +#undef SCOPE_EXIT +#endif + +#include <absl/container/flat_hash_map.h> +#include <array> +#include <folly/container/F14Map.h> +#include <map> +#include <unordered_map> + +#define test_map td::FlatHashMap +//#define test_map folly::F14FastMap +//#define test_map absl::flat_hash_map +//#define test_map std::map +//#define test_map std::unordered_map + +//#define CREATE_MAP(num) CREATE_MAP_IMPL(num) +#define CREATE_MAP(num) + +#define CREATE_MAP_IMPL(num)                      \ +  int f_##num() {                                 \ +    test_map<td::int32, std::array<char, num>> m; \ +    m.emplace(1, std::array<char, num>{});        \ +    int sum = 0;                                  \ +    for (auto &it : m) {                          \ +      sum += it.first;                            \ +    }                                             \ +    auto it = m.find(1);                          \ +    sum += it->first;                             \ +    m.erase(it);                                  \ +    return sum;                                   \ +  }                                               \ +  int x_##num = f_##num() + +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); +CREATE_MAP(__LINE__); + +int main() { +} diff --git a/protocols/Telegram/tdlib/td/benchmark/hashset_memory.cpp b/protocols/Telegram/tdlib/td/benchmark/hashset_memory.cpp new file mode 100644 index 0000000000..8f31b52114 --- /dev/null +++ b/protocols/Telegram/tdlib/td/benchmark/hashset_memory.cpp @@ -0,0 +1,193 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#if USE_MEMPROF +#include "memprof/memprof_stat.h" +#endif + +#include "td/utils/common.h" +#include "td/utils/FlatHashMap.h" +#include "td/utils/FlatHashMapChunks.h" +#include "td/utils/FlatHashTable.h" +#include "td/utils/HashTableUtils.h" +#include "td/utils/logging.h" +#include "td/utils/MapNode.h" +#include "td/utils/misc.h" +#include "td/utils/port/Stat.h" +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" + +#ifdef SCOPE_EXIT +#undef SCOPE_EXIT +#endif + +#include <absl/container/flat_hash_map.h> +#include <array> +#include <folly/container/F14Map.h> +#include <functional> +#include <map> +#include <unordered_map> + +static int mem_stat_i = -1; +static int mem_stat_cur = 0; + +static bool use_memprof() { +#if USE_MEMPROF +  return mem_stat_i < 0 && is_memprof_on(); +#else +  return mem_stat_i < 0; +#endif +} + +static td::uint64 get_memory() { +#if USE_MEMPROF +  if (use_memprof()) { +    return get_used_memory_size(); +  } +#endif +  CHECK(!use_memprof()); +  return td::mem_stat().ok().resident_size_; +} + +template <class T> +class Generator { + public: +  T next() { +    UNREACHABLE(); +  } +  static size_t dyn_size() { +    UNREACHABLE(); +  } +}; + +template <class T> +class IntGenerator { + public: +  T next() { +    return ++value; +  } +  static size_t dyn_size() { +    return 0; +  } + + private: +  T value{}; +}; + +template <> +class Generator<td::int32> final : public IntGenerator<td::int32> {}; +template <> +class Generator<td::int64> final : public IntGenerator<td::int64> {}; + +template <class T> +class Generator<td::unique_ptr<T>> { + public: +  td::unique_ptr<T> next() { +    return td::make_unique<T>(); +  } +  static std::size_t dyn_size() { +    return sizeof(T); +  } +}; + +template <class T, class KeyT, class ValueT> +static void measure(td::StringBuilder &sb, td::Slice name, td::Slice key_name, td::Slice value_name) { +  mem_stat_cur++; +  if (mem_stat_i >= 0 && mem_stat_cur != mem_stat_i) { +    return; +  } +  sb << name << "<" << key_name << "," << value_name << "> " << (use_memprof() ? "memprof" : "os") << "\n"; +  std::size_t ideal_size = sizeof(KeyT) + sizeof(ValueT) + Generator<ValueT>::dyn_size(); + +  sb << "\tempty:" << sizeof(T); +  struct Stat { +    int pi; +    double min_ratio; +    double max_ratio; +  }; +  td::vector<Stat> stat; +  stat.reserve(1024); +  for (std::size_t size : {1000000u}) { +    Generator<KeyT> key_generator; +    Generator<ValueT> value_generator; +    auto start_mem = get_memory(); +    T ht; +    auto ratio = [&] { +      auto end_mem = get_memory(); +      auto used_mem = end_mem - start_mem; +      return static_cast<double>(used_mem) / (static_cast<double>(ideal_size) * static_cast<double>(ht.size())); +    }; +    double min_ratio; +    double max_ratio; +    auto reset = [&] { +      min_ratio = 1e100; +      max_ratio = 0; +    }; +    auto update = [&] { +      auto x = ratio(); +      min_ratio = td::min(min_ratio, x); +      max_ratio = td::max(max_ratio, x); +    }; +    reset(); + +    int p = 10; +    int pi = 1; +    for (std::size_t i = 0; i < size; i++) { +      ht.emplace(key_generator.next(), value_generator.next()); +      update(); +      if ((i + 1) % p == 0) { +        stat.push_back(Stat{pi, min_ratio, max_ratio}); +        reset(); +        pi++; +        p *= 10; +      } +    } +  } +  for (auto &s : stat) { +    sb << " 10^" << s.pi << ":" << s.min_ratio << "->" << s.max_ratio; +  } +  sb << '\n'; +} + +template <std::size_t size> +using Bytes = std::array<char, size>; + +template <template <typename... Args> class T> +void print_memory_stats(td::Slice name) { +  td::string big_buff(1 << 16, '\0'); +  td::StringBuilder sb(big_buff, false); +#define MEASURE(KeyT, ValueT) measure<T<KeyT, ValueT>, KeyT, ValueT>(sb, name, #KeyT, #ValueT); +  MEASURE(td::int32, td::int32); +  MEASURE(td::int64, td::unique_ptr<Bytes<360>>); +  if (!sb.as_cslice().empty()) { +    LOG(PLAIN) << '\n' << sb.as_cslice() << '\n'; +  } +} + +template <class KeyT, class ValueT, class HashT = td::Hash<KeyT>, class EqT = std::equal_to<KeyT>> +using FlatHashMapImpl = td::FlatHashTable<td::MapNode<KeyT, ValueT>, HashT, EqT>; + +#define FOR_EACH_TABLE(F) \ +  F(FlatHashMapImpl)      \ +  F(folly::F14FastMap)    \ +  F(absl::flat_hash_map)  \ +  F(std::unordered_map)   \ +  F(std::map) +#define BENCHMARK_MEMORY(T) print_memory_stats<T>(#T); + +int main(int argc, const char *argv[]) { +  // Usage: +  //  % benchmark/memory-hashset-os 0 +  //  Number of benchmarks = 10 +  //  % for i in {1..10}; do ./benchmark/memory-hashset-os $i; done +  if (argc > 1) { +    mem_stat_i = td::to_integer<td::int32>(td::Slice(argv[1])); +  } +  FOR_EACH_TABLE(BENCHMARK_MEMORY); +  if (mem_stat_i <= 0) { +    LOG(PLAIN) << "Number of benchmarks = " << mem_stat_cur << "\n"; +  } +} diff --git a/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp b/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp index f1676baa63..d61baa0a02 100644 --- a/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/rmdir.cpp @@ -1,5 +1,5 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -14,15 +14,16 @@ int main(int argc, char *argv[]) {    }    td::CSlice dir(argv[1]);    int cnt = 0; -  auto status = td::walk_path(dir, [&](td::CSlice path, bool is_dir) { -    cnt++; -    LOG(INFO) << path << " " << is_dir; -    // if (is_dir) { +  auto status = td::walk_path(dir, [&](td::CSlice path, auto type) { +    if (type != td::WalkPath::Type::EnterDir) { +      cnt++; +      LOG(INFO) << path << " " << (type == td::WalkPath::Type::ExitDir); +    } +    //if (is_dir) {      // td::rmdir(path);      //} else {      // td::unlink(path);      //}    });    LOG(INFO) << status << ": " << cnt; -  return 0;  } diff --git a/protocols/Telegram/tdlib/td/benchmark/wget.cpp b/protocols/Telegram/tdlib/td/benchmark/wget.cpp index dba997e61f..5145f317f1 100644 --- a/protocols/Telegram/tdlib/td/benchmark/wget.cpp +++ b/protocols/Telegram/tdlib/td/benchmark/wget.cpp @@ -1,39 +1,43 @@  // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022  //  // Distributed under the Boost Software License, Version 1.0. (See accompanying  // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)  // -#include "td/actor/actor.h" -#include "td/actor/PromiseFuture.h" -  #include "td/net/HttpQuery.h"  #include "td/net/Wget.h" +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" + +#include "td/utils/common.h"  #include "td/utils/logging.h" +#include "td/utils/Promise.h"  #include "td/utils/Status.h" -#include <memory> -#include <string> -  int main(int argc, char *argv[]) { -  SET_VERBOSITY_LEVEL(VERBOSITY_NAME(INFO)); +  SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG));    td::VERBOSITY_NAME(fd) = VERBOSITY_NAME(INFO); -  std::string url = (argc > 1 ? argv[1] : "https://telegram.org"); -  auto scheduler = std::make_unique<td::ConcurrentScheduler>(); -  scheduler->init(0); +  td::string url = (argc > 1 ? argv[1] : "https://telegram.org"); +  auto timeout = 10; +  auto ttl = 3; +  auto prefer_ipv6 = (argc > 2 && td::string(argv[2]) == "-6"); +  auto scheduler = td::make_unique<td::ConcurrentScheduler>(0, 0);    scheduler -      ->create_actor_unsafe<td::Wget>(0, "Client", td::PromiseCreator::lambda([](td::Result<td::HttpQueryPtr> res) { +      ->create_actor_unsafe<td::Wget>(0, "Client", +                                      td::PromiseCreator::lambda([](td::Result<td::unique_ptr<td::HttpQuery>> res) { +                                        if (res.is_error()) { +                                          LOG(FATAL) << res.error(); +                                        }                                          LOG(ERROR) << *res.ok();                                          td::Scheduler::instance()->finish();                                        }), -                                      url) +                                      url, td::Auto(), timeout, ttl, prefer_ipv6)        .release();    scheduler->start();    while (scheduler->run_main(10)) {      // empty    }    scheduler->finish(); -  return 0;  }  | 
